1. package com.sql
    3. import org.apache.log4j.{Level, Logger}
    4. import org.apache.spark.SparkConf
    5. import org.apache.spark.sql.{DataFrame, SparkSession}
    7. object Spark_SQL_Basic {
    8.   def main(args: Array[String]): Unit = {
    9.     Logger.getLogger("org").setLevel(Level.ERROR)
    11.     //Spark环境
    12.     val conf = new SparkConf().setMaster("local[*]").setAppName(this.getClass.getName)
    13.     val sc = SparkSession.builder().config(conf).getOrCreate()
    15.     //TODO 逻辑操作
    16.     //隐式转换，sc不是包，而是spark的SparkSession类
    17.     import sc.implicits._
    19.     //DataFrame
    20.     println("-----------DataFrame常规展示----------------------")
    21.     //读取JSON数据
    22.     val df: DataFrame = sc.read.json("data/user.json")
    23.     df.show()
    24.     println("-------------DataFrame SQL展示 --------------------")
    25.     //DataFrame => SQL
    26.     df.createTempView("user")
    27.     sc.sql("select * from user").show()
    28.     println("------------DataFrame DSL展示--------------------")
    29.     //DataFrame => DSL
    30.     df.select("username","age").show()
    31.     //运算的时候，一定要加入隐式转换，
    32.     df.select($"age" + 1 as "age").show()
    33.     df.select('age + 2 as "age").show()
    36.     //DataSet
    37.     //DataFrame其实是特殊泛型的DataSet
    38.     println("---------以下为DataSet展示---------")
    39.     val seq = Seq(1,2,3,4)
    40.     val ds = seq.toDS()
    41.     ds.show()
    44.     //RDD <=> DataFrame
    45.     println("---------以下为RDD <=> DataFrame展示---------")
    46.     val list = List((1, "huanhuan",18), (2, "haoge",19))
    47.     val rdd = sc.sparkContext.makeRDD(list)
    48.     val df1: DataFrame = rdd.toDF("id", "name", "age")
    49.     df1.show()
    50.     val rowRDD = df1.rdd
    51.     rowRDD.collect().foreach(println)
    54.     //DataSet <=> DataFrame
    55.     println("---------DataSet <=> DataFrame展示---------")
    56.     val ds1 = df1.as[User]
    57.     ds1.show()
    58.     val df2 = ds1.toDF()
    59.     df2.show(1) //只展示一个
    61.     //RDD <=> DataSet
    62.     println("---------DataSet <=> RDD展示---------")
    63.     //RDD转换DataSet 尽量用模式匹配。
    64.     val df3 = rdd.map {
    65.       case (id, name, age) => {
    66.         User(id, name, age)
    67.       }
    68.     }.toDS()
    70.     df3.show()
    72.     val rdd1 = df3.rdd
    73.     rdd1.collect().foreach(println)
    75.     //停止Spark
    76.     sc.stop()
    77.   }
    79.   case class User(id:Int,name:String,age:Int)
    80. }