宋明的一些记录 引入的依赖

    1. //如果要用MLLib和SparkHive的话还要加相应的libraryDependencies
    2. //宋明（Spark） 2015-07-28 17:36:24 
    4. // for breeze
    5. libraryDependencies += "org.scalanlp" %% "breeze" % "0.10" % "provided"
    7. // for breeze
    8. libraryDependencies += "org.scalanlp" %% "breeze-natives" % "0.10" % "provided"
    10. libraryDependencies ++= Seq(
    11.   ("org.apache.spark" %% "spark-graphx" % "1.2.1" % "provided").
    12.   exclude("org.eclipse.jetty.orbit","javax.transaction").
    13.   exclude("org.eclipse.jetty.orbit","javax.servlet").
    14.   exclude("org.eclipse.jetty.orbit","javax.mail.glassfish").
    15.   exclude("com.esotericsoftware.kryo","kryo").
    16.   exclude("commons-beanutils","commons-beanutils").
    17.   exclude("commons-collections","commons-collections")
    18. )
    20. libraryDependencies ++= Seq(
    21.   ("org.apache.spark" %% "spark-mllib" % "1.2.1" % "provided").
    22.   exclude("org.eclipse.jetty.orbit","javax.transaction").
    23.   exclude("org.eclipse.jetty.orbit","javax.servlet").
    24.   exclude("org.eclipse.jetty.orbit","javax.mail.glassfish").
    25.   exclude("com.esotericsoftware.kryo","kryo").
    26.   exclude("commons-beanutils","commons-beanutils").
    27.   exclude("commons-collections","commons-collections")
    28. )
    30. libraryDependencies ++= Seq(
    31.   ("org.apache.spark" %% "spark-core" % "1.2.1" % "provided").
    32.   exclude("org.eclipse.jetty.orbit","javax.servlet").
    33.   exclude("org.eclipse.jetty.orbit","javax.mail.glassfish").
    34.   exclude("org.eclipse.jetty.orbit","javax.activation").
    35.   exclude("com.esotericsoftware.kryo","kryo").
    36.   exclude("commons-beanutils","commons-beanutils-core").
    37.   exclude("commons-beanutils","commons-beanutils")
    38. )
    40. // libraryDependencies += "org.apache.spark" %% "spark-sql" % "1.0.0" % "provided"
    41. libraryDependencies += "org.apache.spark" %% "spark-hive" % "1.0.0" % "provided"

    没有hsql的情况处理日志 没什么意义

    1. hdfs:///home/hdp-like/channel/online/pay_data/2015-06-20/04_pay_channel.log
    3. val sqlContext = new org.apache.spark.sql.SQLContext(sc)
    4. import sqlContext.implicits._
    5. //case class FirstLogin(qid: String, channel: String, app_key:String, login_date: String)
    6. case class FirstLogin(qid: String)
    7. //val people = sc.textFile("hdfs:///home/hdp-like/channel/online/pay_data").map(_.split("\t")).map(p => FirstLogin(p(0), p(1),p(2),p(3))).toDF()
    8. //val rdd = sc.textFile("hdfs:///home/hdp-like/zhushou/db/first_login_detail_qid_channel_appkey/*").map(_.split("\t")).map(p => FirstLogin(p(0), p(1),p(2),p(3))).toDF()
    9. val rdd = sc.textFile("hdfs:///home/hdp-like/zhushou/db/first_login_detail_qid_channel_appkey/*").map(_.split("\t")).map(p => FirstLogin(p(0))).toDF()
    10. rdd.registerTempTable("table_test")
    11. val res = sqlContext.sql("SELECT count(1) FROM table_test")
    12. teenagers.map(t => "Name: " + t(0)).collect().foreach(println)
    14. // or by field name:
    15. teenagers.map(t => "Name: " + t.getAs[String]("name")).collect().foreach(println)
    17. // row.getValuesMap[T] retrieves multiple columns at once into a Map[String, T]
    18. teenagers.map(_.getValuesMap[Any](List("name", "age"))).collect().foreach(println)
    19. // Map("name" -> "Justin", "age" -> 19)