POM

  1. <?xml version="1.0" encoding="UTF-8"?>
  2. <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  3. xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  4. <modelVersion>4.0.0</modelVersion>
  5. <groupId>com.twx.bigdata</groupId>
  6. <artifactId>learn-spark-streaming</artifactId>
  7. <version>1.0-SNAPSHOT</version>
  8. <name>learn-spark-streaming</name>
  9. <properties>
  10. <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
  11. <maven.compiler.source>1.8</maven.compiler.source>
  12. <maven.compiler.target>1.8</maven.compiler.target>
  13. </properties>
  14. <dependencies>
  15. <dependency>
  16. <groupId>org.apache.spark</groupId>
  17. <artifactId>spark-streaming_2.11</artifactId>
  18. <version>2.0.2</version>
  19. </dependency>
  20. <dependency>
  21. <groupId>junit</groupId>
  22. <artifactId>junit</artifactId>
  23. <version>4.11</version>
  24. <scope>test</scope>
  25. </dependency>
  26. </dependencies>
  27. <build>
  28. <plugins>
  29. <plugin>
  30. <groupId>org.scala-tools</groupId>
  31. <artifactId>maven-scala-plugin</artifactId>
  32. <version>2.15.2</version>
  33. <executions>
  34. <execution>
  35. <goals>
  36. <goal>compile</goal>
  37. <goal>testCompile</goal>
  38. </goals>
  39. </execution>
  40. </executions>
  41. </plugin>
  42. <plugin>
  43. <groupId>org.apache.maven.plugins</groupId>
  44. <artifactId>maven-compiler-plugin</artifactId>
  45. <configuration>
  46. <source>1.8</source>
  47. <target>1.8</target>
  48. </configuration>
  49. </plugin>
  50. </plugins>
  51. </build>
  52. </project>

MainApp

  1. package com.twx.bigdata.sparkstreaming
  2. import org.apache.spark.streaming.{Seconds, StreamingContext}
  3. import org.apache.spark.{SparkConf, SparkContext}
  4. /**
  5. * @author tangwx@soyuan.com.cn
  6. * @date 2020/4/7 17:59
  7. */
  8. object MainApp {
  9. def main(args: Array[String]): Unit = {
  10. val conf: SparkConf = new SparkConf()
  11. .setMaster("local[*]")
  12. .setAppName("learn-spark-streaming")
  13. val sc = new SparkContext(conf)
  14. val scc = new StreamingContext(sc,Seconds(5))
  15. val lines = scc.socketTextStream("localhost",9999)
  16. val words = lines.flatMap(_.split(" "))
  17. val wordMap = words.map(x => (x,1))
  18. val wordCounts = wordMap.reduceByKey(_+_)
  19. wordCounts.print()
  20. scc.start()
  21. scc.awaitTermination()
  22. }
  23. }