POM
<?xml version="1.0" encoding="UTF-8"?><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>com.twx.bigdata</groupId> <artifactId>learn-spark-streaming</artifactId> <version>1.0-SNAPSHOT</version> <name>learn-spark-streaming</name> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <maven.compiler.source>1.8</maven.compiler.source> <maven.compiler.target>1.8</maven.compiler.target> </properties> <dependencies> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-streaming_2.11</artifactId> <version>2.0.2</version> </dependency> <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>4.11</version> <scope>test</scope> </dependency> </dependencies> <build> <plugins> <plugin> <groupId>org.scala-tools</groupId> <artifactId>maven-scala-plugin</artifactId> <version>2.15.2</version> <executions> <execution> <goals> <goal>compile</goal> <goal>testCompile</goal> </goals> </execution> </executions> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> <configuration> <source>1.8</source> <target>1.8</target> </configuration> </plugin> </plugins> </build></project>
MainApp
package com.twx.bigdata.sparkstreamingimport org.apache.spark.streaming.{Seconds, StreamingContext}import org.apache.spark.{SparkConf, SparkContext}/** * @author tangwx@soyuan.com.cn * @date 2020/4/7 17:59 */object MainApp { def main(args: Array[String]): Unit = { val conf: SparkConf = new SparkConf() .setMaster("local[*]") .setAppName("learn-spark-streaming") val sc = new SparkContext(conf) val scc = new StreamingContext(sc,Seconds(5)) val lines = scc.socketTextStream("localhost",9999) val words = lines.flatMap(_.split(" ")) val wordMap = words.map(x => (x,1)) val wordCounts = wordMap.reduceByKey(_+_) wordCounts.print() scc.start() scc.awaitTermination() }}