1. package com.learn.wordcount;
    2. import org.apache.hadoop.io.IntWritable;
    3. import org.apache.hadoop.io.LongWritable;
    4. import org.apache.hadoop.io.Text;
    5. import org.apache.hadoop.mapreduce.Mapper;
    6. import java.io.IOException;
    7. import org.apache.hadoop.conf.Configuration;
    8. import org.apache.hadoop.fs.Path;
    9. import org.apache.hadoop.io.IntWritable;
    10. import org.apache.hadoop.io.Text;
    11. import org.apache.hadoop.mapreduce.Job;
    12. import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
    13. import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
    14. /**
    15.  * Mapper<输入key类型, 输入value类型, 输出key类型, 输出value类型>
    16.  */
    17. public class MyMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
    18.     @Override
    19.     protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
    20.         String[] arr = value.toString().split(" ");
    21.         for (String s : arr) {
    22.             context.write(new Text(s),new IntWritable(1));
    23.         }
    24.     }
    25. }
    27. /**
    28.  * Reducer<Text, IntWritable,Text,IntWritable>
    29.  */
    30. public class MyReducer extends Reducer<Text, IntWritable,Text,IntWritable> {
    31.     @Override
    32.     protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
    33.         int count = 0;
    34.         for (IntWritable value : values) {
    35.             count += value.get();
    36.         }
    37.         context.write(key,new IntWritable(count));
    38.     }
    39. }
    41. /**
    42.  * Job的配置类
    43.  */
    44. public class WordCount extends Configuration {
    45.     public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
    47.         String inputFile = "C:\\Users\\pg\\Desktop\\input.txt";
    48.         String inputFile2 = "C:\\Users\\pg\\Desktop\\input2.txt";
    49.         String outputFile = "C:\\Users\\pg\\Desktop\\out.txt";
    51.         //获取Job对象，组装8个步骤所需要的类
    52.         Configuration conf = new Configuration();
    53.         Job job = Job.getInstance(conf, "WordCount");
    54.         job.setJarByClass(WordCount.class);
    56.         //第一步：设置读取文件的InputFormat,解析成key,value对，k1、v1
    57.         job.setInputFormatClass(TextInputFormat.class);
    58.         TextInputFormat.addInputPath(job,new Path(inputFile,inputFile2));
    60.         //第二步：自定义map逻辑，接收kv对进行处理输出新的kv对，k2、v2
    61.         job.setMapperClass(MyMapper.class);
    63.         //第三步：分区，相同key(k2)的数据发往同一个reduce里面去，key进行合并，value形成一个集合
    64.         job.setPartitionerClass(HashPartitioner.class);
    65.         job.setNumReduceTasks(2);//设置reduceTask数量
    66.         //第四步：排序，对k2进行排序，按照字典排序
    67.         //第五步：规约，combiner过程，调优步骤，可选
    68.         job.setCombinerClass(MyReducer.class);
    69.         //第六步：分组
    70.         job.setGroupingComparatorClass(WritableComparator.class);
    71.         //TODO 这四步是有默认的策略
    73.         //第七步：自定义reducer逻辑接收k2、v2，转换成新的k3、v3输出
    74.         job.setReducerClass(MyReducer.class);
    75.         job.setOutputKeyClass(Text.class);
    76.         job.setOutputValueClass(IntWritable.class);
    78.         //第八步：输出k3、v3进行保存
    79.         job.setOutputFormatClass(TextOutputFormat.class);
    80.         TextOutputFormat.setOutputPath(job,new Path(outputFile));
    82.         //第九步：结束job
    83.         System.exit(job.waitForCompletion(true) ? 0 : 1);
    84.     }
    85. }

    pom.xml

    1. <?xml version="1.0" encoding="UTF-8"?>
    2. <project xmlns="http://maven.apache.org/POM/4.0.0"
    3.          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    4.          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    5.     <modelVersion>4.0.0</modelVersion>
    7.     <groupId>com.learn</groupId>
    8.     <artifactId>HadoopLearn</artifactId>
    9.     <version>1.0-SNAPSHOT</version>
    11.     <dependencies>
    12.         <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
    13.         <dependency>
    14.             <groupId>org.apache.hadoop</groupId>
    15.             <artifactId>hadoop-common</artifactId>
    16.             <version>2.7.6</version>
    17.         </dependency>
    18.         <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-yarn-client -->
    19.         <dependency>
    20.             <groupId>org.apache.hadoop</groupId>
    21.             <artifactId>hadoop-client</artifactId>
    22.             <version>2.7.6</version>
    23.         </dependency>
    24.         <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
    25.         <dependency>
    26.             <groupId>org.apache.hadoop</groupId>
    27.             <artifactId>hadoop-hdfs</artifactId>
    28.             <version>2.7.6</version>
    29.             <scope>test</scope>
    30.         </dependency>
    31.         <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-mapreduce-client-core -->
    32.         <dependency>
    33.             <groupId>org.apache.hadoop</groupId>
    34.             <artifactId>hadoop-mapreduce-client-core</artifactId>
    35.             <version>2.7.6</version>
    36.         </dependency>
    37.         <!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-core -->
    38.         <dependency>
    39.             <groupId>org.apache.logging.log4j</groupId>
    40.             <artifactId>log4j-core</artifactId>
    41.             <version>2.8</version>
    42.         </dependency>
    44.     </dependencies>
    47.     <build>
    48.         <plugins>
    49.             <!-- 该插件可以生成一个带依赖和不带依赖的jar包 -->
    50.             <plugin>
    51.                 <groupId>org.apache.maven.plugins</groupId>
    52.                 <artifactId>maven-shade-plugin</artifactId>
    53.                 <version>3.2.0</version>
    54.                 <configuration>
    55.                     <createDependencyReducedPom>true</createDependencyReducedPom>
    56.                 </configuration>
    57.                 <executions>
    58.                     <execution>
    59.                         <phase>package</phase>
    60.                         <goals>
    61.                             <goal>shade</goal>
    62.                         </goals>
    63.                         <!--<configuration>
    64.                             <transformers>
    65.                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
    66.                                     &lt;!&ndash; 主类的全限定名 &ndash;&gt;
    67.                                     <mainClass>com.healchow.consumer.Main</mainClass>
    68.                                 </transformer>
    69.                             </transformers>
    70.                         </configuration>-->
    71.                     </execution>
    72.                 </executions>
    73.             </plugin>
    74.             <!-- 该插件会把jar放到dependency目录 -->
    75.             <plugin>
    76.                 <groupId>org.apache.maven.plugins</groupId>
    77.                 <artifactId>maven-dependency-plugin</artifactId>
    78.                 <executions>
    79.                     <execution>
    80.                         <id>copy</id>
    81.                         <phase>package</phase>
    82.                         <goals>
    83.                             <goal>copy-dependencies</goal>
    84.                         </goals>
    85.                         <!--<configuration>-->
    86.                         <!--<outputDirectory>${project.build.directory}/lib-->
    87.                         <!--</outputDirectory>-->
    88.                         <!--</configuration>-->
    89.                     </execution>
    90.                 </executions>
    91.             </plugin>
    92.             <plugin>
    93.                 <groupId>org.apache.maven.plugins</groupId>
    94.                 <artifactId>maven-compiler-plugin</artifactId>
    95.                 <version>3.8.0</version>
    96.                 <configuration>
    97.                     <source>1.8</source>
    98.                     <target>1.8</target>
    99.                     <encoding>UTF-8</encoding>
    100.                 </configuration>
    101.             </plugin>
    102.         </plugins>
    103.     </build>
    105. </project>

    日志：

    log4j.rootLogger = debug,stdout

### 输出信息到控制抬 ###
log4j.appender.stdout = org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target = System.out
log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern = [%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n