概述

源自2014年12月的Google发表的MapReduce论文，它是一个编程模型，用于大数据量的计算，MapReduce是分布式计算框架。具有海量数据离线处理。对于大数据量的计算，通常采用的处理方式就是并行计算，MapReduce就是一种分布式计算的编程框架，它使得并没有并行计算经验的开发人员也可以计算并行应用程序。
MapReduce核心功能是将用户编写的业务逻辑代码和自带默认组件整合成一个完整的分布式运算程序，并发运行在一个Hadoop集群上。

设计目标

MapReduce采用的是分而治之的思想，即把大规模数据集的操作，分发给一个主节点管理下的各个子节点共同完成，然后整合各个子节点的中间结果，从而得到最终的计算结果。就是分散任务，汇总结果

编程模型

MapReduce分成Map阶段和Reduce阶段。用户只需要编写map()和reduce两个函数，即可完成简单的分布式程序的设计
map()函数以key/value对作为输入，产生另外一系列key/value对作为中间输出写入本地磁盘，MapReduc框架会自动将这些中间数据按照key值进行聚集，且key值相同（用户可设定聚集策略，默认情况下是对key值进行哈希取模）的数据被统一交给reduce()函数处理。
reduce()函数以key及对应的value列表作为输入，经合并key相同的value值后，产生另外一系列key/value作为最终输出写入HDFS.

优点

1MapReduce易于编程，它简单的实现一些接口，就可以完成一个分布式程序
2 易于拓展
3 高容错性
4 适合PB级别以上海量数据的离线处理

缺点

计算慢，通过分布式解决速度计算速度问题
##常用数据序列化类型
无法使用Java的类型变量
Hadoop——MapReduce分布式计算框架 - 图1

wordcount

pom.xml

  1. <?xml version="1.0" encoding="UTF-8"?>
  2. <project xmlns="http://maven.apache.org/POM/4.0.0"
  3.          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  4.          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  5.     <modelVersion>4.0.0</modelVersion>
  7.     <groupId>com.bigdata</groupId>
  8.     <artifactId>MapReduceNote</artifactId>
  9.     <version>1.0-SNAPSHOT</version>
  11.     <properties>
  12.         <hadoop.version>2.7.2</hadoop.version>
  13.     </properties>
  14.     <dependencies>
  15.         <dependency>
  16.             <groupId>org.apache.hadoop</groupId>
  17.             <artifactId>hadoop-common</artifactId>
  18.             <version>${hadoop.version}</version>
  19.         </dependency>
  20.         <dependency>
  21.             <groupId>org.apache.hadoop</groupId>
  22.             <artifactId>hadoop-client</artifactId>
  23.             <version>${hadoop.version}</version>
  24.         </dependency>
  25.         <dependency>
  26.             <groupId>org.apache.hadoop</groupId>
  27.             <artifactId>hadoop-hdfs</artifactId>
  28.             <version>${hadoop.version}</version>
  29.         </dependency>
  30.         <dependency>
  31.             <groupId>org.apache.hadoop</groupId>
  32.             <artifactId>hadoop-mapreduce-client-core</artifactId>
  33.             <version>${hadoop.version}</version>
  34.         </dependency>
  35.         <dependency>
  36.             <groupId>org.apache.hadoop</groupId>
  37.             <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
  38.             <version>${hadoop.version}</version>
  39.         </dependency>
  40.     </dependencies>
  42.     <build>
  43.         <plugins>
  44.             <plugin>
  45.                 <groupId>org.apache.maven.plugins</groupId>
  46.                 <artifactId>maven-compiler-plugin</artifactId>
  47.                 <version>2.3.2</version>
  48.                 <executions>
  49.                     <execution>
  50.                         <id>default-compile</id>
  51.                         <phase>compile</phase>
  52.                         <goals>
  53.                             <goal>compile</goal>
  54.                         </goals>
  55.                         <configuration>
  56.                             <encoding>UTF-8</encoding>
  57.                         </configuration>
  58.                     </execution>
  59.                 </executions>
  60.             </plugin>
  61.             <plugin>
  62.                 <artifactId>maven-assembly-plugin </artifactId>
  63.                 <configuration>
  64.                     <descriptorRefs>
  65.                         <descriptorRef>jar-with-dependencies</descriptorRef>
  66.                     </descriptorRefs>
  67.                     <archive>
  68.                         <manifest>
  69.                             <mainClass>com.bigdata.mapreduce.wordcount.WordCountDriver</mainClass>
  70.                         </manifest>
  71.                     </archive>
  72.                 </configuration>
  73.                 <executions>
  74.                     <execution>
  75.                         <id>make-assembly</id>
  76.                         <phase>package</phase>
  77.                         <goals>
  78.                             <goal>single</goal>
  79.                         </goals>
  80.                     </execution>
  81.                 </executions>
  82.             </plugin>
  83.         </plugins>
  84.     </build>
  85. </project>

Mapper

  1. package com.bigdata.mapreduce.wordcount;
  3. import org.apache.hadoop.io.IntWritable;
  4. import org.apache.hadoop.io.LongWritable;
  5. import org.apache.hadoop.io.Text;
  6. import org.apache.hadoop.mapreduce.Mapper;
  7. import java.io.IOException;
  9. // 输入 偏移量 输出
  10. public class WordCountMapper extends Mapper<LongWritable, Text,Text, IntWritable> {
  11.     private Text word = new Text();
  12.     private IntWritable one = new IntWritable(1);
  13.     @Override
  14.     protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
  15.         // 获取字符串内容
  16.         String line = value.toString();
  17.         // 安装空格切分数据
  18.         String[] words = line.split(" ");
  19.         // 遍历数组，把单词变成(word,1)
  20.         for (String word : words) {
  21.             this.word.set(word);
  22.             context.write(this.word,one);
  23.         }
  24.     }
  25. }

Reducer

package com.bigdata.mapreduce.wordcount;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class WordCountReduce extends Reducer<Text, IntWritable,Text,IntWritable> {
    private IntWritable total = new IntWritable();
    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
       int sum = 0;
        for (IntWritable value : values) {
            sum +=value.get();
        }
        total.set(sum);
        context.write(key,total);
    }
}

Driver

package com.bigdata.mapreduce.wordcount;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class WordCountDriver {
    public static void main(String[] args) throws IOException ,InterruptedException,ClassNotFoundException{
        // 申请一个job实例
        Job job = Job.getInstance(new Configuration());
        // 设置类路径
        job.setJarByClass(WordCountDriver.class);
        // 设置mapper类
        job.setMapperClass(WordCountMapper.class);
        // 设置reduce类
        job.setReducerClass(WordCountReduce.class);
        //设置map和reduce 的输出类型
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        //设置输入输出

        FileInputFormat.setInputPaths(job,new Path(args[0]));
        FileOutputFormat.setOutputPath(job,new Path(args[1]));

        boolean b = job.waitForCompletion(true);
        System.exit(b?0:1);
    }
}

创建src/main/resources/log4j.properties

log4j.rootLogger=INFO, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
log4j.appender.logfile=org.apache.log4j.FileAppender
log4j.appender.logfile.File=target/spring.log
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n

没有依赖库需要配置com.bigdata.mapreduce.wordcount.WordCountDriver

hadoop jar MapReduceNote-1.0-SNAPSHOT.jar com.bigdata.mapreduce.wordcount.WordCountDriver  /user/baxiang/input/word.txt /user/baxiang/output