遇到问题

windows本地调试Hadoop程序时报错

描述

java.io.FileNotFoundException: java.io.FileNotFoundException: HADOOP_HOME and hadoop.home.dir are unset

原因

其原因是需要在windows本地搭建Hadoop环境，下载winutils文件，并将hadoop-2.8.4包内的bin文件替换，将下载文件中hadoop.dll放到C：\Windows\System32下

解决办法

打包放在linux上的hadoop环境运行。

代码清单

  1. import org.apache.hadoop.conf.Configuration;
  2. import org.apache.hadoop.fs.Path;
  3. import org.apache.hadoop.io.IntWritable;
  4. import org.apache.hadoop.io.Text;
  5. import org.apache.hadoop.mapreduce.Job;
  6. import org.apache.hadoop.mapreduce.Mapper;
  7. import org.apache.hadoop.mapreduce.Reducer;
  8. import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
  9. import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
  12. import java.io.IOException;
  13. import java.util.StringTokenizer;
  15. public class WordCount {
  16.     //编写TokenizerMapper类继承Mapper类
  17.     public static class TokenizerMapper extends Mapper<Object,Text,Text,IntWritable>{
  18.         //定义变量one值设置为1，为每个单词定义value为1
  19.         public static final IntWritable one=new IntWritable(1);
  20.         private Text word=new Text();
  21.         //编写map函数，其中输入参数为value（即为单词），输出参数为context
  22.         public void map(Object key,Text values,Context context) throws IOException, InterruptedException {
  23.             StringTokenizer str=new StringTokenizer(values.toString());
  24.             while(str.hasMoreTokens()){
  25.                 word.set(str.nextToken());
  26.                 context.write(word,one);
  27.             }
  28.         }
  29.     }
  30.     //定义IntSumReducer继承Reducer
  31.     public static class IntSumReducer extends Reducer<Text,IntWritable,Text,IntWritable>{
  32.         private IntWritable result=new IntWritable();
  33.         //定义reduce方法
  34.         public void reduce(Text key,Iterable<IntWritable> values,Context context) throws IOException, InterruptedException {
  35.             //遍历，将统计各个单词的总个数
  36.             int sum=0;
  37.             for (IntWritable val:values) {
  38.                 sum+=val.get();
  39.             }
  40.             result.set(sum);
  41.             context.write(key,result);
  42.         }
  43.     }
  44.     //编写主函数
  45.     public static void main(String[] args) throws Exception{
  46.         Configuration conf=new Configuration();
  48.         Job job=Job.getInstance(conf,"wordCount");
  49.         job.setJarByClass(WordCount.class);
  50.         job.setMapperClass(TokenizerMapper.class);
  51.         job.setCombinerClass(IntSumReducer.class);
  52.         job.setReducerClass(IntSumReducer.class);
  53.         job.setOutputKeyClass(Text.class);
  54.         job.setOutputValueClass(IntWritable.class);
  55.         //添加文件的输入路径
  56.         FileInputFormat.addInputPath(job, new Path(args[0]));
  57.         //添加文件的输出路径
  58.         FileOutputFormat.setOutputPath(job, new Path(args[1]));
  59.         System.exit(job.waitForCompletion(true)?0:1);
  60.     }
  61. }