五、window下开发hadoop

一、环境搭建

• 下载win平台下编译的hadoop安装包

https://archive.apache.org/dist/hadoop/common/hadoop-3.2.1/ — hadoop-3.2.1.tar.gz

• 配置相关文件

https://github.com/steveloughran/winutils
下载bin，然后将winutils.exe复制到本地的hadoop/bin下

• 配置hadoop环境变量

二、springboot集成hadoop

//到服务器上修改hadoop的配置文件 hdfs-site.xml
//去掉访问权限限制
<property>
    <name>dfs.permissions</name>
    <value>false</value>
</property>
//在代码中修改
System.setProperty("HADOOP_USER_NAME","root");
//打印文件列表
@Override
public void listFile() throws Exception{
    Configuration conf = new Configuration();
    conf.set("fs.defaultFS", path);
    FileSystem fileSystem = FileSystem.get(conf);
    RemoteIterator<LocatedFileStatus> listFiles = fileSystem.listFiles(new Path("/"), false);
    while(listFiles.hasNext()){
        LocatedFileStatus locatedFileStatus = listFiles.next();
        String filename = locatedFileStatus.getPath().getName();
        System.out.println(filename);
    }
}
@Override
public void uploadFile() throws Exception{
    Configuration conf = new Configuration();
    conf.set("fs.defaultFS", path);
    //可以通过以下方法伪造客户端，win平台调用linux的情况
    //System.setProperty("HADOOP_USER_NAME", "root");
    FileSystem fileSystem = FileSystem.get(conf);
    fileSystem.copyFromLocalFile(new Path("d://test.rar"), new Path("/"));
    //fileSystem.copyToLocalFile(false, new Path("d://"), new Path("/hello.txt"),true);
    fileSystem.close();
}