1、确认已安装的Hadoop版本

  1. hadoop version

根据计算机安装的Hadoop版本，选择对应的spark版本下载并安装
image.png

2、确定Spark、Scala版本

在http://spark.apache.org/downloads.html 下载页，选择下载
image.png

3、安装Scala

在https://www.scala-lang.org/download/ 下载安装对应版本Scala

  • Scala 2.11.12：https://www.scala-lang.org/download/2.11.12.html
    1. cd ~
    2. # 下载
    3. wget https://downloads.lightbend.com/scala/2.11.12/scala-2.11.12.tgz
    4. # 解压
    5. sudo tar -zxf scala-2.11.12.tgz -C /usr/local
    6. # 重命名
    7. sudo mv /usr/local/scala-2.11.12/ /usr/local/scala
    8. # 修改目录及其子文件用户权限
    9. sudo chown -R hadoop /usr/local/scala/
    10. # 添加环境变量
    11. echo '
    12. export SCALA_HOME=/usr/local/scala
    13. export PATH=$PATH:$SCALA_HOME/bin' >> ~/.bashrc
    14. source ~/.bashrc
    15. # 验证
    16. scala -version

    image.png

4、安装Spark

  1. cd ~
  2. # 下载
  3. wget https://mirrors.bfsu.edu.cn/apache/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz
  4. # 解压
  5. sudo tar -zxf spark-3.1.1-bin-hadoop3.2.tgz -C /usr/local
  6. # 重命名
  7. sudo mv /usr/local/spark-3.1.1-bin-hadoop3.2/ /usr/local/spark
  8. # 修改目录及其子文件用户权限
  9. sudo chown -R hadoop /usr/local/spark/
  10. # 添加环境变量
  11. echo '
  12. export SPARK_HOME=/usr/local/spark
  13. export PATH=$PATH:$SPARK_HOME/bin' >> ~/.bashrc
  14. source ~/.bashrc
  15. # 验证
  16. spark-shell

image.png 输入:quit退出spark-shell

5、配置spark

  1. cd /usr/local/spark/conf
  1. cp spark-env.sh.template spark-env.sh
  2. echo '
  3. JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
  5. HADOOP_HOME=/usr/local/hadoop
  6. HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
  8. SCALA_HOME=/usr/local/scala
  10. SPARK_HOME=/usr/local/spark
  11. SPARK_MASTER_IP=master
  12. SPARK_EXECUTOR_MEMORY=1G
  13. ' >> spark-env.sh
  1. cp workers.template workers
  2. echo 'node1
  3. node2' > workers

6、将Scala、Spark打包发送至node1、node2

  1. cd /usr/local
  2. tar -zcf ~/scala.master.tar.gz ./scala
  3. tar -zcf ~/spark.master.tar.gz ./spark
  5. scp ~/scala.master.tar.gz node1:/home/hadoop
  6. scp ~/spark.master.tar.gz node1:/home/hadoop
  8. scp ~/scala.master.tar.gz node2:/home/hadoop
  9. scp ~/spark.master.tar.gz node2:/home/hadoop

7、配置worker

切换到node1/node2

  1. cd ~
  3. sudo tar -zxf scala.master.tar.gz -C /usr/local
  4. sudo tar -zxf spark.master.tar.gz -C /usr/local
  7. sudo chown -R hadoop /usr/local/scala/
  8. sudo chown -R hadoop /usr/local/spark/
  10. echo '
  11. export SCALA_HOME=/usr/local/scala
  12. export PATH=$PATH:$SCALA_HOME/bin
  14. export SPARK_HOME=/usr/local/spark
  15. export PATH=$PATH:$SPARK_HOME/bin
  16. ' >> ~/.bashrc
  17. source ~/.bashrc
  19. rm scala.master.tar.gz spark.master.tar.gz

8、启动Spark，回到master

  1. # 启动hadoop
  2. start-all.sh
  4. # 启动Spark（注意./，因为spark的很多脚本和hadoop名称一样，所以这里没有为spark的脚本添加环境变量）
  5. cd /usr/local/spark/sbin
  6. ./start-all.sh

9、测试Spark

  1. cd /usr/local/spark/bin
  2. ./run-example SparkPi 2>&1 | grep "Pi is"

image.png

部分参考： https://blog.csdn.net/weixin_39660922/article/details/112744170 https://blog.csdn.net/sjmz30071360/article/details/82561867