拉取hadoop镜像

Hadoop镜像，到docker hub上拉取，也可以使用自己制作的hadoop镜像，这里我们直接到docker hub拉取

  1. docker pull kubeguide/hadoop:latest

配置hadoop.yaml文件

  1. apiVersion: v1
  2. kind: ConfigMap
  3. metadata:
  4.   name: kube-hadoop-conf
  5.   namespace: default
  6. data:
  7.   HDFS_MASTER_SERVICE: hadoop-hdfs-master
  8.   HDOOP_YARN_MASTER: hadoop-yarn-master
  9. ---
  10. apiVersion: v1
  11. kind: Service
  12. metadata:
  13.   name: hadoop-hdfs-master
  14. spec:
  15.   type: NodePort
  16.   selector:
  17.     app: hdfs-master
  18.   ports:
  19.     - name: rpc
  20.       port: 9000
  21.       targetPort: 9000
  22.     - name: http
  23.       port: 50070
  24.       targetPort: 50070
  25.       nodePort: 32007
  26. ---
  27. apiVersion: v1
  28. kind: Pod
  29. metadata:
  30.   name: hdfs-master
  31.   labels:
  32.     app: hdfs-master
  33. spec:
  34.   containers:
  35.     - name: hdfs-master
  36.       image: kubeguide/hadoop
  37.       imagePullPolicy: IfNotPresent
  38.       ports:
  39.         - containerPort: 9000
  40.         - containerPort: 50070    
  41.       env:
  42.         - name: HADOOP_NODE_TYPE
  43.           value: namenode
  44.         - name: HDFS_MASTER_SERVICE
  45.           valueFrom:
  46.             configMapKeyRef:
  47.               name: kube-hadoop-conf
  48.               key: HDFS_MASTER_SERVICE
  49.         - name: HDOOP_YARN_MASTER
  50.           valueFrom:
  51.             configMapKeyRef:
  52.               name: kube-hadoop-conf
  53.               key: HDOOP_YARN_MASTER
  54.   restartPolicy: Always
  55. ---
  56. apiVersion: v1
  57. kind: Pod
  58. metadata:
  59.     name: hadoop-datanode-1
  60.     labels:
  61.       app: hadoop-datanode-1
  62. spec:
  63.   containers:
  64.     - name: hadoop-datanode-1
  65.       image: kubeguide/hadoop
  66.       imagePullPolicy: IfNotPresent
  67.       ports:
  68.         - containerPort: 9000
  69.         - containerPort: 50070    
  70.       env:
  71.         - name: HADOOP_NODE_TYPE
  72.           value: datanode
  73.         - name: HDFS_MASTER_SERVICE
  74.           valueFrom:
  75.             configMapKeyRef:
  76.               name: kube-hadoop-conf
  77.               key: HDFS_MASTER_SERVICE
  78.         - name: HDOOP_YARN_MASTER
  79.           valueFrom:
  80.             configMapKeyRef:
  81.               name: kube-hadoop-conf
  82.               key: HDOOP_YARN_MASTER        
  83.   restartPolicy: Always
  84. ---
  85. apiVersion: v1
  86. kind: Pod
  87. metadata:
  88.     name: hadoop-datanode-2
  89.     labels:
  90.       app: hadoop-datanode-2
  91. spec:
  92.   containers:
  93.     - name: hadoop-datanode-2
  94.       image: kubeguide/hadoop
  95.       imagePullPolicy: IfNotPresent
  96.       ports:
  97.         - containerPort: 9000
  98.         - containerPort: 50070    
  99.       env:
  100.         - name: HADOOP_NODE_TYPE
  101.           value: datanode
  102.         - name: HDFS_MASTER_SERVICE
  103.           valueFrom:
  104.             configMapKeyRef:
  105.               name: kube-hadoop-conf
  106.               key: HDFS_MASTER_SERVICE
  107.         - name: HDOOP_YARN_MASTER
  108.           valueFrom:
  109.             configMapKeyRef:
  110.               name: kube-hadoop-conf
  111.               key: HDOOP_YARN_MASTER        
  112.   restartPolicy: Always
  113. ---
  114. apiVersion: v1
  115. kind: Pod
  116. metadata:
  117.     name: hadoop-datanode-3
  118.     labels:
  119.       app: hadoop-datanode-3
  120. spec:
  121.   containers:
  122.     - name: hadoop-datanode-3
  123.       image: kubeguide/hadoop
  124.       imagePullPolicy: IfNotPresent
  125.       ports:
  126.         - containerPort: 9000
  127.         - containerPort: 50070    
  128.       env:
  129.         - name: HADOOP_NODE_TYPE
  130.           value: datanode
  131.         - name: HDFS_MASTER_SERVICE
  132.           valueFrom:
  133.             configMapKeyRef:
  134.               name: kube-hadoop-conf
  135.               key: HDFS_MASTER_SERVICE
  136.         - name: HDOOP_YARN_MASTER
  137.           valueFrom:
  138.             configMapKeyRef:
  139.               name: kube-hadoop-conf
  140.               key: HDOOP_YARN_MASTER        
  141.   restartPolicy: Always
  142. ---
  143. apiVersion: v1
  144. kind: Service
  145. metadata:
  146.   name: hadoop-yarn-master
  147. spec:
  148.   type: NodePort
  149.   selector:
  150.     app: yarn-master
  151.   ports:
  152.      - name: "8030"       
  153.        port: 8030
  154.      - name: "8031"     
  155.        port: 8031
  156.      - name: "8032"
  157.        port: 8032     
  158.      - name: http
  159.        port: 8088
  160.        targetPort: 8088
  161.        nodePort: 32088
  162. ---
  163. apiVersion: v1
  164. kind: Pod
  165. metadata:
  166.   name: yarn-master
  167.   labels:
  168.     app: yarn-master
  169. spec:
  170.   containers:
  171.     - name: yarn-master
  172.       image: kubeguide/hadoop
  173.       imagePullPolicy: IfNotPresent
  174.       ports:
  175.         - containerPort: 9000
  176.         - containerPort: 50070    
  177.       env:
  178.         - name: HADOOP_NODE_TYPE
  179.           value: resourceman
  180.         - name: HDFS_MASTER_SERVICE
  181.           valueFrom:
  182.             configMapKeyRef:
  183.               name: kube-hadoop-conf
  184.               key: HDFS_MASTER_SERVICE
  185.         - name: HDOOP_YARN_MASTER
  186.           valueFrom:
  187.             configMapKeyRef:
  188.               name: kube-hadoop-conf
  189.               key: HDOOP_YARN_MASTER          
  190.   restartPolicy: Always
  191. ---
  192. apiVersion: v1
  193. kind: Service
  194. metadata:
  195.   name: yarn-node-1
  196. spec:
  197.   clusterIP: None
  198.   selector:
  199.     app: yarn-node-1
  200.   ports:
  201.      - port: 8040
  202. ---
  203. apiVersion: v1
  204. kind: Service
  205. metadata:
  206.   name: yarn-node-2
  207. spec:
  208.   clusterIP: None
  209.   selector:
  210.     app: yarn-node-2
  211.   ports:
  212.      - port: 8040
  213. ---
  214. apiVersion: v1
  215. kind: Service
  216. metadata:
  217.   name: yarn-node-3
  218. spec:
  219.   clusterIP: None
  220.   selector:
  221.     app: yarn-node-3
  222.   ports:
  223.      - port: 8040
  224. ---
  225. apiVersion: v1
  226. kind: Pod
  227. metadata:
  228.   name: yarn-node-1
  229.   labels:
  230.     app: yarn-node-1
  231. spec:
  232.   containers:
  233.     - name: yarn-node-1
  234.       image: kubeguide/hadoop
  235.       imagePullPolicy: IfNotPresent
  236.       ports:
  237.         - containerPort: 8040
  238.         - containerPort: 8041   
  239.         - containerPort: 8042        
  240.       env:
  241.         - name: HADOOP_NODE_TYPE
  242.           value: yarnnode
  243.         - name: HDFS_MASTER_SERVICE
  244.           valueFrom:
  245.             configMapKeyRef:
  246.               name: kube-hadoop-conf
  247.               key: HDFS_MASTER_SERVICE
  248.         - name: HDOOP_YARN_MASTER
  249.           valueFrom:
  250.             configMapKeyRef:
  251.               name: kube-hadoop-conf
  252.               key: HDOOP_YARN_MASTER          
  253.   restartPolicy: Always
  254. ---
  255. apiVersion: v1
  256. kind: Pod
  257. metadata:
  258.   name: yarn-node-2
  259.   labels:
  260.     app: yarn-node-2
  261. spec:
  262.   containers:
  263.     - name: yarn-node-2
  264.       image: kubeguide/hadoop
  265.       imagePullPolicy: IfNotPresent
  266.       ports:
  267.         - containerPort: 8040
  268.         - containerPort: 8041   
  269.         - containerPort: 8042        
  270.       env:
  271.         - name: HADOOP_NODE_TYPE
  272.           value: yarnnode
  273.         - name: HDFS_MASTER_SERVICE
  274.           valueFrom:
  275.             configMapKeyRef:
  276.               name: kube-hadoop-conf
  277.               key: HDFS_MASTER_SERVICE
  278.         - name: HDOOP_YARN_MASTER
  279.           valueFrom:
  280.             configMapKeyRef:
  281.               name: kube-hadoop-conf
  282.               key: HDOOP_YARN_MASTER          
  283.   restartPolicy: Always
  284. ---
  285. apiVersion: v1
  286. kind: Pod
  287. metadata:
  288.   name: yarn-node-3
  289.   labels:
  290.     app: yarn-node-3
  291. spec:
  292.   containers:
  293.     - name: yarn-node-3
  294.       image: kubeguide/hadoop
  295.       imagePullPolicy: IfNotPresent
  296.       ports:
  297.         - containerPort: 8040
  298.         - containerPort: 8041   
  299.         - containerPort: 8042        
  300.       env:
  301.         - name: HADOOP_NODE_TYPE
  302.           value: yarnnode
  303.         - name: HDFS_MASTER_SERVICE
  304.           valueFrom:
  305.             configMapKeyRef:
  306.               name: kube-hadoop-conf
  307.               key: HDFS_MASTER_SERVICE
  308.         - name: HDOOP_YARN_MASTER
  309.           valueFrom:
  310.             configMapKeyRef:
  311.               name: kube-hadoop-conf
  312.               key: HDOOP_YARN_MASTER          
  313.   restartPolicy: Always

这个yaml文件包含一个ConfigMap，5个Service，8个pod，这里需要注意的是ConfigMap中HDFS_MASTER_SERVICE和HDOOP_YARN_MASTER不要使用IP，使用HDFS service的名称，否则datanode将会连接不上namenode，出现错误【ipc.Client: Retrying connect to server: xxx:9000.】

启动pod

执行创建命令

  1. kubectl create -f hadoop.yaml

检查是否启动成功

  1. 查看pod
    1. kubectl get po -o wide
    如下，全部running则为启动成功
    image-20200424175506438.png

2.查看service

  1. kubectl get svc -o wide

image-20200424175626625.png

3.查看configmap

  1. kubectl get cm -o wide

image-20200424175723917.png

外部访问

通过浏览器访问：http://10.8.0.117:32007/
因为设置了nodeport,所以无论是哪台物理机的ip都可以访问

使用pod控制器搭建

上面的搭建方式是以单个POD声明的，这种方式不稳定，如果不小心删除后就没有了，其实我们可以使用Replication Controller方式进行搭建，这样的话始终可以确保保留相应数量的POD，具体的yaml文件如下：

  1. apiVersion: v1
  2. kind: ConfigMap
  3. metadata:
  4.   name: kube-hadoop-conf
  5.   namespace: default
  6. data:
  7.   HDFS_MASTER_SERVICE: hadoop-hdfs-master
  8.   HDOOP_YARN_MASTER: hadoop-yarn-master
  9. ---
  10. apiVersion: v1
  11. kind: Service
  12. metadata:
  13.   name: hadoop-hdfs-master
  14. spec:
  15.   type: NodePort
  16.   selector:
  17.     name: hdfs-master
  18.   ports:
  19.     - name: rpc
  20.       port: 9000
  21.       targetPort: 9000
  22.     - name: http
  23.       port: 50070
  24.       targetPort: 50070
  25.       nodePort: 32007
  26. ---
  27. apiVersion: v1
  28. kind: ReplicationController
  29. metadata:
  30.   name: hdfs-master
  31.   labels:
  32.     name: hdfs-master
  33. spec:
  34.   replicas: 1
  35.   selector:
  36.     name: hdfs-master
  37.   template:
  38.     metadata:
  39.       labels:
  40.         name: hdfs-master
  41.     spec:
  42.       containers:
  43.         - name: hdfs-master
  44.           image: kubeguide/hadoop
  45.           imagePullPolicy: IfNotPresent
  46.           ports:
  47.             - containerPort: 9000
  48.             - containerPort: 50070    
  49.           env:
  50.             - name: HADOOP_NODE_TYPE
  51.               value: namenode
  52.             - name: HDFS_MASTER_SERVICE
  53.               valueFrom:
  54.                 configMapKeyRef:
  55.                   name: kube-hadoop-conf
  56.                   key: HDFS_MASTER_SERVICE
  57.             - name: HDOOP_YARN_MASTER
  58.               valueFrom:
  59.                 configMapKeyRef:
  60.                   name: kube-hadoop-conf
  61.                   key: HDOOP_YARN_MASTER
  62.       restartPolicy: Always
  63. ---
  64. apiVersion: v1
  65. kind: ReplicationController
  66. metadata:
  67.   name: hadoop-datanode
  68.   labels:
  69.     app: hadoop-datanode
  70. spec:
  71.   replicas: 3
  72.   selector:
  73.     name: hadoop-datanode
  74.   template:
  75.     metadata:
  76.       labels:
  77.         name: hadoop-datanode
  78.     spec:
  79.       containers:
  80.         - name: hadoop-datanode
  81.           image: kubeguide/hadoop
  82.           imagePullPolicy: IfNotPresent
  83.           ports:
  84.             - containerPort: 9000
  85.             - containerPort: 50070    
  86.           env:
  87.             - name: HADOOP_NODE_TYPE
  88.               value: datanode
  89.             - name: HDFS_MASTER_SERVICE
  90.               valueFrom:
  91.                 configMapKeyRef:
  92.                   name: kube-hadoop-conf
  93.                   key: HDFS_MASTER_SERVICE
  94.             - name: HDOOP_YARN_MASTER
  95.               valueFrom:
  96.                 configMapKeyRef:
  97.                   name: kube-hadoop-conf
  98.                   key: HDOOP_YARN_MASTER
  99.       restartPolicy: Always
  100. ---
  101. apiVersion: v1
  102. kind: Service
  103. metadata:
  104.   name: hadoop-yarn-master
  105. spec:
  106.   type: NodePort
  107.   selector:
  108.     name: yarn-master
  109.   ports:
  110.      - name: "8030"       
  111.        port: 8030
  112.      - name: "8031"     
  113.        port: 8031
  114.      - name: "8032"
  115.        port: 8032     
  116.      - name: http
  117.        port: 8088
  118.        targetPort: 8088
  119.        nodePort: 32088
  120. ---
  121. apiVersion: v1
  122. kind: ReplicationController
  123. metadata:
  124.   name: yarn-master
  125.   labels:
  126.     name: yarn-master
  127. spec:
  128.   replicas: 1
  129.   selector:
  130.     name: yarn-master
  131.   template:
  132.     metadata:
  133.       labels:
  134.         name: yarn-master
  135.     spec:
  136.       containers:
  137.         - name: yarn-master
  138.           image: kubeguide/hadoop
  139.           imagePullPolicy: IfNotPresent
  140.           ports:
  141.             - containerPort: 9000
  142.             - containerPort: 50070    
  143.           env:
  144.             - name: HADOOP_NODE_TYPE
  145.               value: resourceman
  146.             - name: HDFS_MASTER_SERVICE
  147.               valueFrom:
  148.                 configMapKeyRef:
  149.                   name: kube-hadoop-conf
  150.                   key: HDFS_MASTER_SERVICE
  151.             - name: HDOOP_YARN_MASTER
  152.               valueFrom:
  153.                 configMapKeyRef:
  154.                   name: kube-hadoop-conf
  155.                   key: HDOOP_YARN_MASTER          
  156.       restartPolicy: Always
  157. ---
  158. apiVersion: v1
  159. kind: Service
  160. metadata:
  161.   name: yarn-node
  162. spec:
  163.   clusterIP: None
  164.   selector:
  165.     name: yarn-node
  166.   ports:
  167.      - port: 8040
  168. ---
  169. apiVersion: v1
  170. kind: ReplicationController
  171. metadata:
  172.   name: yarn-node
  173.   labels:
  174.     name: yarn-node
  175. spec:
  176.   replicas: 3
  177.   selector:
  178.     name: yarn-node
  179.   template:
  180.     metadata:
  181.       labels:
  182.         name: yarn-node
  183.     spec:
  184.       containers:
  185.         - name: yarn-node
  186.           image: kubeguide/hadoop
  187.           imagePullPolicy: IfNotPresent
  188.           ports:
  189.             - containerPort: 8040
  190.             - containerPort: 8041   
  191.             - containerPort: 8042        
  192.           env:
  193.             - name: HADOOP_NODE_TYPE
  194.               value: yarnnode
  195.             - name: HDFS_MASTER_SERVICE
  196.               valueFrom:
  197.                 configMapKeyRef:
  198.                   name: kube-hadoop-conf
  199.                   key: HDFS_MASTER_SERVICE
  200.             - name: HDOOP_YARN_MASTER
  201.               valueFrom:
  202.                 configMapKeyRef:
  203.                   name: kube-hadoop-conf
  204.                   key: HDOOP_YARN_MASTER          
  205.       restartPolicy: Always

使用kubectl命令创建

  1. kubectl apply -f hadoop.yaml

查看pod,全部running则启动成功

  1. kubectl get pods -o wide

image-20200424175506438.png
至此，我们的hadoop就搭建完成了

参考连接：https://www.jianshu.com/p/e768ab139842