1、Flannel VxLAN Mode跨节点通信原理解析

  1. #环境Found network config - Backend type: vxlan模式
  2. #env
  3. [root@master ~]# kubectl get nodes -owide 
  4. NAME     STATUS   ROLES    AGE   VERSION   INTERNAL-IP   EXTERNAL-IP   OS-IMAGE                KERNEL-VERSION                CONTAINER-RUNTIME
  5. master   Ready    master   22h   v1.19.3   10.0.4.2      <none>        CentOS Linux 7 (Core)   3.10.0-1160.45.1.el7.x86_64   docker://19.3.11
  6. node     Ready    <none>   21h   v1.19.3   10.0.4.14     <none>        CentOS Linux 7 (Core)   3.10.0-1160.45.1.el7.x86_64   docker://19.3.11
  8. [root@master ~]# kubectl run wang1 --image=burlyluo/nettoolbox
  9. pod/wang1 created
  10. [root@master ~]# kubectl run wang2 --image=burlyluo/nettoolbox
  11. pod/wang2 created
  12. [root@master ~]# kubectl get pods -o wide 创建两个容器，容器在不同的node节点上
  13. NAME    READY   STATUS    RESTARTS   AGE   IP           NODE     NOMINATED NODE   READINESS GATES
  14. wang1   1/1     Running   0          30s   10.244.1.3   node     <none>           <none>
  15. wang2   1/1     Running   0          26s   10.244.0.3   master   <none>           <none>
  17. 两pod的ip和mac地址如下:
  19. 容器wang1: 
  20. [root@master ~]# kubectl  exec -it wang1 -- ifconfig 
  21. eth0      Link encap:Ethernet  HWaddr BE:AB:DC:A9:63:00  
  22.           inet addr:10.244.1.3  Bcast:10.244.1.255  Mask:255.255.255.0
  23.           UP BROADCAST RUNNING MULTICAST  MTU:1450  Metric:1
  24.           RX packets:6 errors:0 dropped:0 overruns:0 frame:0
  25.           TX packets:1 errors:0 dropped:0 overruns:0 carrier:0
  26.           collisions:0 txqueuelen:0 
  27.           RX bytes:488 (488.0 B)  TX bytes:42 (42.0 B)
  29. lo        Link encap:Local Loopback  
  30.           inet addr:127.0.0.1  Mask:255.0.0.0
  31.           UP LOOPBACK RUNNING  MTU:65536  Metric:1
  32.           RX packets:0 errors:0 dropped:0 overruns:0 frame:0
  33.           TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
  34.           collisions:0 txqueuelen:1000 
  35.           RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)
  37. 容器wang2: 
  38. [root@master ~]# kubectl  exec -it wang2 -- ifconfig 
  39. eth0      Link encap:Ethernet  HWaddr 7E:85:89:39:FC:C7  
  40.           inet addr:10.244.0.3  Bcast:10.244.0.255  Mask:255.255.255.0
  41.           UP BROADCAST RUNNING MULTICAST  MTU:1450  Metric:1
  42.           RX packets:6 errors:0 dropped:0 overruns:0 frame:0
  43.           TX packets:1 errors:0 dropped:0 overruns:0 carrier:0
  44.           collisions:0 txqueuelen:0 
  45.           RX bytes:488 (488.0 B)  TX bytes:42 (42.0 B)
  47. lo        Link encap:Local Loopback  
  48.           inet addr:127.0.0.1  Mask:255.0.0.0
  49.           UP LOOPBACK RUNNING  MTU:65536  Metric:1
  50.           RX packets:0 errors:0 dropped:0 overruns:0 frame:0
  51.           TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
  52.           collisions:0 txqueuelen:1000 
  53.           RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)
  59. master节点跟Node节点ip跟mac地址如下：
  60. master节点：
  61. [root@master ~]# ifconfig 
  62. cni0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
  63.         inet 10.244.0.1  netmask 255.255.255.0  broadcast 10.244.0.255
  64.         inet6 fe80::b091:f9ff:feda:fb9a  prefixlen 64  scopeid 0x20<link>
  65.         ether b2:91:f9:da:fb:9a  txqueuelen 1000  (Ethernet)
  66.         RX packets 54534  bytes 3830716 (3.6 MiB)
  67.         RX errors 0  dropped 0  overruns 0  frame 0
  68.         TX packets 61543  bytes 22338790 (21.3 MiB)
  69.         TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
  71. docker0: flags=4099<UP,BROADCAST,MULTICAST>  mtu 1500
  72.         inet 172.17.0.1  netmask 255.255.0.0  broadcast 172.17.255.255
  73.         ether 02:42:44:d7:3f:13  txqueuelen 0  (Ethernet)
  74.         RX packets 0  bytes 0 (0.0 B)
  75.         RX errors 0  dropped 0  overruns 0  frame 0
  76.         TX packets 0  bytes 0 (0.0 B)
  77.         TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
  79. eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
  80.         inet 10.0.4.2  netmask 255.255.252.0  broadcast 10.0.7.255
  81.         inet6 fe80::5054:ff:fe9c:9d7  prefixlen 64  scopeid 0x20<link>
  82.         ether 52:54:00:9c:09:d7  txqueuelen 1000  (Ethernet)
  83.         RX packets 219858  bytes 69797512 (66.5 MiB)
  84.         RX errors 0  dropped 0  overruns 0  frame 0
  85.         TX packets 210984  bytes 61017824 (58.1 MiB)
  86.         TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
  88. flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
  89.         inet 10.244.0.0  netmask 255.255.255.255  broadcast 0.0.0.0
  90.         inet6 fe80::d0ba:c1ff:fedb:b377  prefixlen 64  scopeid 0x20<link>
  91.         ether d2:ba:c1:db:b3:77  txqueuelen 0  (Ethernet)
  92.         RX packets 0  bytes 0 (0.0 B)
  93.         RX errors 0  dropped 0  overruns 0  frame 0
  94.         TX packets 0  bytes 0 (0.0 B)
  95.         TX errors 0  dropped 8 overruns 0  carrier 0  collisions 0
  97. lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
  98.         inet 127.0.0.1  netmask 255.0.0.0
  99.         inet6 ::1  prefixlen 128  scopeid 0x10<host>
  100.         loop  txqueuelen 1000  (Local Loopback)
  101.         RX packets 4910850  bytes 933338377 (890.1 MiB)
  102.         RX errors 0  dropped 0  overruns 0  frame 0
  103.         TX packets 4910850  bytes 933338377 (890.1 MiB)
  104.         TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
  106. veth5d26ec6b: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
  107.         inet6 fe80::b4b0:74ff:fe53:6a05  prefixlen 64  scopeid 0x20<link>
  108.         ether b6:b0:74:53:6a:05  txqueuelen 0  (Ethernet)
  109.         RX packets 54533  bytes 4594150 (4.3 MiB)
  110.         RX errors 0  dropped 0  overruns 0  frame 0
  111.         TX packets 61550  bytes 22339272 (21.3 MiB)
  112.         TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
  114. veth6fc1c522: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
  115.         inet6 fe80::60e3:71ff:fe7a:19a2  prefixlen 64  scopeid 0x20<link>
  116.         ether 62:e3:71:7a:19:a2  txqueuelen 0  (Ethernet)
  117.         RX packets 1  bytes 42 (42.0 B)
  118.         RX errors 0  dropped 0  overruns 0  frame 0
  119.         TX packets 6  bytes 488 (488.0 B)
  120.         TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
  123. node节点：
  124. [root@node ~]# ifconfig 
  125. cni0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
  126.         inet 10.244.1.1  netmask 255.255.255.0  broadcast 10.244.1.255
  127.         inet6 fe80::cced:88ff:fe53:aa9e  prefixlen 64  scopeid 0x20<link>
  128.         ether ce:ed:88:53:aa:9e  txqueuelen 1000  (Ethernet)
  129.         RX packets 54317  bytes 3819992 (3.6 MiB)
  130.         RX errors 0  dropped 0  overruns 0  frame 0
  131.         TX packets 62016  bytes 22382034 (21.3 MiB)
  132.         TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
  134. docker0: flags=4099<UP,BROADCAST,MULTICAST>  mtu 1500
  135.         inet 172.17.0.1  netmask 255.255.0.0  broadcast 172.17.255.255
  136.         ether 02:42:e2:e0:fd:6f  txqueuelen 0  (Ethernet)
  137.         RX packets 0  bytes 0 (0.0 B)
  138.         RX errors 0  dropped 0  overruns 0  frame 0
  139.         TX packets 0  bytes 0 (0.0 B)
  140.         TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
  142. eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
  143.         inet 10.0.4.14  netmask 255.255.252.0  broadcast 10.0.7.255
  144.         inet6 fe80::5054:ff:fec8:f795  prefixlen 64  scopeid 0x20<link>
  145.         ether 52:54:00:c8:f7:95  txqueuelen 1000  (Ethernet)
  146.         RX packets 267726  bytes 126835531 (120.9 MiB)
  147.         RX errors 0  dropped 0  overruns 0  frame 0
  148.         TX packets 205609  bytes 37793600 (36.0 MiB)
  149.         TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
  151. flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
  152.         inet 10.244.1.0  netmask 255.255.255.255  broadcast 0.0.0.0
  153.         inet6 fe80::cf0:b5ff:fe8c:28f5  prefixlen 64  scopeid 0x20<link>
  154.         ether 0e:f0:b5:8c:28:f5  txqueuelen 0  (Ethernet)
  155.         RX packets 0  bytes 0 (0.0 B)
  156.         RX errors 0  dropped 0  overruns 0  frame 0
  157.         TX packets 0  bytes 0 (0.0 B)
  158.         TX errors 0  dropped 8 overruns 0  carrier 0  collisions 0
  160. lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
  161.         inet 127.0.0.1  netmask 255.0.0.0
  162.         inet6 ::1  prefixlen 128  scopeid 0x10<host>
  163.         loop  txqueuelen 1000  (Local Loopback)
  164.         RX packets 114  bytes 9350 (9.1 KiB)
  165.         RX errors 0  dropped 0  overruns 0  frame 0
  166.         TX packets 114  bytes 9350 (9.1 KiB)
  167.         TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
  169. veth0e17916d: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
  170.         inet6 fe80::14ff:d4ff:fe74:2c3e  prefixlen 64  scopeid 0x20<link>
  171.         ether 16:ff:d4:74:2c:3e  txqueuelen 0  (Ethernet)
  172.         RX packets 54316  bytes 4580388 (4.3 MiB)
  173.         RX errors 0  dropped 0  overruns 0  frame 0
  174.         TX packets 62023  bytes 22382516 (21.3 MiB)
  175.         TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
  177. veth98110d62: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
  178.         inet6 fe80::c079:4bff:fed9:ebee  prefixlen 64  scopeid 0x20<link>
  179.         ether c2:79:4b:d9:eb:ee  txqueuelen 0  (Ethernet)
  180.         RX packets 1  bytes 42 (42.0 B)
  181.         RX errors 0  dropped 0  overruns 0  frame 0
  182.         TX packets 6  bytes 488 (488.0 B)
  183.         TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

2、Flannel VxLAN Mode跨节点通信原理解析-Packet Flow

03-Kubernetes-Flannel VxLAN Mode - 图1

#######vxlan跨节点通信原理及抓包
[root@node ~]# kubectl get pods -owide 
NAME    READY   STATUS    RESTARTS   AGE     IP           NODE     NOMINATED NODE   READINESS GATES
wang1   1/1     Running   0          8m14s   10.244.1.3   node     <none>           <none>
wang2   1/1     Running   0          8m10s   10.244.0.3   master   <none>           <none>

使用master节点上的wang1容器去ping测node节点上的wang1容器：
对于pod:wang2容器要去的目的地址10.244.1.3和自己10.244.0.3并不是一个网段
所以进行路由查询
[root@master ~]# kubectl  exec -it wang2 -- route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.244.0.1      0.0.0.0         UG    0      0        0 eth0
10.244.0.0      0.0.0.0         255.255.255.0   U     0      0        0 eth0
10.244.0.0      10.244.0.1      255.255.0.0     UG    0      0        0 eth0 # 此时
路由显示去往10.244.0.0/16网络需要发给Gateway：10.244.0.1。也就是最后一条

所以需要查询10.244.0.1的mac地址，首先会先发个arp广播有个ff:ff:ff:ff的地址
15:17:32.964023 7e:85:89:39:fc:c7 > b2:91:f9:da:fb:9a, ethertype IPv4 (0x0800), length 98: 10.244.0.3 > 10.244.1.3: ICMP echo request, id 5632, seq 873, length 64
src_mac:7e:85:89:39:fc:c7_wang2  -> des_mac:b2:91:f9:da:fb:9a网关mac
src_ip:10.244.0.3 -> des_ip:10.244.1.3

此时查一下b2:91:f9:da:fb:9a的mac地址属于哪里
[root@master ~]# ifconfig  (可以看到目前des_mac地址属于master节点上的cni0)
cni0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
        inet 10.244.0.1  netmask 255.255.255.0  broadcast 10.244.0.255
        inet6 fe80::b091:f9ff:feda:fb9a  prefixlen 64  scopeid 0x20<link>
        ether b2:91:f9:da:fb:9a  txqueuelen 1000  (Ethernet)
        RX packets 59169  bytes 4168694 (3.9 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 66646  bytes 23881575 (22.7 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

此时数据报文由pod到达master节点上的root namespace上，查看一下master节点上的路由表
[root@master ~]# route -n (目的ip是10.244.1.3，所以匹配是第四条，Destination是10.244.1.0
 而我们目标ip属于这个网络，出接口是flannel.1)
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.0.4.1        0.0.0.0         UG    0      0        0 eth0
10.0.4.0        0.0.0.0         255.255.252.0   U     0      0        0 eth0
10.244.0.0      0.0.0.0         255.255.255.0   U     0      0        0 cni0
10.244.1.0      10.244.1.0      255.255.255.0   UG    0      0        0 flannel.1
169.254.0.0     0.0.0.0         255.255.0.0     U     1002   0        0 eth0
172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0

flannel1.1相当于是一个vxlan的设备
[root@master ~]# ip -d link  show flannel.1
6: flannel.1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN mode DEFAULT group default 
    link/ether d2:ba:c1:db:b3:77 brd ff:ff:ff:ff:ff:ff promiscuity 0 
    vxlan id 1 local 10.0.4.2 dev eth0 srcport 0 0 dstport 8472 nolearning ageing 300 noudpcsum noudp6zerocsumtx noudp6zerocsumrx addrgenmode eui64 numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535

现在根据路由表可以得知，下一跳是 10.244.1.0，数据报文应该如何封装？
多条路由条目时候，除了网关很重要以外，出接口也非常的重要，因为直接决定从一个接口送出去，就意味着
使用哪一个接口的MAC地址。
现在源mac地址就是出接口flannel.1的mac地址，而目的mac地址就是下一跳的mac地址(此地址为node1上的flannel.1网卡地址)

由上述路由信息，我们可以得到两条重要信息：
   1: 网关是10.244.1.0 该地址是node节点上的flannel.1（VTEP）的地址。
   2. 数据的出接口为flannel.1。也就是说数据包要从此接口发出。
这里的转发：需要查询到地址10.244.1.0对应的MAC地址。
####################
根据路由表信息我们知道了目的VTEP设备的IP地址，而根据三层IP地址查询二层MAC地址正是ARP表的功能。# 而这里用ARP表的记录，也就是flanneld进程在node节点启动时，自动添加到master上的.
所以此时ARP的缓存就自动在master的节点上缓存下来了，也就意味着此时master不再需要查询ARP了。
####################
所以此时的数据报文形式为：
S_IP: 10.244.0.3       S_MAC: $master_flannel.1_MAC # [fe80::d0ba:c1ff:fedb:b377]
D_IP: 10.244.1.3       D_MAC: $node_flannel.1_MAC # [fe80::cf0:b5ff:fe8c:28f5]

[root@master ~]# netstat -nlptu | grep 8472 (运行在内核空间，就不需要像udp穿插到用户空间)
udp        0      0 0.0.0.0:8472            0.0.0.0:*  -

证据：
[root@master ~]# tcpdump -ne -i flannel.1
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on flannel.1, link-type EN10MB (Ethernet), capture size 262144 bytes
23:43:21.817853 d2:ba:c1:db:b3:77 > 0e:f0:b5:8c:28:f5, ethertype IPv4 (0x0800), length 98: 10.244.0.3 > 10.244.1.3: ICMP echo request, id 15104, seq 6, length 64
23:43:21.818175 0e:f0:b5:8c:28:f5 > d2:ba:c1:db:b3:77, ethertype IPv4 (0x0800), length 98: 10.244.1.3 > 10.244.0.3: ICMP echo reply, id 15104, seq 6, length 64

S_IP: 10.244.0.3       S_MAC: $master_flannel.1_MAC # [fe80::d0ba:c1ff:fedb:b377]
D_IP: 10.244.1.3       D_MAC: $node_flannel.1_MAC # [fe80::cf0:b5ff:fe8c:28f5]

由于flannel.1在此环境中同时还扮演者VxLAN VTEP的角色，所以需要安装对应的方式去封装VxLAN的数据报文。
此时涉及到Inner和Outer的IP和MAC信息问题：[此时需要注意的是：此时的封装数据包是在flannel.1这个VTEP设备来指导下做数据报文的封装----非常重要！！！]
那么对于master上的这个VTEP flannel.1它现在知道它所在宿主机上的Outer的S_IP和S_MAC。也知道Inner的S_IP和S_MAC 和 D_IP和D_MAC。
但是唯独不知道的是D_IP。也就是说不知道10.244.1.0这个地址下的flannel.1在哪一个节点上。
# 在Linux内核里面，网桥设备进行转发的依据来自FDB的转发数据库。这个flannel网桥对应的FDB信息，就是flannel进程维护的。
所以我们查询：
[root@master ~]# bridge fdb show | grep 0e:f0:b5:8c:28:f5 (是查看对端的地址，不是pod的mac地址，现在是要vxlan上面转)
0e:f0:b5:8c:28:f5 dev flannel.1 dst 10.0.4.14 self permanent
由fdb转发信息，我们可以得出10.244.1.0此vtep所在node是10.0.4.14
到此master上的VTEP flannel.1获得了所有的封装包信息

所以需要再次查询路由表信息：
[root@master ~]# route -n 
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.0.4.1        0.0.0.0         UG    0      0        0 eth0
10.0.4.0        0.0.0.0         255.255.252.0   U     0      0        0 eth0 被该条路由条目匹配到
10.244.0.0      0.0.0.0         255.255.255.0   U     0      0        0 cni0
10.244.1.0      10.244.1.0      255.255.255.0   UG    0      0        0 flannel.1
169.254.0.0     0.0.0.0         255.255.0.0     U     1002   0        0 eth0
172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0

[root@master ~]# tcpdump -ne -i eth0 port 8472
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes
00:03:07.157412 52:54:00:9c:09:d7 > fe:ee:8f:bf:86:99, ethertype IPv4 (0x0800), length 148: 10.0.4.2.51891 > 10.0.4.14.otv: OTV, flags [I] (0x08), overlay 0, instance 1
d2:ba:c1:db:b3:77 > 0e:f0:b5:8c:28:f5, ethertype IPv4 (0x0800), length 98: 10.244.0.3 > 10.244.1.3: ICMP echo request, id 16640, seq 3, length 64
此时数据包就可转发出本端

3、VxLAN之MAC in UDP

# ECMP
在多负载均衡的环境中，通常我们有多条链路可供选择，此时就需要使用到基于流的负载分担了。
基于流的负载分担我们需要使用到5元组：
S_IP
S_PORT
D_IP
D_PORT
Protocol
在我们环境中如果想要实现基于流的负载分担，我们可以得到：
S_IP,D_IP,Protocol都是固定的。如果没有引入一个UDP头，此时hash出来的结果都是固定的，就无法实现基于流的负载分担了。
而UDP的Port中虽然目的端口(8472)是固定的，但是S_PORT是随机的，这样hash出来的结果就不一样了，就可以实现基于流的负载分担了。

而这里之所以没有使用MAC in TCP的原因是：
由于TCP需要三次握手，所以会使得其效率变得非常的低。

4、Flannel VxLAN Benchmark

# 1.环境配置VxLAN Tunnel：
[root@k8s-1 ~]# yum -y install bridge-utils
#1.k8s-1节点的配置：
ip link add vxlan_docker type vxlan id 20 remote 172.12.1.12 dstport 4789 dev ens33
ip link set vxlan_docker up
brctl addif docker0 vxlan_docker

#2.k8s-2节点的配置：
ip link add vxlan_docker type vxlan id 20 remote 172.12.1.11 dstport 4789 dev ens33
ip link set vxlan_docker up
brctl addif docker0 vxlan_docker
#配置说明
第一步：创建接口vxlan0 type为vxlan，vni为20.
第二部：创建的vxlan0 up起来.
第三步：使用brctl 添加interface vxlan0 到bridge docker0上，连接docker0和vxlan的接口，这样数据包从docker0上来以后，就可以被vxlan封装了。


# 2.启动相关容器：
# 在k8s-1上创建容器：
docker run --name vxlan-c01  -td burlyluo/nettoolbox   //该容器的地址为：172.17.0.2。
# 在k8s-2上创建容器： 
docker run --name vxlan-c01  -td burlyluo/nettoolbox   //该容器的地址为：172.17.0.2。
docker run --name vxlan-c02  -td burlyluo/nettoolbox   //该容器的地址为：172.17.0.3。

# 3.ping测：
[root@k8s-1 ~]# docker exec -it vxlan-c01 bash 
bash-5.0# ping 172.17.0.3
PING 172.17.0.3 (172.17.0.3): 56 data bytes
64 bytes from 172.17.0.3: seq=0 ttl=64 time=1.446 ms
64 bytes from 172.17.0.3: seq=1 ttl=64 time=0.643 ms
^C
--- 172.17.0.3 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max = 0.643/1.044/1.446 ms
bash-5.0# 

#####################
地址信息：
k8s-1节点：

[root@k8s-1 ~]# ifconfig 
[root@k8s-1 ~]# docker exec -it  vxlan-c01 ifconfig 
eth0      Link encap:Ethernet  HWaddr 02:42:AC:11:00:02  
          inet addr:172.17.0.2  Bcast:172.17.255.255  Mask:255.255.0.0
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:19 errors:0 dropped:0 overruns:0 frame:0
          TX packets:3 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0 
          RX bytes:1398 (1.3 KiB)  TX bytes:182 (182.0 B)
---#
docker0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
        inet 172.17.0.1  netmask 255.255.0.0  broadcast 172.17.255.255
        inet6 fe80::42:c9ff:feeb:2769  prefixlen 64  scopeid 0x20<link>
        ether 02:42:c9:eb:27:69  txqueuelen 0  (Ethernet)
        RX packets 18  bytes 1252 (1.2 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 5  bytes 526 (526.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

ens33: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 172.12.1.11  netmask 255.255.255.0  broadcast 172.12.1.255
        inet6 fe80::e222:32bb:f400:f0c3  prefixlen 64  scopeid 0x20<link>
        ether 00:0c:29:bd:fb:4a  txqueuelen 1000  (Ethernet)
        RX packets 2833  bytes 356496 (348.1 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 3103  bytes 1317158 (1.2 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

vxlan_docker: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
        inet6 fe80::4431:deff:fe40:88d1  prefixlen 64  scopeid 0x20<link>
        ether 46:31:de:40:88:d1  txqueuelen 1000  (Ethernet)
        RX packets 20  bytes 1364 (1.3 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 20  bytes 1364 (1.3 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

# k8s-2 节点：
[root@k8s-2 ~]# docker exec -it vxlan-c02 ifconfig 
eth0      Link encap:Ethernet  HWaddr 02:42:AC:11:00:03  
          inet addr:172.17.0.3  Bcast:172.17.255.255  Mask:255.255.0.0
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:18 errors:0 dropped:0 overruns:0 frame:0
          TX packets:3 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0 
          RX bytes:1328 (1.2 KiB)  TX bytes:182 (182.0 B)
---#
docker0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
        inet 172.17.0.1  netmask 255.255.0.0  broadcast 172.17.255.255
        inet6 fe80::42:dcff:fea0:192e  prefixlen 64  scopeid 0x20<link>
        ether 02:42:dc:a0:19:2e  txqueuelen 0  (Ethernet)
        RX packets 6  bytes 308 (308.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 5  bytes 526 (526.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

ens33: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 172.12.1.12  netmask 255.255.255.0  broadcast 172.12.1.255
        inet6 fe80::a9cd:74a4:47fc:9fec  prefixlen 64  scopeid 0x20<link>
        ether 00:0c:29:e2:bf:86  txqueuelen 1000  (Ethernet)
        RX packets 2620  bytes 1116242 (1.0 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 2065  bytes 282698 (276.0 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
vxlan_docker: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
        inet6 fe80::d463:d8ff:fee1:8b73  prefixlen 64  scopeid 0x20<link>
        ether d6:63:d8:e1:8b:73  txqueuelen 1000  (Ethernet)
        RX packets 8  bytes 420 (420.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 20  bytes 1364 (1.3 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0


#####################



#4.抓包：
#4.1：从容器到docker0网桥：eth0接口抓包
No.     Time                          Source                Destination           Protocol Length Info
      1 2021-07-26 20:54:03.857948    02:42:ac:11:00:02     Broadcast             ARP      42     Who has 172.17.0.3? Tell 172.17.0.2

Frame 1: 42 bytes on wire (336 bits), 42 bytes captured (336 bits)
Ethernet II, Src: 02:42:ac:11:00:02 (02:42:ac:11:00:02), Dst: Broadcast (ff:ff:ff:ff:ff:ff)
Address Resolution Protocol (request)

No.     Time                          Source                Destination           Protocol Length Info
      2 2021-07-26 20:54:03.858531    02:42:ac:11:00:03     02:42:ac:11:00:02     ARP      42     172.17.0.3 is at 02:42:ac:11:00:03

Frame 2: 42 bytes on wire (336 bits), 42 bytes captured (336 bits)
Ethernet II, Src: 02:42:ac:11:00:03 (02:42:ac:11:00:03), Dst: 02:42:ac:11:00:02 (02:42:ac:11:00:02)
Address Resolution Protocol (reply)

No.     Time                          Source                Destination           Protocol Length Info
      3 2021-07-26 20:54:03.858537    172.17.0.2            172.17.0.3            ICMP     98     Echo (ping) request  id=0x1d00, seq=0/0, ttl=64 (reply in 4)

Frame 3: 98 bytes on wire (784 bits), 98 bytes captured (784 bits)
Ethernet II, Src: 02:42:ac:11:00:02 (02:42:ac:11:00:02), Dst: 02:42:ac:11:00:03 (02:42:ac:11:00:03)
Internet Protocol Version 4, Src: 172.17.0.2, Dst: 172.17.0.3
Internet Control Message Protocol

No.     Time                          Source                Destination           Protocol Length Info
      4 2021-07-26 20:54:03.858948    172.17.0.3            172.17.0.2            ICMP     98     Echo (ping) reply    id=0x1d00, seq=0/0, ttl=64 (request in 3)

Frame 4: 98 bytes on wire (784 bits), 98 bytes captured (784 bits)
Ethernet II, Src: 02:42:ac:11:00:03 (02:42:ac:11:00:03), Dst: 02:42:ac:11:00:02 (02:42:ac:11:00:02)
Internet Protocol Version 4, Src: 172.17.0.3, Dst: 172.17.0.2
Internet Control Message Protocol

#4.2：学习对端VTEP的MAC地址：vxlan_docker上抓包
No.     Time                          Source                Destination           Protocol Length Info
      3 2021-07-26 20:54:03.857965    02:42:ac:11:00:02     Broadcast             ARP      42     Who has 172.17.0.3? Tell 172.17.0.2

Frame 3: 42 bytes on wire (336 bits), 42 bytes captured (336 bits)
Ethernet II, Src: 02:42:ac:11:00:02 (02:42:ac:11:00:02), Dst: Broadcast (ff:ff:ff:ff:ff:ff)
Address Resolution Protocol (request)

No.     Time                          Source                Destination           Protocol Length Info
      4 2021-07-26 20:54:03.858469    02:42:ac:11:00:03     02:42:ac:11:00:02     ARP      42     172.17.0.3 is at 02:42:ac:11:00:03

Frame 4: 42 bytes on wire (336 bits), 42 bytes captured (336 bits)
Ethernet II, Src: 02:42:ac:11:00:03 (02:42:ac:11:00:03), Dst: 02:42:ac:11:00:02 (02:42:ac:11:00:02)
Address Resolution Protocol (reply)

No.     Time                          Source                Destination           Protocol Length Info
      5 2021-07-26 20:54:03.858549    172.17.0.2            172.17.0.3            ICMP     98     Echo (ping) request  id=0x1d00, seq=0/0, ttl=64 (reply in 6)

Frame 5: 98 bytes on wire (784 bits), 98 bytes captured (784 bits)
Ethernet II, Src: 02:42:ac:11:00:02 (02:42:ac:11:00:02), Dst: 02:42:ac:11:00:03 (02:42:ac:11:00:03)
Internet Protocol Version 4, Src: 172.17.0.2, Dst: 172.17.0.3
Internet Control Message Protocol

No.     Time                          Source                Destination           Protocol Length Info
      6 2021-07-26 20:54:03.858915    172.17.0.3            172.17.0.2            ICMP     98     Echo (ping) reply    id=0x1d00, seq=0/0, ttl=64 (request in 5)

Frame 6: 98 bytes on wire (784 bits), 98 bytes captured (784 bits)
Ethernet II, Src: 02:42:ac:11:00:03 (02:42:ac:11:00:03), Dst: 02:42:ac:11:00:02 (02:42:ac:11:00:02)
Internet Protocol Version 4, Src: 172.17.0.3, Dst: 172.17.0.2
Internet Control Message Protocol

#4.3：构造VxLAN数据报文：
No.     Time                          Source                Destination           Protocol Length Info
    683 2021-07-26 20:54:03.857992    02:42:ac:11:00:02     Broadcast             ARP      92     Who has 172.17.0.3? Tell 172.17.0.2

Frame 683: 92 bytes on wire (736 bits), 92 bytes captured (736 bits)
Ethernet II, Src: Vmware_bd:fb:4a (00:0c:29:bd:fb:4a), Dst: Vmware_e2:bf:86 (00:0c:29:e2:bf:86)
Internet Protocol Version 4, Src: 172.12.1.11, Dst: 172.12.1.12
User Datagram Protocol, Src Port: 51305, Dst Port: 4789
Virtual eXtensible Local Area Network
Ethernet II, Src: 02:42:ac:11:00:02 (02:42:ac:11:00:02), Dst: Broadcast (ff:ff:ff:ff:ff:ff)
Address Resolution Protocol (request)

No.     Time                          Source                Destination           Protocol Length Info
    684 2021-07-26 20:54:03.858469    02:42:ac:11:00:03     02:42:ac:11:00:02     ARP      92     172.17.0.3 is at 02:42:ac:11:00:03

Frame 684: 92 bytes on wire (736 bits), 92 bytes captured (736 bits)
Ethernet II, Src: Vmware_e2:bf:86 (00:0c:29:e2:bf:86), Dst: Vmware_bd:fb:4a (00:0c:29:bd:fb:4a)
Internet Protocol Version 4, Src: 172.12.1.12, Dst: 172.12.1.11
User Datagram Protocol, Src Port: 51881, Dst Port: 4789
Virtual eXtensible Local Area Network
Ethernet II, Src: 02:42:ac:11:00:03 (02:42:ac:11:00:03), Dst: 02:42:ac:11:00:02 (02:42:ac:11:00:02)
Address Resolution Protocol (reply)

No.     Time                          Source                Destination           Protocol Length Info
    685 2021-07-26 20:54:03.858562    172.17.0.2            172.17.0.3            ICMP     148    Echo (ping) request  id=0x1d00, seq=0/0, ttl=64 (reply in 688)

Frame 685: 148 bytes on wire (1184 bits), 148 bytes captured (1184 bits)
Ethernet II, Src: Vmware_bd:fb:4a (00:0c:29:bd:fb:4a), Dst: Vmware_e2:bf:86 (00:0c:29:e2:bf:86)
Internet Protocol Version 4, Src: 172.12.1.11, Dst: 172.12.1.12
User Datagram Protocol, Src Port: 45497, Dst Port: 4789
Virtual eXtensible Local Area Network
Ethernet II, Src: 02:42:ac:11:00:02 (02:42:ac:11:00:02), Dst: 02:42:ac:11:00:03 (02:42:ac:11:00:03)
Internet Protocol Version 4, Src: 172.17.0.2, Dst: 172.17.0.3
Internet Control Message Protocol

No.     Time                          Source                Destination           Protocol Length Info
    688 2021-07-26 20:54:03.858915    172.17.0.3            172.17.0.2            ICMP     148    Echo (ping) reply    id=0x1d00, seq=0/0, ttl=64 (request in 685)

Frame 688: 148 bytes on wire (1184 bits), 148 bytes captured (1184 bits)
Ethernet II, Src: Vmware_e2:bf:86 (00:0c:29:e2:bf:86), Dst: Vmware_bd:fb:4a (00:0c:29:bd:fb:4a)
Internet Protocol Version 4, Src: 172.12.1.12, Dst: 172.12.1.11
User Datagram Protocol, Src Port: 59330, Dst Port: 4789
Virtual eXtensible Local Area Network
Ethernet II, Src: 02:42:ac:11:00:03 (02:42:ac:11:00:03), Dst: 02:42:ac:11:00:02 (02:42:ac:11:00:02)
Internet Protocol Version 4, Src: 172.17.0.3, Dst: 172.17.0.2
Internet Control Message Protoco