音频、视频

采用 FFmpeg 将流媒体切片（编码），类似 m3u8 作为目录，切成多个视频 ts 文件
image.png
再根据不同清晰度编码多个版本，来应对用户再不同网络环境情况
image.png

直播

动态 m3u8 文件，不断编码写入目录，并上传内容，再由流媒体服务器分发

传输

通常视频文件较大，所以传输需要压缩，播放需要解码
image.png

压缩算法中，有一个概念【宏块】：抽象提取连续画面（帧）中未发生变化的区域

视频通话

两台不在同一内网下的主机 Host1，Host2 需要进行视频通话，由于 NAT 和 路由的存在，是不能直接通信的，需要 NAT 穿透
image.png
image.png
image.png
image.png

例子：https://github.com/c-yyy/webRTC-JS-Demo

image.png

部署线上视频通话服务

  1. 前端（具体请看示例代码 Vue 3） ```javascript /* socket 配置 / export const socket_config = { url: ‘ws://127.0.0.1:3479’, config: {} }

/* webRTC ICE 配置 / export const webRTC_configuration = { iceServers: [ { urls: ‘stun:stun.l.google.com:19302’ } ], iceTransportPolicy: ‘relay’, // 可选值 all or relay iceCandidatePoolSize: 0, optionalArgument: { optional: [{ DtlsSrtpKeyAgreement: true }, { googImprovedWifiBwe: true }, { googScreencastMinBitrate: 300 }, { googIPv6: true }, { googDscp: true }, { googCpuUnderuseThreshold: 55 }, { googCpuOveruseThreshold: 85 }, { googSuspendBelowMinBitrate: true }, { googCpuOveruseDetection: true }], mandatory: {} } }

/* webRTC offer 配置 / export const webRTC_offerOptions = { offerToReceiveAudio: 1, offerToReceiveVideo: 1 }

/* webRTC 流媒体 配置 / export const webRTC_mediaStreamQuery = { video: { facingMode: “user”, width: { min: 600, ideal: 1280, max: 1920, }, height: { min: 400, ideal: 800, max: 1080 } }, audio: true }

  1. ```vue
  2. <template>
  3.   <div class="video-page">
  4.     <video class="localVideo" autoplay ref="localVideo" muted />
  5.     <video class="remoteVideo" autoplay ref="remoteVideo" />
  6.     <audio src="https://qiniu.terminal.laway.cn/wait.mp3" loop ref="waitAudio" />
  8.     <section class="phone-section">
  9.       <PhoneIcon />
  10.       <span>{{ callTime }}</span>
  11.     </section>
  13.     <section class="ping-fps" @click="showLog = !showLog">
  14.       <span v-show="FPS !== undefined">fps {{ FPS }}</span>
  15.       <p v-show="showLog" style="width:30vw">{{ Logs }}</p>
  16.     </section>
  17.   </div>
  18.   <Calling v-if="callTime === '00:00:00'" />
  19. </template>
  21. <script>
  22. import { ref, defineComponent, onMounted, onUnmounted } from 'vue'
  23. import PhoneIcon from '@/components/PhoneIcon.vue'
  24. import Calling from '@/components/Calling.vue'
  25. import { Dialog } from 'vant';
  26. import router from '@/router'
  27. import { webRTC_mediaStreamQuery, webRTC_configuration, socket_config } from '@/config/webRTC'
  29. export default defineComponent({
  30.   components: {
  31.     PhoneIcon,
  32.     Calling
  33.   },
  34.   setup () {
  35.     let localVideo = ref(null)
  36.     let remoteVideo = ref(null)
  37.     let waitAudio = ref(null)
  38.     let showLog = ref(false)
  39.     let Logs = ref('log')
  40.     // let RTT = ref('')
  41.     let FPS = ref('')
  42.     let timer = null
  43.     let callTime = ref('00:00:00')
  44.     const isTest = !window.location.href.includes('type=test')
  45.     // eslint-disable-next-line no-unused-vars
  46.     let localMediaStream, remoteMediaStream = null
  47.     const connection = new RTCPeerConnection(webRTC_configuration)
  48.     // eslint-disable-next-line no-undef
  49.     const socket = io(socket_config.url, socket_config.config)
  50.     console.log('RTCPeerConnection', connection)
  52.     const closeVideo = () => {
  53.       localMediaStream.getTracks().map(track => track.stop())
  54.       localMediaStream = null
  55.       remoteMediaStream = null
  56.       clearInterval(timer)
  57.       connection.close()
  58.       socket.emit('leave', {
  59.         from: 'test'
  60.       })
  61.       socket.close()
  62.     }
  64.     const formateTime = (time) => {
  65.       if (time < 60) {
  66.         return `00:00:${time < 10 ? '0' + time : time}`;
  67.       } else if (time > 60 && time < 3600) {
  68.         const min = parseInt(time / 60);
  69.         const sed = time % 60;
  70.         return `00:${min < 10 ? '0' + min : min}:${sed < 10 ? '0' + sed : sed}`;
  71.       } else if (time > 3600) {
  72.         const h = parseInt(time / 3600);
  73.         const min = parseInt((time % 3600) / 60);
  74.         const sed = (time % 3600) % 60;
  75.         return `${h < 10 ? '0' + h : h}:${min < 10 ? '0' + min : min}:${
  76.           sed < 10 ? '0' + sed : sed
  77.         }`;
  78.       }
  79.     }
  81.     const startTiming = () => {
  82.       let time = 0;
  83.       timer = setInterval(async () => {
  84.         time += 1;
  85.         callTime.value = formateTime(time);
  86.         const stats = await connection.getStats(null)
  87.         stats.forEach(report => {
  88.           if (report.type === "inbound-rtp" && report.kind === "video") {
  89.             Logs.value = report
  90.             // if (report.totalRoundTripTime && report.responsesReceived) {
  91.             //   RTT.value = parseInt((report.totalRoundTripTime / report.responsesReceived) * 1000)
  92.             // }
  93.             FPS.value = report.framesPerSecond
  94.           }
  95.         })
  96.       }, 1e3);
  97.     }
  99.     const getLocalVideo = async () => {
  100.       try {
  101.         document.querySelector('.localVideo').setAttribute('autoplay', '');
  102.         document.querySelector('.localVideo').setAttribute('muted', '');
  103.         document.querySelector('.localVideo').setAttribute('playsinline', '');
  104.         document.querySelector('.remoteVideo').setAttribute('autoplay', '');
  105.         document.querySelector('.remoteVideo').setAttribute('muted', '');
  106.         document.querySelector('.remoteVideo').setAttribute('playsinline', '');
  107.         localMediaStream = await navigator.mediaDevices.getUserMedia(webRTC_mediaStreamQuery)
  108.         localVideo.value.srcObject = localMediaStream
  109.         console.log(localMediaStream)
  110.         isTest && waitAudio.value.play()
  111.         socket.emit('message', {
  112.           type: 'call',
  113.           from: 'test'
  114.         })
  115.         socket.on('leaved', () => {
  116.           console.log('leaved')
  117.           Dialog.alert({
  118.             title: '通话结束',
  119.             message: '对方已挂断～',
  120.           }).then(() => {
  121.             // on close
  122.             router.replace('/')
  123.           });
  124.         })
  126.         /** Track 方式 */
  127.         localMediaStream.getTracks().forEach(track => connection.addTrack(track, localMediaStream))
  128.         connection.ontrack = (e) => {
  129.           console.log('ontrack', e)
  130.           if (remoteVideo.value.srcObject !== e.streams[0]) {
  131.             waitAudio.value.pause()
  133.             remoteMediaStream = e.streams[0]
  134.             remoteVideo.value.srcObject = e.streams[0]
  136.             startTiming()
  137.           }
  138.         }
  140.         /** Stream 方式 */
  141.         // connection.addStream(localMediaStream)
  142.         // connection.onaddstream = e => {
  143.         //   console.log('onaddstream', e)
  144.         //   if (remoteVideo.value.srcObject !== e.streams[0]) {
  145.         //     waitAudio.value.pause()
  147.         //     remoteMediaStream = e.streams[0]
  148.         //     remoteVideo.value.srcObject = e.streams[0]
  150.         //     startTiming()
  151.         //   }
  152.         // }
  154.         connection.onicecandidate = e => {
  155.           console.log('onicecandidate', e)
  156.           if (e.candidate) {
  157.             socket.emit('message', {
  158.               type: 'candidate',
  159.               from: 'test',
  160.               candidate: e.candidate
  161.             })
  162.           }
  163.         }
  165.         if (!isTest) {
  166.           connection.createOffer({
  167.             offerToReceiveAudio: true,
  168.             offerToReceiveVideo: true
  169.           }).then(offer => {
  170.             console.log('offer', offer)
  171.             connection.setLocalDescription(offer)
  172.             socket.emit('message', {
  173.               type: 'offer',
  174.               from: 'test',
  175.               offer
  176.             })
  177.           })
  178.         }
  179.       } catch (error) {
  180.         Dialog.alert({
  181.             title: '提示',
  182.             message: '请打开授权～',
  183.           }).then(() => {
  184.             // on close
  185.             router.replace('/')
  186.           });
  187.         console.error(error)
  188.       }
  189.     }
  191.     const getRemoteVide = () => {
  192.       socket.on('message', async data => {
  193.         const { type } = data
  194.         switch (type) {
  195.           case 'offer':
  196.             if (isTest && data.offer) {
  197.               await connection.setRemoteDescription(new RTCSessionDescription(data.offer))
  198.               connection.createAnswer().then(answer => {
  199.                 console.log('answer', answer)
  200.                 connection.setLocalDescription(answer)
  201.                 socket.emit('message', {
  202.                   type: 'answer',
  203.                   from: 'test',
  204.                   answer
  205.                 })
  206.               })
  207.               break;
  208.             }
  209.             return
  210.           case 'answer':
  211.             if (!isTest && data.answer) {
  212.               connection.setRemoteDescription(new RTCSessionDescription(data.answer))
  213.             }
  214.             break;
  215.           case 'candidate':
  216.             if (connection && data.candidate && data.from !== isTest) {
  217.               connection.addIceCandidate(new RTCIceCandidate(data.candidate))
  218.             }
  219.             break;
  220.           case 'close':
  221.             closeVideo()
  222.         }
  223.       })
  224.     }
  226.     onMounted(() => {
  227.       getLocalVideo()
  228.       getRemoteVide()
  229.     })
  231.     onUnmounted(() => {
  232.       closeVideo()
  233.     })
  235.     return {
  236.       localVideo,
  237.       remoteVideo,
  238.       waitAudio,
  239.       callTime,
  240.       showLog,
  241.       Logs,
  242.       // RTT,
  243.       FPS
  244.     }
  245.   },
  246. })
  247. </script>
  249. <style scoped>
  250. .video-page {
  251.   width: 100vw;
  252.   height: 100vh;
  253.   position: relative;
  254. }
  255. .localVideo,
  256. .remoteVideo {
  257.   background: #303133;
  258. }
  259. .localVideo {
  260.   position: absolute;
  261.   top: 0;
  262.   right: 0;
  263.   width: 30%;
  264.   height: 30vh;
  265.   background: #fff;
  266.   z-index: 3;
  267. }
  268. .remoteVideo {
  269.   width: 100%;
  270.   height: 100%;
  271. }
  272. .phone-section {
  273.   position: absolute;
  274.   top: 80%;
  275.   left: 50%;
  276.   transform: translate(-80%, -50%);
  277.   z-index: 5;
  278.   color: #fff;
  279. }
  280. .ping-fps {
  281.   color: #fff;
  282.   position: absolute;
  283.   top: 10px;
  284.   left: 10px;
  285.   z-index: 3;
  286. }
  287. </style>

  1. ICE 服务（穿透 NAT ）

    1. # docker 部署一个私有 ICE 服务
    2. # coTurn 开源地址：https://github.com/coturn/coturn
    3. # 参考：https://www.wyr.me/post/640
    4. docker run -d --network=host instrumentisto/coturn --realm=my.realm.org --external-ip='$(detect-external-ip)' --user=user123:pass123

  2. 本地视频流接入、发送流

  3. 发送 offer
  4. Socket监听远程视频流接入
  5. 另一端接收 offer 并发送 answer
  6. 实时显示延迟和 FPS
  7. 正常 Nginx 部署（最好支持 https）
    1. location /webRTC-test {
    2. alias /home/ubuntu/site/yls/dist;
    3. index index.html;
    4. expires -1;
    5. try_files $uri $uri/ /dist/index.html;
    6. }
  1. 后端
    1. Socket 服务（支持跨域）
    2. 支持开启房间 room
    3. 丢服务器 pm2 start main.js --name webRTC-socket-test
    4. Nginx proxy 反向代理
      1. location /socket.io {
      2. add_header 'Access-Control-Allow-Headers' 'Accept, Authorization, Cache-Control, Content-Type, DNT, If-Modified-Since, Keep-Alive, Origin, User-Agen
      3. t, X-Requested-With, Token, x-access-token' always;
      4. # 设置主机头和客户端真实地址，以便服务器获取客户端真实 IP
      5. proxy_redirect off;
      6. proxy_set_header Host $host;
      7. proxy_set_header X-Real-IP $remote_addr;
      8. proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
      9. proxy_http_version 1.1;
      10. proxy_set_header X-NginX-Proxy true;
      11. proxy_set_header Upgrade $http_upgrade;
      12. proxy_set_header Connection "upgrade";
      13. # 禁用缓存
      14. proxy_buffering off;
      15. # 反向代理的地址
      16. proxy_pass http://127.0.0.1:3479;
      17. }