1、runc使用

2、runc执行过程

根据用户的config.json构建逻辑container，然后调用container.Start()方法，此处重点关注container.Start()方法

2.1、创建exec fifo

exec fifi用户监听用户执行start命令，真正启动用户进程用的 execFifoFilename = “exec.fifo” root默认为/run/runc

  1. // 若为1号进程，则需要创建exec fifo
  2. // 当在容器内在起一个进程则不需要
  3. func (c *linuxContainer) createExecFifo() error {
  4.     rootuid, err := c.Config().HostRootUID()
  5.     if err != nil {
  6.         return err
  7.     }
  8.     rootgid, err := c.Config().HostRootGID()
  9.     if err != nil {
  10.         return err
  11.     }
  13.     fifoName := filepath.Join(c.root, execFifoFilename)
  14.     if _, err := os.Stat(fifoName); err == nil {
  15.         return fmt.Errorf("exec fifo %s already exists", fifoName)
  16.     }
  17.     oldMask := unix.Umask(0000)
  18.     if err := unix.Mkfifo(fifoName, 0622); err != nil {
  19.         unix.Umask(oldMask)
  20.         return err
  21.     }
  22.     unix.Umask(oldMask)
  23.     return os.Chown(fifoName, rootuid, rootgid)
  24. }

2.2、初始化父进程(create操作对应的为initProcess)

initProcess负责对runc init进行控制和信号处理

2.2.1、构建ParentProcess

  1. func (c *linuxContainer) newParentProcess(p *Process) (parentProcess, error) {
  2.     // 构建通信pipe
  3.     parentInitPipe, childInitPipe, err := utils.NewSockPair("init")
  4.     if err != nil {
  5.         return nil, newSystemErrorWithCause(err, "creating new init pipe")
  6.     }
  7.     messageSockPair := filePair{parentInitPipe, childInitPipe}
  9.     // 构建日志pipe
  10.     // parent负责读，child负责写
  11.     parentLogPipe, childLogPipe, err := os.Pipe()
  12.     if err != nil {
  13.         return nil, fmt.Errorf("Unable to create the log pipe:  %s", err)
  14.     }
  15.     logFilePair := filePair{parentLogPipe, childLogPipe}
  17.     // 此处就是封装runc init cmd，并且将用户二进制路径作为参数传递到runc init命令中
  18.     cmd := c.commandTemplate(p, childInitPipe, childLogPipe)
  20.     // 若不是第一次init，则封装为SetnsProcess
  21.     if !p.Init {
  22.         return c.newSetnsProcess(p, cmd, messageSockPair, logFilePair)
  23.     }
  25.     // We only set up fifoFd if we're not doing a `runc exec`. The historic
  26.     // reason for this is that previously we would pass a dirfd that allowed
  27.     // for container rootfs escape (and not doing it in `runc exec` avoided
  28.     // that problem), but we no longer do that. However, there's no need to do
  29.     // this for `runc exec` so we just keep it this way to be safe.
  30.     if err := c.includeExecFifo(cmd); err != nil {
  31.         return nil, newSystemErrorWithCause(err, "including execfifo in cmd.Exec setup")
  32.     }
  33.     return c.newInitProcess(p, cmd, messageSockPair, logFilePair)
  34. }

2.2.1、构建initProcess

  1. func (c *linuxContainer) newInitProcess(p *Process, cmd *exec.Cmd, messageSockPair, logFilePair filePair) (*initProcess, error) {
  2.     cmd.Env = append(cmd.Env, "_LIBCONTAINER_INITTYPE="+string(initStandard))
  3.     nsMaps := make(map[configs.NamespaceType]string)
  4.     for _, ns := range c.config.Namespaces {
  5.         if ns.Path != "" {
  6.             nsMaps[ns.Type] = ns.Path
  7.         }
  8.     }
  9.     _, sharePidns := nsMaps[configs.NEWPID]
  10.     data, err := c.bootstrapData(c.config.Namespaces.CloneFlags(), nsMaps)
  11.     if err != nil {
  12.         return nil, err
  13.     }
  14.     init := &initProcess{
  15.         cmd:             cmd,
  16.         messageSockPair: messageSockPair,
  17.         logFilePair:     logFilePair,
  18.         manager:         c.cgroupManager,
  19.         intelRdtManager: c.intelRdtManager,
  20.         config:          c.newInitConfig(p),
  21.         container:       c,
  22.         process:         p,
  23.         bootstrapData:   data,
  24.         sharePidns:      sharePidns,
  25.     }
  26.     c.initProcess = init
  27.     return init, nil
  28. }

2.3、启动父进程(create操作对应的为initProcess)

启动父进程之前需要重新向子进程日志，就是从parentPipe读取日志，然后打到标准输出

  1. func (p *initProcess) start() (retErr error) {
  3.     // 结束之后关闭parent的unix socket
  4.     defer p.messageSockPair.parent.Close()
  6.     // 启动子进程，此处为runc init
  7.     // 在factory创建逻辑容器的时候指定
  8.     err := p.cmd.Start()
  10.     p.process.ops = p
  12.     // 父进程不需要操作子进程端的文件句柄，因此需要close
  13.     p.messageSockPair.child.Close()
  14.     p.logFilePair.child.Close()
  15.     if err != nil {
  16.         p.process.ops = nil
  17.         return newSystemErrorWithCause(err, "starting init process command")
  18.     }
  20.     defer func() {
  21.         if retErr != nil {
  22.             // init might be killed by the kernel's OOM killer.
  23.             oom, err := p.manager.OOMKillCount()
  24.             if err != nil {
  25.                 logrus.WithError(err).Warn("unable to get oom kill count")
  26.             } else if oom > 0 {
  27.                 // Does not matter what the particular error was,
  28.                 // its cause is most probably OOM, so report that.
  29.                 const oomError = "container init was OOM-killed (memory limit too low?)"
  31.                 if logrus.GetLevel() >= logrus.DebugLevel {
  32.                     // Only show the original error if debug is set,
  33.                     // as it is not generally very useful.
  34.                     retErr = newSystemErrorWithCause(retErr, oomError)
  35.                 } else {
  36.                     retErr = newSystemError(errors.New(oomError))
  37.                 }
  38.             }
  40.             // terminate the process to ensure we can remove cgroups
  41.             if err := ignoreTerminateErrors(p.terminate()); err != nil {
  42.                 logrus.WithError(err).Warn("unable to terminate initProcess")
  43.             }
  45.             p.manager.Destroy()
  46.             if p.intelRdtManager != nil {
  47.                 p.intelRdtManager.Destroy()
  48.             }
  49.         }
  50.     }()
  52.     // Do this before syncing with child so that no children can escape the
  53.     // cgroup. We don't need to worry about not doing this and not being root
  54.     // because we'd be using the rootless cgroup manager in that case.
  55.     if err := p.manager.Apply(p.pid()); err != nil {
  56.         return newSystemErrorWithCause(err, "applying cgroup configuration for process")
  57.     }
  58.     if p.intelRdtManager != nil {
  59.         if err := p.intelRdtManager.Apply(p.pid()); err != nil {
  60.             return newSystemErrorWithCause(err, "applying Intel RDT configuration for process")
  61.         }
  62.     }
  63.     if _, err := io.Copy(p.messageSockPair.parent, p.bootstrapData); err != nil {
  64.         return newSystemErrorWithCause(err, "copying bootstrap data to pipe")
  65.     }
  66.     childPid, err := p.getChildPid()
  67.     if err != nil {
  68.         return newSystemErrorWithCause(err, "getting the final child's pid from pipe")
  69.     }
  71.     // Save the standard descriptor names before the container process
  72.     // can potentially move them (e.g., via dup2()).  If we don't do this now,
  73.     // we won't know at checkpoint time which file descriptor to look up.
  74.     fds, err := getPipeFds(childPid)
  75.     if err != nil {
  76.         return newSystemErrorWithCausef(err, "getting pipe fds for pid %d", childPid)
  77.     }
  78.     p.setExternalDescriptors(fds)
  80.     // Now it's time to setup cgroup namesapce
  81.     if p.config.Config.Namespaces.Contains(configs.NEWCGROUP) && p.config.Config.Namespaces.PathOf(configs.NEWCGROUP) == "" {
  82.         if _, err := p.messageSockPair.parent.Write([]byte{createCgroupns}); err != nil {
  83.             return newSystemErrorWithCause(err, "sending synchronization value to init process")
  84.         }
  85.     }
  87.     // Wait for our first child to exit
  88.     if err := p.waitForChildExit(childPid); err != nil {
  89.         return newSystemErrorWithCause(err, "waiting for our first child to exit")
  90.     }
  92.     if err := p.createNetworkInterfaces(); err != nil {
  93.         return newSystemErrorWithCause(err, "creating network interfaces")
  94.     }
  95.     if err := p.updateSpecState(); err != nil {
  96.         return newSystemErrorWithCause(err, "updating the spec state")
  97.     }
  98.     if err := p.sendConfig(); err != nil {
  99.         return newSystemErrorWithCause(err, "sending config to init process")
  100.     }
  101.     var (
  102.         sentRun    bool
  103.         sentResume bool
  104.     )
  106.     ierr := parseSync(p.messageSockPair.parent, func(sync *syncT) error {
  107.         switch sync.Type {
  108.         case procReady:
  109.             // set rlimits, this has to be done here because we lose permissions
  110.             // to raise the limits once we enter a user-namespace
  111.             if err := setupRlimits(p.config.Rlimits, p.pid()); err != nil {
  112.                 return newSystemErrorWithCause(err, "setting rlimits for ready process")
  113.             }
  114.             // call prestart and CreateRuntime hooks
  115.             if !p.config.Config.Namespaces.Contains(configs.NEWNS) {
  116.                 // Setup cgroup before the hook, so that the prestart and CreateRuntime hook could apply cgroup permissions.
  117.                 if err := p.manager.Set(p.config.Config); err != nil {
  118.                     return newSystemErrorWithCause(err, "setting cgroup config for ready process")
  119.                 }
  120.                 if p.intelRdtManager != nil {
  121.                     if err := p.intelRdtManager.Set(p.config.Config); err != nil {
  122.                         return newSystemErrorWithCause(err, "setting Intel RDT config for ready process")
  123.                     }
  124.                 }
  126.                 if p.config.Config.Hooks != nil {
  127.                     s, err := p.container.currentOCIState()
  128.                     if err != nil {
  129.                         return err
  130.                     }
  131.                     // initProcessStartTime hasn't been set yet.
  132.                     s.Pid = p.cmd.Process.Pid
  133.                     s.Status = specs.StateCreating
  134.                     hooks := p.config.Config.Hooks
  136.                     if err := hooks[configs.Prestart].RunHooks(s); err != nil {
  137.                         return err
  138.                     }
  139.                     if err := hooks[configs.CreateRuntime].RunHooks(s); err != nil {
  140.                         return err
  141.                     }
  142.                 }
  143.             }
  145.             // generate a timestamp indicating when the container was started
  146.             p.container.created = time.Now().UTC()
  147.             p.container.state = &createdState{
  148.                 c: p.container,
  149.             }
  151.             // NOTE: If the procRun state has been synced and the
  152.             // runc-create process has been killed for some reason,
  153.             // the runc-init[2:stage] process will be leaky. And
  154.             // the runc command also fails to parse root directory
  155.             // because the container doesn't have state.json.
  156.             //
  157.             // In order to cleanup the runc-init[2:stage] by
  158.             // runc-delete/stop, we should store the status before
  159.             // procRun sync.
  160.             state, uerr := p.container.updateState(p)
  161.             if uerr != nil {
  162.                 return newSystemErrorWithCause(err, "store init state")
  163.             }
  164.             p.container.initProcessStartTime = state.InitProcessStartTime
  166.             // Sync with child.
  167.             if err := writeSync(p.messageSockPair.parent, procRun); err != nil {
  168.                 return newSystemErrorWithCause(err, "writing syncT 'run'")
  169.             }
  170.             sentRun = true
  171.         case procHooks:
  172.             // Setup cgroup before prestart hook, so that the prestart hook could apply cgroup permissions.
  173.             if err := p.manager.Set(p.config.Config); err != nil {
  174.                 return newSystemErrorWithCause(err, "setting cgroup config for procHooks process")
  175.             }
  176.             if p.intelRdtManager != nil {
  177.                 if err := p.intelRdtManager.Set(p.config.Config); err != nil {
  178.                     return newSystemErrorWithCause(err, "setting Intel RDT config for procHooks process")
  179.                 }
  180.             }
  181.             if p.config.Config.Hooks != nil {
  182.                 s, err := p.container.currentOCIState()
  183.                 if err != nil {
  184.                     return err
  185.                 }
  186.                 // initProcessStartTime hasn't been set yet.
  187.                 s.Pid = p.cmd.Process.Pid
  188.                 s.Status = specs.StateCreating
  189.                 hooks := p.config.Config.Hooks
  191.                 if err := hooks[configs.Prestart].RunHooks(s); err != nil {
  192.                     return err
  193.                 }
  194.                 if err := hooks[configs.CreateRuntime].RunHooks(s); err != nil {
  195.                     return err
  196.                 }
  197.             }
  198.             // Sync with child.
  199.             if err := writeSync(p.messageSockPair.parent, procResume); err != nil {
  200.                 return newSystemErrorWithCause(err, "writing syncT 'resume'")
  201.             }
  202.             sentResume = true
  203.         default:
  204.             return newSystemError(errors.New("invalid JSON payload from child"))
  205.         }
  207.         return nil
  208.     })
  210.     if !sentRun {
  211.         return newSystemErrorWithCause(ierr, "container init")
  212.     }
  213.     if p.config.Config.Namespaces.Contains(configs.NEWNS) && !sentResume {
  214.         return newSystemError(errors.New("could not synchronise after executing prestart and CreateRuntime hooks with container process"))
  215.     }
  216.     if err := unix.Shutdown(int(p.messageSockPair.parent.Fd()), unix.SHUT_WR); err != nil {
  217.         return newSystemErrorWithCause(err, "shutting down init pipe")
  218.     }
  220.     // Must be done after Shutdown so the child will exit and we can wait for it.
  221.     if ierr != nil {
  222.         p.wait()
  223.         return ierr
  224.     }
  225.     return nil
  226. }

2.4、runc init进程

  1. func (l *linuxStandardInit) Init() error {
  2.     runtime.LockOSThread()
  3.     defer runtime.UnlockOSThread()
  5.     if !l.config.Config.NoNewKeyring {
  6.         if err := selinux.SetKeyLabel(l.config.ProcessLabel); err != nil {
  7.             return err
  8.         }
  9.         defer selinux.SetKeyLabel("")
  10.         ringname, keepperms, newperms := l.getSessionRingParams()
  12.         // Do not inherit the parent's session keyring.
  13.         if sessKeyId, err := keys.JoinSessionKeyring(ringname); err != nil {
  14.             // If keyrings aren't supported then it is likely we are on an
  15.             // older kernel (or inside an LXC container). While we could bail,
  16.             // the security feature we are using here is best-effort (it only
  17.             // really provides marginal protection since VFS credentials are
  18.             // the only significant protection of keyrings).
  19.             //
  20.             // TODO(cyphar): Log this so people know what's going on, once we
  21.             //               have proper logging in 'runc init'.
  22.             if errors.Cause(err) != unix.ENOSYS {
  23.                 return errors.Wrap(err, "join session keyring")
  24.             }
  25.         } else {
  26.             // Make session keyring searcheable. If we've gotten this far we
  27.             // bail on any error -- we don't want to have a keyring with bad
  28.             // permissions.
  29.             if err := keys.ModKeyringPerm(sessKeyId, keepperms, newperms); err != nil {
  30.                 return errors.Wrap(err, "mod keyring permissions")
  31.             }
  32.         }
  33.     }
  35.     if err := setupNetwork(l.config); err != nil {
  36.         return err
  37.     }
  39.     if err := setupRoute(l.config.Config); err != nil {
  40.         return err
  41.     }
  43.     if err := prepareRootfs(l.pipe, l.config); err != nil {
  44.         return err
  45.     }
  47.     // Set up the console. This has to be done *before* we finalize the rootfs,
  48.     // but *after* we've given the user the chance to set up all of the mounts
  49.     // they wanted.
  50.     if l.config.CreateConsole {
  51.         if err := setupConsole(l.consoleSocket, l.config, true); err != nil {
  52.             return err
  53.         }
  54.         if err := system.Setctty(); err != nil {
  55.             return errors.Wrap(err, "setctty")
  56.         }
  57.     }
  59.     // 若开启了mount ns，dev设备需要重新mount
  60.     if l.config.Config.Namespaces.Contains(configs.NEWNS) {
  61.         if err := finalizeRootfs(l.config.Config); err != nil {
  62.             return err
  63.         }
  64.     }
  66.     // 设置主机名
  67.     if hostname := l.config.Config.Hostname; hostname != "" {
  68.         if err := unix.Sethostname([]byte(hostname)); err != nil {
  69.             return errors.Wrap(err, "sethostname")
  70.         }
  71.     }
  73.     // 配置AppArmorProfile
  74.     if err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil {
  75.         return errors.Wrap(err, "apply apparmor profile")
  76.     }
  78.     for key, value := range l.config.Config.Sysctl {
  79.         if err := writeSystemProperty(key, value); err != nil {
  80.             return errors.Wrapf(err, "write sysctl key %s", key)
  81.         }
  82.     }
  84.     // 设置只读路径和MaskPaths
  85.     for _, path := range l.config.Config.ReadonlyPaths {
  86.         if err := readonlyPath(path); err != nil {
  87.             return errors.Wrapf(err, "readonly path %s", path)
  88.         }
  89.     }
  90.     for _, path := range l.config.Config.MaskPaths {
  91.         if err := maskPath(path, l.config.Config.MountLabel); err != nil {
  92.             return errors.Wrapf(err, "mask path %s", path)
  93.         }
  94.     }
  96.     pdeath, err := system.GetParentDeathSignal()
  97.     if err != nil {
  98.         return errors.Wrap(err, "get pdeath signal")
  99.     }
  100.     if l.config.NoNewPrivileges {
  101.         if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil {
  102.             return errors.Wrap(err, "set nonewprivileges")
  103.         }
  104.     }
  105.     // Tell our parent that we're ready to Execv. This must be done before the
  106.     // Seccomp rules have been applied, because we need to be able to read and
  107.     // write to a socket.
  108.     if err := syncParentReady(l.pipe); err != nil {
  109.         return errors.Wrap(err, "sync ready")
  110.     }
  111.     if err := selinux.SetExecLabel(l.config.ProcessLabel); err != nil {
  112.         return errors.Wrap(err, "set process label")
  113.     }
  114.     defer selinux.SetExecLabel("")
  115.     // Without NoNewPrivileges seccomp is a privileged operation, so we need to
  116.     // do this before dropping capabilities; otherwise do it as late as possible
  117.     // just before execve so as few syscalls take place after it as possible.
  118.     if l.config.Config.Seccomp != nil && !l.config.NoNewPrivileges {
  119.         if err := seccomp.InitSeccomp(l.config.Config.Seccomp); err != nil {
  120.             return err
  121.         }
  122.     }
  123.     if err := finalizeNamespace(l.config); err != nil {
  124.         return err
  125.     }
  126.     // finalizeNamespace can change user/group which clears the parent death
  127.     // signal, so we restore it here.
  128.     if err := pdeath.Restore(); err != nil {
  129.         return errors.Wrap(err, "restore pdeath signal")
  130.     }
  131.     // Compare the parent from the initial start of the init process and make
  132.     // sure that it did not change.  if the parent changes that means it died
  133.     // and we were reparented to something else so we should just kill ourself
  134.     // and not cause problems for someone else.
  135.     if unix.Getppid() != l.parentPid {
  136.         return unix.Kill(unix.Getpid(), unix.SIGKILL)
  137.     }
  138.     // Check for the arg before waiting to make sure it exists and it is
  139.     // returned as a create time error.
  140.     name, err := exec.LookPath(l.config.Args[0])
  141.     if err != nil {
  142.         return err
  143.     }
  144.     // Close the pipe to signal that we have completed our init.
  145.     l.pipe.Close()
  147.     //-----------------------------------------------------------
  148.     //----------以下需要runc start {container_id}才能执行----------
  149.     //-----------------------------------------------------------
  151.     // Wait for the FIFO to be opened on the other side before exec-ing the
  152.     // user process. We open it through /proc/self/fd/$fd, because the fd that
  153.     // was given to us was an O_PATH fd to the fifo itself. Linux allows us to
  154.     // re-open an O_PATH fd through /proc.
  155.     fd, err := unix.Open("/proc/self/fd/"+strconv.Itoa(l.fifoFd), unix.O_WRONLY|unix.O_CLOEXEC, 0)
  156.     if err != nil {
  157.         return newSystemErrorWithCause(err, "open exec fifo")
  158.     }
  159.     if _, err := unix.Write(fd, []byte("0")); err != nil {
  160.         return newSystemErrorWithCause(err, "write 0 exec fifo")
  161.     }
  162.     // Close the O_PATH fifofd fd before exec because the kernel resets
  163.     // dumpable in the wrong order. This has been fixed in newer kernels, but
  164.     // we keep this to ensure CVE-2016-9962 doesn't re-emerge on older kernels.
  165.     // N.B. the core issue itself (passing dirfds to the host filesystem) has
  166.     // since been resolved.
  167.     // https://github.com/torvalds/linux/blob/v4.9/fs/exec.c#L1290-L1318
  168.     unix.Close(l.fifoFd)
  170.     // Set seccomp as close to execve as possible, so as few syscalls take
  171.     // place afterward (reducing the amount of syscalls that users need to
  172.     // enable in their seccomp profiles).
  173.     if l.config.Config.Seccomp != nil && l.config.NoNewPrivileges {
  174.         if err := seccomp.InitSeccomp(l.config.Config.Seccomp); err != nil {
  175.             return newSystemErrorWithCause(err, "init seccomp")
  176.         }
  177.     }
  179.     s := l.config.SpecState
  180.     s.Pid = unix.Getpid()
  181.     s.Status = specs.StateCreated
  182.     if err := l.config.Config.Hooks[configs.StartContainer].RunHooks(s); err != nil {
  183.         return err
  184.     }
  186.     // 切换镜像
  187.     if err := unix.Exec(name, l.config.Args[0:], os.Environ()); err != nil {
  188.         return newSystemErrorWithCause(err, "exec user process")
  189.     }
  190.     return nil
  191. }