1 深度剖析-CRI篇

b. 容器镜像规范：定义了容器的镜像如何打包如何将镜像转换成一个bundle。

1.2 “Containerd CRI简介”

1.3 “Containerd CRI实现”

RunPodSandbox

// RunPodSandbox creates and starts a pod-level sandbox. Runtimes should ensure
// the sandbox is in ready state.
func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandboxRequest) (_ *runtime.RunPodSandboxResponse, retErr error) {
  config := r.GetConfig()
  log.G(ctx).Debugf("Sandbox config %+v", config)
  // Generate unique id and name for the sandbox and reserve the name.
  id := util.GenerateID()
  metadata := config.GetMetadata()
  if metadata == nil {
    return nil, errors.New("sandbox config must include metadata")
  }
  name := makeSandboxName(metadata)
  log.G(ctx).WithField("podsandboxid", id).Debugf("generated id for sandbox name %q", name)
  // cleanupErr records the last error returned by the critical cleanup operations in deferred functions,
  // like CNI teardown and stopping the running sandbox task.
  // If cleanup is not completed for some reason, the CRI-plugin will leave the sandbox
  // in a not-ready state, which can later be cleaned up by the next execution of the kubelet's syncPod workflow.
  var cleanupErr error
  // Reserve the sandbox name to avoid concurrent `RunPodSandbox` request starting the
  // same sandbox.
  if err := c.sandboxNameIndex.Reserve(name, id); err != nil {
    return nil, fmt.Errorf("failed to reserve sandbox name %q: %w", name, err)
  }
  defer func() {
    // Release the name if the function returns with an error.
    // When cleanupErr != nil, the name will be cleaned in sandbox_remove.
    if retErr != nil && cleanupErr == nil {
      c.sandboxNameIndex.ReleaseByName(name)
    }
  }()
  var (
    err         error
    sandboxInfo = sb.Sandbox{ID: id}
  )
  ociRuntime, err := c.getSandboxRuntime(config, r.GetRuntimeHandler())
  if err != nil {
    return nil, fmt.Errorf("unable to get OCI runtime for sandbox %q: %w", id, err)
  }
  sandboxInfo.Runtime.Name = ociRuntime.Type
  // Retrieve runtime options
  runtimeOpts, err := generateRuntimeOptions(ociRuntime, c.config)
  if err != nil {
    return nil, fmt.Errorf("failed to generate sandbox runtime options: %w", err)
  }
  ...
  // Create initial internal sandbox object.
  sandbox := sandboxstore.NewSandbox(
    ...
  )
  if _, err := c.client.SandboxStore().Create(ctx, sandboxInfo); err != nil {
    return nil, fmt.Errorf("failed to save sandbox metadata: %w", err)
  }
  ...
  // Setup the network namespace if host networking wasn't requested.
  if !hostNetwork(config) {
    netStart := time.Now()
    // If it is not in host network namespace then create a namespace and set the sandbox
    // handle. NetNSPath in sandbox metadata and NetNS is non empty only for non host network
    // namespaces. If the pod is in host network namespace then both are empty and should not
    // be used.
    var netnsMountDir = "/var/run/netns"
    if c.config.NetNSMountsUnderStateDir {
      netnsMountDir = filepath.Join(c.config.StateDir, "netns")
    }
    sandbox.NetNS, err = netns.NewNetNS(netnsMountDir)
    if err != nil {
      return nil, fmt.Errorf("failed to create network namespace for sandbox %q: %w", id, err)
    }
    // Update network namespace in the store, which is used to generate the container's spec
    sandbox.NetNSPath = sandbox.NetNS.GetPath()
    defer func() {
      // Remove the network namespace only if all the resource cleanup is done
      if retErr != nil && cleanupErr == nil {
        if cleanupErr = sandbox.NetNS.Remove(); cleanupErr != nil {
          log.G(ctx).WithError(cleanupErr).Errorf("Failed to remove network namespace %s for sandbox %q", sandbox.NetNSPath, id)
          return
        }
        sandbox.NetNSPath = ""
      }
    }()
    if err := sandboxInfo.AddExtension(podsandbox.MetadataKey, &sandbox.Metadata); err != nil {
      return nil, fmt.Errorf("unable to save sandbox %q to store: %w", id, err)
    }
    // Save sandbox metadata to store
    if sandboxInfo, err = c.client.SandboxStore().Update(ctx, sandboxInfo, "extensions"); err != nil {
      return nil, fmt.Errorf("unable to update extensions for sandbox %q: %w", id, err)
    }
    // Define this defer to teardownPodNetwork prior to the setupPodNetwork function call.
    // This is because in setupPodNetwork the resource is allocated even if it returns error, unlike other resource
    // creation functions.
    defer func() {
      // Remove the network namespace only if all the resource cleanup is done.
      if retErr != nil && cleanupErr == nil {
        deferCtx, deferCancel := ctrdutil.DeferContext()
        defer deferCancel()
        // Teardown network if an error is returned.
        if cleanupErr = c.teardownPodNetwork(deferCtx, sandbox); cleanupErr != nil {
          log.G(ctx).WithError(cleanupErr).Errorf("Failed to destroy network for sandbox %q", id)
        }
      }
    }()
    // Setup network for sandbox.
    // Certain VM based solutions like clear containers (Issue containerd/cri-containerd#524)
    // rely on the assumption that CRI shim will not be querying the network namespace to check the
    // network states such as IP.
    // In future runtime implementation should avoid relying on CRI shim implementation details.
    // In this case however caching the IP will add a subtle performance enhancement by avoiding
    // calls to network namespace of the pod to query the IP of the veth interface on every
    // SandboxStatus request.
    if err := c.setupPodNetwork(ctx, &sandbox); err != nil {
      return nil, fmt.Errorf("failed to setup network for sandbox %q: %w", id, err)
    }
    sandboxCreateNetworkTimer.UpdateSince(netStart)
  }
  if err := sandboxInfo.AddExtension(podsandbox.MetadataKey, &sandbox.Metadata); err != nil {
    return nil, fmt.Errorf("unable to save sandbox %q to store: %w", id, err)
  }
  controller, err := c.getSandboxController(config, r.GetRuntimeHandler())
  if err != nil {
    return nil, fmt.Errorf("failed to get sandbox controller: %w", err)
  }
  // Save sandbox metadata to store
  if sandboxInfo, err = c.client.SandboxStore().Update(ctx, sandboxInfo, "extensions"); err != nil {
    return nil, fmt.Errorf("unable to update extensions for sandbox %q: %w", id, err)
  }
  runtimeStart := time.Now()
  if err := controller.Create(ctx, id); err != nil {
    return nil, fmt.Errorf("failed to create sandbox %q: %w", id, err)
  }
  resp, err := controller.Start(ctx, id)
  if err != nil {
    sandbox.Container, _ = c.client.LoadContainer(ctx, id)
    if resp != nil && resp.SandboxID == "" && resp.Pid == 0 && resp.CreatedAt == nil && len(resp.Labels) == 0 {
      // if resp is a non-nil zero-value, an error occurred during cleanup
      cleanupErr = fmt.Errorf("failed to cleanup sandbox")
    }
    return nil, fmt.Errorf("failed to start sandbox %q: %w", id, err)
  }
  // TODO: get rid of this. sandbox object should no longer have Container field.
  if ociRuntime.SandboxMode == string(criconfig.ModePodSandbox) {
    container, err := c.client.LoadContainer(ctx, id)
    if err != nil {
      return nil, fmt.Errorf("failed to load container %q for sandbox: %w", id, err)
    }
    sandbox.Container = container
  }
  labels := resp.GetLabels()
  if labels == nil {
    labels = map[string]string{}
  }
  sandbox.ProcessLabel = labels["selinux_label"]
  if err := sandbox.Status.Update(func(status sandboxstore.Status) (sandboxstore.Status, error) {
    // Set the pod sandbox as ready after successfully start sandbox container.
    status.Pid = resp.Pid
    status.State = sandboxstore.StateReady
    status.CreatedAt = protobuf.FromTimestamp(resp.CreatedAt)
    return status, nil
  }); err != nil {
    return nil, fmt.Errorf("failed to update sandbox status: %w", err)
  }
  // Add sandbox into sandbox store in INIT state.
  if err := c.sandboxStore.Add(sandbox); err != nil {
    return nil, fmt.Errorf("failed to add sandbox %+v into store: %w", sandbox, err)
  }
  // Send CONTAINER_CREATED event with both ContainerId and SandboxId equal to SandboxId.
  // Note that this has to be done after sandboxStore.Add() because we need to get
  // SandboxStatus from the store and include it in the event.
  c.generateAndSendContainerEvent(ctx, id, id, runtime.ContainerEventType_CONTAINER_CREATED_EVENT)
  // start the monitor after adding sandbox into the store, this ensures
  // that sandbox is in the store, when event monitor receives the TaskExit event.
  //
  // TaskOOM from containerd may come before sandbox is added to store,
  // but we don't care about sandbox TaskOOM right now, so it is fine.
  go func() {
    resp, err := controller.Wait(ctrdutil.NamespacedContext(), id)
    if err != nil {
      log.G(ctx).WithError(err).Error("failed to wait for sandbox controller, skipping exit event")
      return
    }
    e := &eventtypes.TaskExit{
      ContainerID: id,
      ID:          id,
      // Pid is not used
      Pid:        0,
      ExitStatus: resp.ExitStatus,
      ExitedAt:   resp.ExitedAt,
    }
    c.eventMonitor.backOff.enBackOff(id, e)
  }()
  // Send CONTAINER_STARTED event with ContainerId equal to SandboxId.
  c.generateAndSendContainerEvent(ctx, id, id, runtime.ContainerEventType_CONTAINER_STARTED_EVENT)
  sandboxRuntimeCreateTimer.WithValues(labels["oci_runtime_type"]).UpdateSince(runtimeStart)
  return &runtime.RunPodSandboxResponse{PodSandboxId: id}, nil
}

CreateContainer

func (c *criService) CreateContainer(ctx context.Context, r
*runtime.CreateContainerRequest) (_ *runtime.CreateContainerResponse, retErr error) { 
    sandbox, err := c.sandboxStore.Get(r.GetPodSandboxId())
    s, err := sandbox.Container.Task(ctx, nil) 
    sandboxPid := s.Pid()
    image, err := c.localResolve(config.GetImage().GetImage()) 
    if err != nil {
        return nil, errors.Wrapf(err, "failed to resolve image %q", config.GetImage().GetImage())
    }
    containerdImage, err := c.toContainerdImage(ctx, image)
    // Run container using the same runtime with sandbox.
    sandboxInfo, err := sandbox.Container.Info(ctx) 
    if err != nil {
        return nil, errors.Wrapf(err, "failed to get sandbox %q info", sandboxID)
    }
// Create container root directory. containerRootDir := c.getContainerRootDir(id)
    if err = c.os.MkdirAll(containerRootDir, 0755); err != nil {
        return nil, errors.Wrapf(err, "failed to create container root directory %q", containerRootDir)
    }
    ociRuntime, err := c.getSandboxRuntime(sandboxConfig, sandbox.Metadata.RuntimeHandler) 
    if err != nil {
        return nil, errors.Wrap(err, "failed to get sandbox runtime")
    }
    spec, err := c.containerSpec(id, sandboxID, sandboxPid, sandbox.NetNSPath, containerName, containerdImage.Name(), config, sandboxConfig,
&image.ImageSpec.Config, append(mounts, volumeMounts...), ociRuntime) 
    if err != nil {
        return nil, errors.Wrapf(err, "failed to generate container %q spec", id)
    }
    opts = append(opts, containerd.WithSpec(spec, specOpts...),
containerd.WithRuntime(sandboxInfo.Runtime.Name, runtimeOptions), containerd.WithContainerLabels(containerLabels), containerd.WithContainerExtension(containerMetadataExtension, &meta))
    var cntr containerd.Container
     if cntr, err = c.client.NewContainer(ctx, id, opts...); err != nil { 
         return nil, errors.Wrap(err, "failed to create containerd container")
    }
// Add container into container store.
    if err := c.containerStore.Add(container); err != nil {
      return nil, errors.Wrapf(err, "failed to add container %q into store", id)
    }
}

StartContainer

func (c *criService) StartContainer(ctx context.Context, r *runtime.StartContainerRequest) (retRes *runtime.StartContainerResponse, retErr error) {
    cntr, err := c.containerStore.Get(r.GetContainerId())
    // Get sandbox config from sandbox store. sandbox, err := c.sandboxStore.Get(meta.SandboxID) ctrInfo, err := container.Info(ctx)
    if err != nil {
        return nil, errors.Wrap(err, "failed to get container info")
    }
    taskOpts := c.taskOpts(ctrInfo.Runtime.Name)
    task, err := container.NewTask(ctx, ioCreation, taskOpts...) 
    if err != nil {
        return nil, errors.Wrap(err, "failed to create containerd task")
    }
    // wait is a long running background request, no timeout needed. exitCh, err := task.Wait(ctrdutil.NamespacedContext())
    // Start containerd task.
    if err := task.Start(ctx); err != nil {
        return nil, errors.Wrapf(err, "failed to start containerd task %q", id)
    }
}

func (c *container) NewTask(ctx context.Context, ioCreate cio.Creator, opts
...NewTaskOpts) (_ Task, err error) {
    ......
    request := &tasks.CreateTaskRequest{ 
                  ContainerID: c.id,
                  Terminal:  cfg.Terminal, Stdin:  cfg.Stdin,
                  Stdout:  cfg.Stdout,
                  Stderr:  cfg.Stderr,
                }
     ......
     response, err := c.client.TaskService().Create(ctx, request)
     ......

func (t *task) Start(ctx context.Context) error {
  r, err := t.client.TaskService().Start(ctx, &tasks.StartRequest{
    ContainerID: t.id,
  })
  if err != nil {
    if t.io != nil {
      t.io.Cancel()
      t.io.Close()
    }
    return errdefs.FromGRPC(err)
  }
  t.pid = r.Pid
  return nil
}

1.4 “Task Service”

Task Service创建task流程

func (l *local) Create(ctx context.Context, r *api.CreateTaskRequest, _
...grpc.CallOption) (*api.CreateTaskResponse, error) { container, err := l.getContainer(ctx, r.ContainerID)
......
    rtime, err := l.getRuntime(container.Runtime.Name) 
    if err != nil {
      return nil, err
    }
    _, err = rtime.Get(ctx, r.ContainerID)
    if err != nil && err != runtime.ErrTaskNotExists { 
        return nil, errdefs.ToGRPC(err)
    }
    if err == nil {
        return nil, errdefs.ToGRPC(fmt.Errorf("task %s already exists", r.ContainerID))
    }
    c, err := rtime.Create(ctx, r.ContainerID, opts)
    ......
    return &api.CreateTaskResponse{ 
        ContainerID: r.ContainerID, Pid: c.PID(),
    }, nil

func (m *TaskManager) Create(ctx context.Context, id string, opts runtime.CreateOpts) (_ runtime.Task, retErr error) {
    ......
    shim, err := m.startShim(ctx, bundle, id, opts)
    t, err := shim.Create(ctx, opts)
    .....
}

func (m *TaskManager) startShim(ctx context.Context, bundle *Bundle, id string, opts runtime.CreateOpts) (*shim, error) {
    ......
    b := shimBinary(ctx, bundle, opts.Runtime, m.containerdAddress, m.containerdTTRPCAddress, m.events, m.tasks)
    shim, err := b.Start(ctx, topts, func() { 
        log.G(ctx).WithField("id", id).Info("shim disconnected")
        cleanupAfterDeadShim(context.Background(), id, ns, m.tasks, m.events, b) m.tasks.Delete(ctx, id)
    })
    ......

containerd-shim-runc-v2 -namespace xxxx -address xxxx -publish-binary xxxx -id xxxx start

func (b *binary) Start(ctx context.Context, opts *types.Any, onClose func()) (_ *shim, err error) {
    args := []string{"­id", b.bundle.ID}
    args = append(args, "start")
    cmd, err := 
    client.Command(ctx,
                       b.runtime, 
                       b.containerdAddress, 
                       b.containerdTTRPCAddress, 
                       b.bundle.Path,
                       opts, 
                       args...,
                      )
    ......
    out, err := cmd.CombinedOutput() 
    if err != nil {
        return nil, errors.Wrapf(err, "%s", out)
    }
    address := strings.TrimSpace(string(out))
    conn, err := client.Connect(address, client.AnonDialer) 
    if err != nil {
        return nil, err
    }
    onCloseWithShimLog := func() { 
        onClose()
        cancelShimLog() 
        f.Close()
    }
    client := ttrpc.NewClient(conn, ttrpc.WithOnClose(onCloseWithShimLog))
    return &shim{
        bundle: b.bundle,
        client: client,
    }, nil

Task Service启动task流程

func (l *local) Start(ctx context.Context, r *api.StartRequest, _ ...grpc.CallOption) (*api.StartResponse, error) {
  t, err := l.getTask(ctx, r.ContainerID)
  if err != nil {
    return nil, err
  }
  p := runtime.Process(t)
  if r.ExecID != "" {
    if p, err = t.Process(ctx, r.ExecID); err != nil {
      return nil, errdefs.ToGRPC(err)
    }
  }
  if err := p.Start(ctx); err != nil {
    return nil, errdefs.ToGRPC(err)
  }
  state, err := p.State(ctx)
  if err != nil {
    return nil, errdefs.ToGRPC(err)
  }
  return &api.StartResponse{
    Pid: state.Pid,
  }, nil
}

func (s *shim) Start(ctx context.Context) error { 
    response, err := s.task.Start(ctx, 
      &task.StartRequest{
          ID: s.ID(),
      })
    if err != nil {
        return errdefs.FromGRPC(err)
      }
    s.taskPid = int(response.Pid) 
    return nil

1.5 “Containerd-shim启动流程”

RunManager(ctx context.Context, manager Manager, opts ...BinaryOpts)

case "start":
    opts := StartOpts{
      Address:      addressFlag,
      TTRPCAddress: ttrpcAddress,
      Debug:        debugFlag,
    }
    address, err := manager.Start(ctx, id, opts)
    if err != nil {
      return err
    }
    if _, err := os.Stdout.WriteString(address); err != nil {
      return err
    }
    return nil
  }

func (manager) Start(ctx context.Context, id string, opts shim.StartOpts) (_ string, retErr error) {
    cmd, err := newCommand(ctx, id, opts.Address, opts.TTRPCAddress, opts.Debug)
    ...
    // make sure that reexec shim-v2 binary use the value if need
    if err := shim.WriteAddress("address", address); err != nil {
        return "", err
    }
    ...
    if err := cmd.Start(); err != nil {
        f.Close()
        return "", err
    }
    ...
    // make sure to wait after start
    go cmd.Wait()
    ...
    server, err := newServer(ttrpc.WithUnaryServerInterceptor(unaryInterceptor))
    if err != nil {
        return fmt.Errorf("failed creating server: %w", err)
    }
    for _, srv := range ttrpcServices {
        if err := srv.RegisterTTRPC(server); err != nil {
            return fmt.Errorf("failed to register service: %w", err)
        }
    }
    if err := serve(ctx, server, signals, sd.Shutdown); err != nil {
        if err != shutdown.ErrShutdown {
            return err
        }
    }
}

type TaskService interface {
  State(context.Context, *StateRequest) (*StateResponse, error)
  Create(context.Context, *CreateTaskRequest) (*CreateTaskResponse, error)
  Start(context.Context, *StartRequest) (*StartResponse, error)
  Delete(context.Context, *DeleteRequest) (*DeleteResponse, error)
  Pids(context.Context, *PidsRequest) (*PidsResponse, error)
  Pause(context.Context, *PauseRequest) (*emptypb.Empty, error)
  Resume(context.Context, *ResumeRequest) (*emptypb.Empty, error)
  Checkpoint(context.Context, *CheckpointTaskRequest) (*emptypb.Empty, error)
  Kill(context.Context, *KillRequest) (*emptypb.Empty, error)
  Exec(context.Context, *ExecProcessRequest) (*emptypb.Empty, error)
  ResizePty(context.Context, *ResizePtyRequest) (*emptypb.Empty, error)
  CloseIO(context.Context, *CloseIORequest) (*emptypb.Empty, error)
  Update(context.Context, *UpdateTaskRequest) (*emptypb.Empty, error)
  Wait(context.Context, *WaitRequest) (*WaitResponse, error)
  Stats(context.Context, *StatsRequest) (*StatsResponse, error)
  Connect(context.Context, *ConnectRequest) (*ConnectResponse, error)
  Shutdown(context.Context, *ShutdownRequest) (*emptypb.Empty, error)
}

func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOpts) error
{
    args := []string{"create", "­­bundle", bundle}
    ......
    cmd := r.command(context, append(args, id)...)
    ......
    ec, err := Monitor.Start(cmd)
    ......
}

func (r *Runc) Start(context context.Context, id string) error { 
    return r.runOrError(r.command(context, "start", id))
}

2 Containerd深入浅出-安全容器篇

2.1 技术简介

2.1.1 K0s

2.1.2 Containerd

注：TTRPC 是一种基于 gRPC 的裁剪版通信协议。

2.1.3 Containerd-shim

注：简单来说引入shim是允许低级运行时（如runc、youki等）在创建和运行容器之后退出, 并将shim作为容器的父进程, 而不是containerd作为父进程,是否还记得Containerd 抽象uds漏洞？

2.2 创建集群

$ k0sctl init > k0sctl.yaml

apiVersion: k0sctl.k0sproject.io/v1beta1
kind: Cluster
metadata:
name: k0s-cluster
spec:
hosts:
- ssh:
address: 10.0.0.1
user: root
port: 22
keyPath: /Users/luc/.ssh/id_rsa
role: server
- ssh:
address: 10.0.0.2
user: root
port: 22
keyPath: /Users/luc/.ssh/id_rsa
role: worker
k0s:
version: 0.10.0

• master node: 163.172.190.5
• worker node: 163.172.190.5

$ k0sctl apply

$ k0sctl kubeconfig -c k0sctl.yaml > kubeconfig
$ export KUBECONFIG=$PWD/kubeconfig

$ kubectl get no
NAME     STATUS   ROLES    AGE   VERSION
worker   Ready    <none>   5m  v1.20.2-k0s1

注意：创建k0s的主要原因之一是控制平面隔离。

• Docker
• CRI-O

注意：下图实际docker(即moby)默认使用containerd容器运行时。

2.3 容器运行时

2.3.1 默认运行时

$ cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
name: www-runc
spec:
containers:
- image: nginx:1.18
name: www
ports:
- containerPort: 80
EOF

$ kubectl get po -A | awk '{print $1" "$2}'
default www-runc
kube-system calico-kube-controllers-5f6546844f-tw8t6
kube-system calico-node-4p98g
kube-system coredns-5c98d7d4d8-9z425
kube-system konnectivity-agent-m4gst
kube-system kube-proxy-jq6qd
kube-system metrics-server-6fbcd86f7b-m8cnd

2.3.2 GVisor

set -e
URL=https://storage.googleapis.com/gvisor/releases/release/latest
wget ${URL}/runsc ${URL}/runsc.sha512 \
${URL}/gvisor-containerd-shim ${URL}/gvisor-containerd-shim.sha512 \
${URL}/containerd-shim-runsc-v1 ${URL}/containerd-shim-runsc-v1.sha512
sha512sum -c runsc.sha512 \
-c gvisor-containerd-shim.sha512 \
-c containerd-shim-runsc-v1.sha512
rm -f *.sha512
chmod a+rx runsc gvisor-containerd-shim containerd-shim-runsc-v1
sudo mv runsc gvisor-containerd-shim containerd-shim-runsc-v1 /usr/local/bin

$ cat<<EOF | sudo tee /etc/k0s/containerd.toml
disabled_plugins = ["restart"]
[plugins.linux]
shim_debug = true
[plugins.cri.containerd.runtimes.runsc]
runtime_type = "io.containerd.runsc.v1"
EOF

$ kill -s SIGHUP CONTAINER_PID

$ cat<<EOF | kubectl apply -f -
apiVersion: node.k8s.io/v1beta1
kind: RuntimeClass
metadata:
name: gvisor
handler: runsc
EOF

$ cat<<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
labels:
app: untrusted
name: www-gvisor
spec:
runtimeClassName: gvisor
containers:
- image: nginx:1.18
name: www
ports:
- containerPort: 80
EOF

$ kubectl get po -l app=untrusted
NAME         READY   STATUS    RESTARTS   AGE
www-gvisor   1/1     Running   0          9s

2.3.3 Kata Containers

$ cat /sys/module/kvm_amd/parameters/nested
1

$ bash -c "$(curl -fsSL https://raw.githubusercontent.com/kata-containers/tests/master/cmd/kata-manager/kata-manager.sh) install-packages"

$ cat<<EOF | sudo tee -a /etc/k0s/containerd.toml
[plugins.cri.containerd.runtimes.kata]
runtime_type = "io.containerd.kata.v2"
EOF

$ kill -s SIGHUP CONTAINER_PID

注意：如果是大规模集群，建议使用批量工具，如ansible，或者使用高级节点管理工具等

$ cat<<EOF | kubectl apply -f -
kind: RuntimeClass
apiVersion: node.k8s.io/v1beta1
metadata:
name: kata
handler: kata
EOF

$ cat<<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
labels:
app: untrusted
name: www-kata
spec:
runtimeClassName: kata
containers:
- image: nginx:1.18
name: www
ports:
- containerPort: 80
EOF

$ kubectl get po -l app=untrusted
NAME         READY   STATUS    RESTARTS   AGE
www-gvisor   1/1     Running   0          8m1s
www-kata     1/1     Running   0          3h25m

root@worker:~# ps -ef | grep qemu | awk '{print $11}'
/usr/bin/qemu-vanilla-system-x86_64

参考文献

https://betterprogramming.pub/kata-container-and-gvisor-with-k0s-82efbbcc240b

3 Containerd深度剖析-runtime篇

3.1 需求简介

注: Container runtime统称为容器运行时

• 下载镜像
• 将镜像解压成一个bundle，即将各层文件平铺到一个单一的文件系统中。
• 运行容器

3.2 低级容器运行时

• 只认识 rootfs 和 config.json，没有其他镜像能力
• 不提供网络实现
• 不提供持久实现
• 无法跨平台等

低级运行时demo

$ CID=$(docker create busybox)
$ ROOTFS=$(mktemp -d)
$ docker export $CID | tar -xf - -C $ROOTFS

$ UUID=$(uuidgen)
$ cgcreate -g cpu,memory:$UUID
$ cgset -r memory.limit_in_bytes=100000000 $UUID
$ cgset -r cpu.shares=512 $UUID

$ cgset -r cpu.cfs_period_us=1000000 $UUID
$ cgset -r cpu.cfs_quota_us=2000000 $UUID

$ cgexec -g cpu,memory:$UUID \
>     unshare -uinpUrf --mount-proc \
>     sh -c "/bin/hostname $UUID && chroot $ROOTFS /bin/sh"
/ # echo "Hello from in a container"
Hello from in a container
/ # exit

$ cgdelete -r -g cpu,memory:$UUID
$ rm -r $ROOTFS

低级运行时demo

runC

$ mkdir rootfs
$ docker export $(docker create busybox) | tar -xf - -C rootfs

$ runc spec

$ cat config.json
{
        "ociVersion": "1.0.2",
        "process": {
                "terminal": true,
                "user": {
                        "uid": 0,
                        "gid": 0
                },
                "args": [
                        "sh"
                ],
                "env": [
                        "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
                        "TERM=xterm"
                ],
                "cwd": "/",
                "capabilities": {
...

$ sudo runc run mycontainerid
/ # echo "Hello from in a container"
Hello from in a container

rkt(已废弃)

runV

youki

3.3 高级容器运行时

3.3.1 高级运行时代表

Docker

Containerd

$ sudo ctr images pull docker.io/library/redis:latest

$ sudo ctr images list

$ sudo ctr container create docker.io/library/redis:latest redis

$ sudo ctr container list

$ sudo ctr container delete redis

rkt(已废弃)

3.3.2 Kubernetes CRI

CRI在Kubernetes 1.5中引入，作为kubelet和容器运行时之间的桥梁。社区希望Kubernetes集成的高级容器运行时实现CRI。该运行时处理镜像的管理，支持Kubernetes pods，并管理容器，因此根据高级运行时的定义，支持CRI的运行时必须是一个高级运行时。低级别的运行时并不具备上述功能。

3.3.3 CRI规范

ImageService.PullImage({image: "image1"})
ImageService.PullImage({image: "image2"})
podID = RuntimeService.RunPodSandbox({name: "mypod"})
id1 = RuntimeService.CreateContainer({
    pod: podID,
    name: "container1",
    image: "image1",
})
id2 = RuntimeService.CreateContainer({
    pod: podID,
    name: "container2",
    image: "image2",
})
RuntimeService.StartContainer({id: id1})
RuntimeService.StartContainer({id: id2})

cat <<EOF | sudo tee /etc/crictl.yaml
runtime-endpoint: unix:///run/containerd/containerd.sock
EOF

crictl --runtime-endpoint unix:///run/containerd/containerd.sock …

3.3.4 支持CRI的运行时

Containerd

Docker

CRI-O

参考文献

1. https://blog.mobyproject.org/where-are-containerds-graph-drivers-145fc9b7255
2. https://insujang.github.io/2019-10-31/container-runtime/
3. https://github.com/cri-o/cri-o