运行时文件/var/lib/docker

├── aufs/
│ ├── diff/
│ │ ├── 3e5dad646755b80ceeb2776db26d93cd8b2a45b66c670173eb0a5def51b27a26/
│ │ ├── 890f6393af02ab70fac8d9de2e9d37b92fc0a461d72bf3a17b73e1f809e1b01a/
│ │ ├── 9f2849b92cad358be0cfc0cbada718203184791d8b9dc4d02fb966ab7ef7f703/
│ │ ├── 9f2849b92cad358be0cfc0cbada718203184791d8b9dc4d02fb966ab7ef7f703-init/
│ │ ├── b2fe7fefbb164b8a8adb84273b94eeccc6794c231fc3958bc98a35cc72225cbb/
│ │ ├── d0ff844478a02c0f0544115e77626cf4f0c4a7996a6a6764eea2a2dd29201910/
│ │ └── d0ff844478a02c0f0544115e77626cf4f0c4a7996a6a6764eea2a2dd29201910-init/
│ ├── layers/
│ │ ├── 3e5dad646755b80ceeb2776db26d93cd8b2a45b66c670173eb0a5def51b27a26
│ │ ├── 890f6393af02ab70fac8d9de2e9d37b92fc0a461d72bf3a17b73e1f809e1b01a
│ │ ├── 9f2849b92cad358be0cfc0cbada718203184791d8b9dc4d02fb966ab7ef7f703
│ │ ├── 9f2849b92cad358be0cfc0cbada718203184791d8b9dc4d02fb966ab7ef7f703-init
│ │ ├── b2fe7fefbb164b8a8adb84273b94eeccc6794c231fc3958bc98a35cc72225cbb
│ │ ├── d0ff844478a02c0f0544115e77626cf4f0c4a7996a6a6764eea2a2dd29201910
│ │ └── d0ff844478a02c0f0544115e77626cf4f0c4a7996a6a6764eea2a2dd29201910-init
│ └── mnt/
│ ├── 3e5dad646755b80ceeb2776db26d93cd8b2a45b66c670173eb0a5def51b27a26/
│ ├── 890f6393af02ab70fac8d9de2e9d37b92fc0a461d72bf3a17b73e1f809e1b01a/
│ ├── 9f2849b92cad358be0cfc0cbada718203184791d8b9dc4d02fb966ab7ef7f703/
│ ├── 9f2849b92cad358be0cfc0cbada718203184791d8b9dc4d02fb966ab7ef7f703-init/
│ ├── b2fe7fefbb164b8a8adb84273b94eeccc6794c231fc3958bc98a35cc72225cbb/
│ ├── d0ff844478a02c0f0544115e77626cf4f0c4a7996a6a6764eea2a2dd29201910/
│ └── d0ff844478a02c0f0544115e77626cf4f0c4a7996a6a6764eea2a2dd29201910-init/
├── builder/
│ └── fscache.db
├── buildkit/
│ ├── content/
│ │ └── ingest/
│ ├── executor/
│ ├── cache.db
│ ├── metadata.db
│ └── snapshots.db
├── containerd/
│ └── daemon/
│ ├── io.containerd.content.v1.content/
│ ├── io.containerd.metadata.v1.bolt/
│ ├── io.containerd.runtime.v1.linux/
│ ├── io.containerd.runtime.v2.task/
│ ├── io.containerd.snapshotter.v1.btrfs/
│ ├── io.containerd.snapshotter.v1.native/
│ ├── io.containerd.snapshotter.v1.overlayfs/
│ └── tmpmounts/
├── containers/
│ ├── a8205034ae074d1fbc73327fa348ffaac3a1568d6a3de1953aeb49ea99dcfdb0/
│ │ ├── checkpoints/
│ │ ├── mounts/
│ │ ├── a8205034ae074d1fbc73327fa348ffaac3a1568d6a3de1953aeb49ea99dcfdb0-json.log
│ │ ├── config.v2.json
│ │ ├── hostconfig.json
│ │ ├── hostname
│ │ ├── hosts
│ │ ├── resolv.conf
│ │ └── resolv.conf.hash
│ └── bf1b6932346bfd4793f4ebea51ca7a0ce5d460620b07bea3b3a6e1beb9e502b8/
│ ├── checkpoints/
│ ├── mounts/
│ ├── bf1b6932346bfd4793f4ebea51ca7a0ce5d460620b07bea3b3a6e1beb9e502b8-json.log
│ ├── config.v2.json
│ ├── hostconfig.json
│ ├── hostname
│ ├── hosts
│ ├── resolv.conf
│ └── resolv.conf.hash
├── image/
│ ├── aufs/
│ │ ├── distribution/
│ │ ├── imagedb/
│ │ ├── layerdb/
│ │ └── repositories.json
│ └── overlay/
│ ├── distribution/
│ ├── imagedb/
│ ├── layerdb/
│ └── repositories.json
├── network/
│ └── files/
│ └── local-kv.db
├── overlay/
├── plugins/
│ ├── storage/
│ │ └── blobs/
│ └── tmp/
├── runtimes/
├── swarm/
├── tmp/
├── trust/
└── volumes/
└── metadata.db

api/server/router/container/container_routes.go#containerRouter.postContainersStart

  1. func (s *containerRouter) postContainersStart(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
  2.     // If contentLength is -1, we can assumed chunked encoding
  3.     // or more technically that the length is unknown
  4.     // https://golang.org/src/pkg/net/http/request.go#L139
  5.     // net/http otherwise seems to swallow any headers related to chunked encoding
  6.     // including r.TransferEncoding
  7.     // allow a nil body for backwards compatibility
  9.     version := httputils.VersionFromContext(ctx)
  10.     var hostConfig *container.HostConfig
  11.     // A non-nil json object is at least 7 characters.
  12.     if r.ContentLength > 7 || r.ContentLength == -1 {
  13.         if versions.GreaterThanOrEqualTo(version, "1.24") {
  14.             return bodyOnStartError{}
  15.         }
  17.         if err := httputils.CheckForJSON(r); err != nil {
  18.             return err
  19.         }
  21.         c, err := s.decoder.DecodeHostConfig(r.Body)
  22.         if err != nil {
  23.             return err
  24.         }
  25.         hostConfig = c
  26.     }
  28.     if err := httputils.ParseForm(r); err != nil {
  29.         return err
  30.     }
  32.     checkpoint := r.Form.Get("checkpoint")
  33.     checkpointDir := r.Form.Get("checkpoint-dir")
  34.   // ********************************** NOTICE ********************************** //
  35.     if err := s.backend.ContainerStart(vars["name"], hostConfig, checkpoint, checkpointDir); err != nil {
  36.     // ********************************** NOTICE ********************************** //
  37.         return err
  38.     }
  40.     w.WriteHeader(http.StatusNoContent)
  41.     return nil
  42. }

1) daemon/start.go#Daemon.ContainerStart

  1. // ContainerStart starts a container.
  2. func (daemon *Daemon) ContainerStart(name string, hostConfig *containertypes.HostConfig, checkpoint string, checkpointDir string) error {
  3.     if checkpoint != "" && !daemon.HasExperimental() {
  4.         return errdefs.InvalidParameter(errors.New("checkpoint is only supported in experimental mode"))
  5.     }
  6.     // ********************************** NOTICE ********************************** //
  7.     container, err := daemon.GetContainer(name)
  8.   // ********************************** NOTICE ********************************** //
  9.     if err != nil {
  10.         return err
  11.     }
  13.     validateState := func() error {
  14.         container.Lock()
  15.         defer container.Unlock()
  17.         if container.Paused {
  18.             return errdefs.Conflict(errors.New("cannot start a paused container, try unpause instead"))
  19.         }
  21.         if container.Running {
  22.             return containerNotModifiedError{running: true}
  23.         }
  25.         if container.RemovalInProgress || container.Dead {
  26.             return errdefs.Conflict(errors.New("container is marked for removal and cannot be started"))
  27.         }
  28.         return nil
  29.     }
  31.     if err := validateState(); err != nil {
  32.         return err
  33.     }
  35.     // Windows does not have the backwards compatibility issue here.
  36.     if runtime.GOOS != "windows" {
  37.         // This is kept for backward compatibility - hostconfig should be passed when
  38.         // creating a container, not during start.
  39.         if hostConfig != nil {
  40.             logrus.Warn("DEPRECATED: Setting host configuration options when the container starts is deprecated and has been removed in Docker 1.12")
  41.             oldNetworkMode := container.HostConfig.NetworkMode
  42.             if err := daemon.setSecurityOptions(container, hostConfig); err != nil {
  43.                 return errdefs.InvalidParameter(err)
  44.             }
  45.             if err := daemon.mergeAndVerifyLogConfig(&hostConfig.LogConfig); err != nil {
  46.                 return errdefs.InvalidParameter(err)
  47.             }
  48.             if err := daemon.setHostConfig(container, hostConfig); err != nil {
  49.                 return errdefs.InvalidParameter(err)
  50.             }
  51.             newNetworkMode := container.HostConfig.NetworkMode
  52.             if string(oldNetworkMode) != string(newNetworkMode) {
  53.                 // if user has change the network mode on starting, clean up the
  54.                 // old networks. It is a deprecated feature and has been removed in Docker 1.12
  55.                 container.NetworkSettings.Networks = nil
  56.                 if err := container.CheckpointTo(daemon.containersReplica); err != nil {
  57.                     return errdefs.System(err)
  58.                 }
  59.             }
  60.             container.InitDNSHostConfig()
  61.         }
  62.     } else {
  63.         if hostConfig != nil {
  64.             return errdefs.InvalidParameter(errors.New("Supplying a hostconfig on start is not supported. It should be supplied on create"))
  65.         }
  66.     }
  68.     // check if hostConfig is in line with the current system settings.
  69.     // It may happen cgroups are umounted or the like.
  70.     if _, err = daemon.verifyContainerSettings(container.OS, container.HostConfig, nil, false); err != nil {
  71.         return errdefs.InvalidParameter(err)
  72.     }
  73.     // Adapt for old containers in case we have updates in this function and
  74.     // old containers never have chance to call the new function in create stage.
  75.     if hostConfig != nil {
  76.         if err := daemon.adaptContainerSettings(container.HostConfig, false); err != nil {
  77.             return errdefs.InvalidParameter(err)
  78.         }
  79.     }
  80.   // ********************************** NOTICE ********************************** //
  81.     return daemon.containerStart(container, checkpoint, checkpointDir, true)
  82.   // ********************************** NOTICE ********************************** //
  83. }

1.1) daemon/container.go#Daemon.GetContainer

  1. func (daemon *Daemon) GetContainer(prefixOrName string) (*container.Container, error) {
  2.     if len(prefixOrName) == 0 {
  3.         return nil, errors.WithStack(invalidIdentifier(prefixOrName))
  4.     }
  6.     if containerByID := daemon.containers.Get(prefixOrName); containerByID != nil {
  7.         // prefix is an exact match to a full container ID
  8.         return containerByID, nil
  9.     }
  11.     // GetByName will match only an exact name provided; we ignore errors
  12.     if containerByName, _ := daemon.GetByName(prefixOrName); containerByName != nil {
  13.         // prefix is an exact match to a full container Name
  14.         return containerByName, nil
  15.     }
  17.     containerID, indexError := daemon.idIndex.Get(prefixOrName)
  18.     if indexError != nil {
  19.         // When truncindex defines an error type, use that instead
  20.         if indexError == truncindex.ErrNotExist {
  21.             return nil, containerNotFound(prefixOrName)
  22.         }
  23.         return nil, errdefs.System(indexError)
  24.     }
  25.   // ********************************** NOTICE ********************************** //
  26.     return daemon.containers.Get(containerID), nil
  27.   // ********************************** NOTICE ********************************** //
  28. }

1.2) daemon/start.go#Daemon.containerStart

  1. func (daemon *Daemon) containerStart(container *container.Container, checkpoint string, checkpointDir string, resetRestartManager bool) (err error) {
  2.     start := time.Now()
  3.     container.Lock()
  4.     defer container.Unlock()
  6.     if resetRestartManager && container.Running { // skip this check if already in restarting step and resetRestartManager==false
  7.         return nil
  8.     }
  10.     if container.RemovalInProgress || container.Dead {
  11.         return errdefs.Conflict(errors.New("container is marked for removal and cannot be started"))
  12.     }
  14.     if checkpointDir != "" {
  15.         // TODO(mlaventure): how would we support that?
  16.         return errdefs.Forbidden(errors.New("custom checkpointdir is not supported"))
  17.     }
  19.     // if we encounter an error during start we need to ensure that any other
  20.     // setup has been cleaned up properly
  21.     defer func() {
  22.         if err != nil {
  23.             container.SetError(err)
  24.             // if no one else has set it, make sure we don't leave it at zero
  25.             if container.ExitCode() == 0 {
  26.                 container.SetExitCode(128)
  27.             }
  28.             if err := container.CheckpointTo(daemon.containersReplica); err != nil {
  29.                 logrus.Errorf("%s: failed saving state on start failure: %v", container.ID, err)
  30.             }
  31.             container.Reset(false)
  33.             daemon.Cleanup(container)
  34.             // if containers AutoRemove flag is set, remove it after clean up
  35.             if container.HostConfig.AutoRemove {
  36.                 container.Unlock()
  37.                 if err := daemon.ContainerRm(container.ID, &types.ContainerRmConfig{ForceRemove: true, RemoveVolume: true}); err != nil {
  38.                     logrus.Errorf("can't remove container %s: %v", container.ID, err)
  39.                 }
  40.                 container.Lock()
  41.             }
  42.         }
  43.     }()
  45.     if err := daemon.conditionalMountOnStart(container); err != nil {
  46.         return err
  47.     }
  48.      // ********************************** NOTICE ********************************** //
  49.     if err := daemon.initializeNetworking(container); err != nil {
  50.     // ********************************** NOTICE ********************************** //
  51.         return err
  52.     }
  54.     // ********************************** NOTICE ********************************** //
  55.     spec, err := daemon.createSpec(container)
  56.   // ********************************** NOTICE ********************************** //
  57.     if err != nil {
  58.         return errdefs.System(err)
  59.     }
  61.     if resetRestartManager {
  62.         container.ResetRestartManager(true)
  63.         container.HasBeenManuallyStopped = false
  64.     }
  66.     if daemon.saveApparmorConfig(container); err != nil {
  67.         return err
  68.     }
  70.     if checkpoint != "" {
  71.         checkpointDir, err = getCheckpointDir(checkpointDir, checkpoint, container.Name, container.ID, container.CheckpointDir(), false)
  72.         if err != nil {
  73.             return err
  74.         }
  75.     }
  77.     createOptions, err := daemon.getLibcontainerdCreateOptions(container)
  78.     if err != nil {
  79.         return err
  80.     }
  82.   // ********************************** NOTICE ********************************** //
  83.     err = daemon.containerd.Create(context.Background(), container.ID, spec, createOptions)
  84.   // ********************************** NOTICE ********************************** //
  85.     if err != nil {
  86.         return translateContainerdStartErr(container.Path, container.SetExitCode, err)
  87.     }
  90.     // TODO(mlaventure): we need to specify checkpoint options here
  91.   // ********************************** NOTICE ********************************** //
  92.     pid, err := daemon.containerd.Start(context.Background(), container.ID, checkpointDir,
  93.         container.StreamConfig.Stdin() != nil || container.Config.Tty,
  94.         container.InitializeStdio)
  95.   // ********************************** NOTICE ********************************** //
  96.     if err != nil {
  97.         if err := daemon.containerd.Delete(context.Background(), container.ID); err != nil {
  98.             logrus.WithError(err).WithField("container", container.ID).
  99.                 Error("failed to delete failed start container")
  100.         }
  101.         return translateContainerdStartErr(container.Path, container.SetExitCode, err)
  102.     }
  104.     container.SetRunning(pid, true)
  105.     container.HasBeenStartedBefore = true
  106.     daemon.setStateCounter(container)
  108.     daemon.initHealthMonitor(container)
  110.     if err := container.CheckpointTo(daemon.containersReplica); err != nil {
  111.         logrus.WithError(err).WithField("container", container.ID).
  112.             Errorf("failed to store container")
  113.     }
  115.     daemon.LogContainerEvent(container, "start")
  116.     containerActions.WithValues("start").UpdateSince(start)
  118.     return nil
  119. }

1.2.1) daemon/container_operations.go#Daemon.initializingNetworking

  1. func (daemon *Daemon) initializeNetworking(container *container.Container) error {
  2.     var err error
  3.     // container模式
  4.     if container.HostConfig.NetworkMode.IsContainer() {
  5.         // we need to get the hosts files from the container to join
  6.         nc, err := daemon.getNetworkedContainer(container.ID, container.HostConfig.NetworkMode.ConnectedContainer())
  7.         if err != nil {
  8.             return err
  9.         }
  11.         err = daemon.initializeNetworkingPaths(container, nc)
  12.         if err != nil {
  13.             return err
  14.         }
  16.         container.Config.Hostname = nc.Config.Hostname
  17.         container.Config.Domainname = nc.Config.Domainname
  18.         return nil
  19.     }
  21.   // host模式
  22.     if container.HostConfig.NetworkMode.IsHost() {
  23.         if container.Config.Hostname == "" {
  24.             container.Config.Hostname, err = os.Hostname()
  25.             if err != nil {
  26.                 return err
  27.             }
  28.         }
  29.     }
  30.   // 其他模式，包括bridge和none
  31.     // ********************************** NOTICE ********************************** //
  32.     if err := daemon.allocateNetwork(container); err != nil {
  33.     // ********************************** NOTICE ********************************** //
  34.         return err
  35.     }
  36.     // ********************************** NOTICE ********************************** //
  37.     return container.BuildHostnameFile()
  38.   // ********************************** NOTICE ********************************** //
  39. }

1.2.1.1) daemon/container_operations.go#Daemon.allocateNetwork

  1. func (daemon *Daemon) allocateNetwork(container *container.Container) error {
  2.     start := time.Now()
  3.     controller := daemon.netController
  5.     if daemon.netController == nil {
  6.         return nil
  7.     }
  9.     // Cleanup any stale sandbox left over due to ungraceful daemon shutdown
  10.     if err := controller.SandboxDestroy(container.ID); err != nil {
  11.         logrus.Errorf("failed to cleanup up stale network sandbox for container %s", container.ID)
  12.     }
  14.     if container.Config.NetworkDisabled || container.HostConfig.NetworkMode.IsContainer() {
  15.         return nil
  16.     }
  18.     updateSettings := false
  20.     if len(container.NetworkSettings.Networks) == 0 {
  21.     // ********************************** NOTICE ********************************** //
  22.         daemon.updateContainerNetworkSettings(container, nil)
  23.     // ********************************** NOTICE ********************************** //
  24.         updateSettings = true
  25.     }
  27.   // 如果有bridge网络，那么连接即可（默认是有的）
  28.     // always connect default network first since only default
  29.     // network mode support link and we need do some setting
  30.     // on sandbox initialize for link, but the sandbox only be initialized
  31.     // on first network connecting.
  32.     defaultNetName := runconfig.DefaultDaemonNetworkMode().NetworkName()
  33.     if nConf, ok := container.NetworkSettings.Networks[defaultNetName]; ok {
  34.         cleanOperationalData(nConf)
  35.     // ********************************** NOTICE ********************************** //
  36.         if err := daemon.connectToNetwork(container, defaultNetName, nConf.EndpointSettings, updateSettings); err != nil {
  37.       // ********************************** NOTICE ********************************** //
  38.             return err
  39.         }
  40.     }
  42.   // 下面不用看了
  43.     // the intermediate map is necessary because "connectToNetwork" modifies "container.NetworkSettings.Networks"
  44.     networks := make(map[string]*network.EndpointSettings)
  45.     for n, epConf := range container.NetworkSettings.Networks {
  46.         if n == defaultNetName {
  47.             continue
  48.         }
  50.         networks[n] = epConf
  51.     }
  53.     for netName, epConf := range networks {
  54.         cleanOperationalData(epConf)
  55.         if err := daemon.connectToNetwork(container, netName, epConf.EndpointSettings, updateSettings); err != nil {
  56.             return err
  57.         }
  58.     }
  60.     // If the container is not to be connected to any network,
  61.     // create its network sandbox now if not present
  62.     if len(networks) == 0 {
  63.         if nil == daemon.getNetworkSandbox(container) {
  64.             options, err := daemon.buildSandboxOptions(container)
  65.             if err != nil {
  66.                 return err
  67.       }
  68.             sb, err := daemon.netController.NewSandbox(container.ID, options...)
  69.             if err != nil {
  70.                 return err
  71.             }
  72.             updateSandboxNetworkSettings(container, sb)
  73.             defer func() {
  74.                 if err != nil {
  75.                     sb.Delete()
  76.                 }
  77.             }()
  78.         }
  80.     }
  82.   // ********************************** NOTICE ********************************** //
  83.     if _, err := container.WriteHostConfig(); err != nil {
  84.     // ********************************** NOTICE ********************************** //
  85.         return err
  86.     }
  87.     networkActions.WithValues("allocate").UpdateSince(start)
  88.     return nil
  89. }

—1.2.1.1.1) daemon/container_operations.go#Daemon.updateContainerNetworkSettings

  1. // updateContainerNetworkSettings updates the network settings
  2. // 第二个参数为nil
  3. func (daemon *Daemon) updateContainerNetworkSettings(container *container.Container, endpointsConfig map[string]*networktypes.EndpointSettings) {
  4.     var n libnetwork.Network
  6.     mode := container.HostConfig.NetworkMode
  7.     if container.Config.NetworkDisabled || mode.IsContainer() {
  8.         return
  9.     }
  11.     networkName := mode.NetworkName()
  12.     if mode.IsDefault() {
  13.     // ********************************** NOTICE ********************************** //
  14.         networkName = daemon.netController.Config().Daemon.DefaultNetwork
  15.     // ********************************** NOTICE ********************************** //
  16.     }
  18.     if mode.IsUserDefined() {
  19.         var err error
  21.         n, err = daemon.FindNetwork(networkName)
  22.         if err == nil {
  23.             networkName = n.Name()
  24.         }
  25.     }
  27.     if container.NetworkSettings == nil {
  28.         container.NetworkSettings = &network.Settings{}
  29.     }
  31.     if len(endpointsConfig) > 0 {
  32.         if container.NetworkSettings.Networks == nil {
  33.             container.NetworkSettings.Networks = make(map[string]*network.EndpointSettings)
  34.         }
  36.         for name, epConfig := range endpointsConfig {
  37.             container.NetworkSettings.Networks[name] = &network.EndpointSettings{
  38.                 EndpointSettings: epConfig,
  39.             }
  40.         }
  41.     }
  43.     if container.NetworkSettings.Networks == nil {
  44.     // ********************************** NOTICE ********************************** //
  45.         container.NetworkSettings.Networks = make(map[string]*network.EndpointSettings)
  46.     // ********************************** NOTICE ********************************** //
  47.         container.NetworkSettings.Networks[networkName] = &network.EndpointSettings{
  48.             EndpointSettings: &networktypes.EndpointSettings{},
  49.         }
  50.     }
  52.     // Convert any settings added by client in default name to
  53.     // engine's default network name key
  54.     if mode.IsDefault() {
  55.         if nConf, ok := container.NetworkSettings.Networks[mode.NetworkName()]; ok {
  56.             container.NetworkSettings.Networks[networkName] = nConf
  57.             delete(container.NetworkSettings.Networks, mode.NetworkName())
  58.         }
  59.     }
  61.     if !mode.IsUserDefined() {
  62.         return
  63.     }
  64.     // Make sure to internally store the per network endpoint config by network name
  65.     if _, ok := container.NetworkSettings.Networks[networkName]; ok {
  66.         return
  67.     }
  69.     if n != nil {
  70.         if nwConfig, ok := container.NetworkSettings.Networks[n.ID()]; ok {
  71.             container.NetworkSettings.Networks[networkName] = nwConfig
  72.             delete(container.NetworkSettings.Networks, n.ID())
  73.             return
  74.         }
  75.     }
  76. }

—1.2.1.1.2) daemon/container_operations.go#Daemon.connectToNetwork

  1. func (daemon *Daemon) connectToNetwork(container *container.Container, idOrName string, endpointConfig *networktypes.EndpointSettings, updateSettings bool) (err error) {
  2.     start := time.Now()
  3.     if container.HostConfig.NetworkMode.IsContainer() {
  4.         return runconfig.ErrConflictSharedNetwork
  5.     }
  6.     if containertypes.NetworkMode(idOrName).IsBridge() &&
  7.         daemon.configStore.DisableBridge {
  8.         container.Config.NetworkDisabled = true
  9.         return nil
  10.     }
  11.     if endpointConfig == nil {
  12.         endpointConfig = &networktypes.EndpointSettings{}
  13.     }
  15.   // ********************************** NOTICE ********************************** //
  16.     n, config, err := daemon.findAndAttachNetwork(container, idOrName, endpointConfig)
  17.   // ********************************** NOTICE ********************************** //
  18.     if err != nil {
  19.         return err
  20.     }
  21.     if n == nil {
  22.         return nil
  23.     }
  25.     var operIPAM bool
  26.     if config != nil {
  27.         if epConfig, ok := config.EndpointsConfig[n.Name()]; ok {
  28.             if endpointConfig.IPAMConfig == nil ||
  29.                 (endpointConfig.IPAMConfig.IPv4Address == "" &&
  30.                     endpointConfig.IPAMConfig.IPv6Address == "" &&
  31.                     len(endpointConfig.IPAMConfig.LinkLocalIPs) == 0) {
  32.                 operIPAM = true
  33.             }
  35.             // copy IPAMConfig and NetworkID from epConfig via AttachNetwork
  36.             endpointConfig.IPAMConfig = epConfig.IPAMConfig
  37.             endpointConfig.NetworkID = epConfig.NetworkID
  38.         }
  39.     }
  41.     err = daemon.updateNetworkConfig(container, n, endpointConfig, updateSettings)
  42.     if err != nil {
  43.         return err
  44.     }
  46.     controller := daemon.netController
  47.   // ********************************** NOTICE ********************************** //
  48.     sb := daemon.getNetworkSandbox(container)
  49.   // ********************************** NOTICE ********************************** //
  50.     createOptions, err := buildCreateEndpointOptions(container, n, endpointConfig, sb, daemon.configStore.DNS)
  51.   // ********************************** NOTICE ********************************** //
  52.     if err != nil {
  53.         return err
  54.     }
  56.     endpointName := strings.TrimPrefix(container.Name, "/")
  57.   // ********************************** NOTICE ********************************** //
  58.     ep, err := n.CreateEndpoint(endpointName, createOptions...)
  59.   // ********************************** NOTICE ********************************** //
  60.     if err != nil {
  61.         return err
  62.     }
  63.     defer func() {
  64.         if err != nil {
  65.             if e := ep.Delete(false); e != nil {
  66.                 logrus.Warnf("Could not rollback container connection to network %s", idOrName)
  67.             }
  68.         }
  69.     }()
  70.     container.NetworkSettings.Networks[n.Name()] = &network.EndpointSettings{
  71.         EndpointSettings: endpointConfig,
  72.         IPAMOperational:  operIPAM,
  73.     }
  74.     if _, ok := container.NetworkSettings.Networks[n.ID()]; ok {
  75.         delete(container.NetworkSettings.Networks, n.ID())
  76.     }
  77.     // ********************************** NOTICE ********************************** //
  78.     if err := daemon.updateEndpointNetworkSettings(container, n, ep); err != nil {
  79.     // ********************************** NOTICE ********************************** //
  80.         return err
  81.     }
  83.     if sb == nil {
  84.     // ********************************** NOTICE ********************************** //
  85.         options, err := daemon.buildSandboxOptions(container)
  86.     // ********************************** NOTICE ********************************** //
  87.         if err != nil {
  88.             return err
  89.         }
  90.     // ********************************** NOTICE ********************************** //
  91.         sb, err = controller.NewSandbox(container.ID, options...)
  92.     // ********************************** NOTICE ********************************** //
  93.         if err != nil {
  94.             return err
  95.         }
  97.     // ********************************** NOTICE ********************************** //
  98.         updateSandboxNetworkSettings(container, sb)
  99.     // ********************************** NOTICE ********************************** //
  100.     }
  102.   // ********************************** NOTICE ********************************** //
  103.     joinOptions, err := buildJoinOptions(container.NetworkSettings, n)
  104.   // ********************************** NOTICE ********************************** //
  105.     if err != nil {
  106.         return err
  107.     }
  108.     // ********************************** NOTICE ********************************** //
  109.     if err := ep.Join(sb, joinOptions...); err != nil {
  110.     // ********************************** NOTICE ********************************** //
  111.         return err
  112.     }
  114.     if !container.Managed {
  115.         // add container name/alias to DNS
  116.         if err := daemon.ActivateContainerServiceBinding(container.Name); err != nil {
  117.             return fmt.Errorf("Activate container service binding for %s failed: %v", container.Name, err)
  118.         }
  119.     }
  121.     if err := updateJoinInfo(container.NetworkSettings, n, ep); err != nil {
  122.         return fmt.Errorf("Updating join info failed: %v", err)
  123.     }
  125.     container.NetworkSettings.Ports = getPortMapInfo(sb)
  127.     daemon.LogNetworkEventWithAttributes(n, "connect", map[string]string{"container": container.ID})
  128.     networkActions.WithValues("connect").UpdateSince(start)
  129.     return nil
  130. }

——1.2.1.1.2.1) daemon/container_operations.go#findAndAttachNetwork

  1. func (daemon *Daemon) findAndAttachNetwork(container *container.Container, idOrName string, epConfig *networktypes.EndpointSettings) (libnetwork.Network, *networktypes.NetworkingConfig, error) {
  2.     id := getNetworkID(idOrName, epConfig)
  4.     n, err := daemon.FindNetwork(id)
  5.     if err != nil {
  6.         // We should always be able to find the network for a
  7.         // managed container.
  8.         if container.Managed {
  9.             return nil, nil, err
  10.         }
  11.     }
  13.     // If we found a network and if it is not dynamically created
  14.     // we should never attempt to attach to that network here.
  15.     if n != nil {
  16.         if container.Managed || !n.Info().Dynamic() {
  17.             return n, nil, nil
  18.         }
  19.     }
  21.     var addresses []string
  22.     if epConfig != nil && epConfig.IPAMConfig != nil {
  23.         if epConfig.IPAMConfig.IPv4Address != "" {
  24.             addresses = append(addresses, epConfig.IPAMConfig.IPv4Address)
  25.         }
  27.         if epConfig.IPAMConfig.IPv6Address != "" {
  28.             addresses = append(addresses, epConfig.IPAMConfig.IPv6Address)
  29.         }
  30.     }
  32.     var (
  33.         config     *networktypes.NetworkingConfig
  34.         retryCount int
  35.     )
  37.     if n == nil && daemon.attachableNetworkLock != nil {
  38.         daemon.attachableNetworkLock.Lock(id)
  39.         defer daemon.attachableNetworkLock.Unlock(id)
  40.     }
  42.     for {
  43.         // In all other cases, attempt to attach to the network to
  44.         // trigger attachment in the swarm cluster manager.
  45.         if daemon.clusterProvider != nil {
  46.             var err error
  47.             config, err = daemon.clusterProvider.AttachNetwork(id, container.ID, addresses)
  48.             if err != nil {
  49.                 return nil, nil, err
  50.             }
  51.         }
  53.         n, err = daemon.FindNetwork(id)
  54.         if err != nil {
  55.             if daemon.clusterProvider != nil {
  56.                 if err := daemon.clusterProvider.DetachNetwork(id, container.ID); err != nil {
  57.                     logrus.Warnf("Could not rollback attachment for container %s to network %s: %v", container.ID, idOrName, err)
  58.                 }
  59.             }
  61.             // Retry network attach again if we failed to
  62.             // find the network after successful
  63.             // attachment because the only reason that
  64.             // would happen is if some other container
  65.             // attached to the swarm scope network went down
  66.             // and removed the network while we were in
  67.             // the process of attaching.
  68.             if config != nil {
  69.                 if _, ok := err.(libnetwork.ErrNoSuchNetwork); ok {
  70.                     if retryCount >= 5 {
  71.                         return nil, nil, fmt.Errorf("could not find network %s after successful attachment", idOrName)
  72.                     }
  73.                     retryCount++
  74.                     continue
  75.                 }
  76.             }
  78.             return nil, nil, err
  79.         }
  81.         break
  82.     }
  84.     // This container has attachment to a swarm scope
  85.     // network. Update the container network settings accordingly.
  86.     container.NetworkSettings.HasSwarmEndpoint = true
  87.     return n, config, nil
  88. }

——1.2.1.1.2.2) daemon/container_operations.go#Daemon.getNetworkSandbox

  1. func (daemon *Daemon) getNetworkSandbox(container *container.Container) libnetwork.Sandbox {
  2.     var sb libnetwork.Sandbox
  3.     daemon.netController.WalkSandboxes(func(s libnetwork.Sandbox) bool {
  4.         if s.ContainerID() == container.ID {
  5.             sb = s
  6.             return true
  7.         }
  8.         return false
  9.     })
  10.     return sb
  11. }

——1.2.1.1.2.3) daemon/network.go#buildCreateEndpointOptions

  1. // buildCreateEndpointOptions builds endpoint options from a given network.
  2. func buildCreateEndpointOptions(c *container.Container, n libnetwork.Network, epConfig *network.EndpointSettings, sb libnetwork.Sandbox, daemonDNS []string) ([]libnetwork.EndpointOption, error) {
  3.     var (
  4.         bindings      = make(nat.PortMap)
  5.         pbList        []networktypes.PortBinding
  6.         exposeList    []networktypes.TransportPort
  7.         createOptions []libnetwork.EndpointOption
  8.     )
  10.     defaultNetName := runconfig.DefaultDaemonNetworkMode().NetworkName()
  12.     if (!c.EnableServiceDiscoveryOnDefaultNetwork() && n.Name() == defaultNetName) ||
  13.         c.NetworkSettings.IsAnonymousEndpoint {
  14.         createOptions = append(createOptions, libnetwork.CreateOptionAnonymous())
  15.     }
  17.     if epConfig != nil {
  18.         ipam := epConfig.IPAMConfig
  20.         if ipam != nil {
  21.             var (
  22.                 ipList          []net.IP
  23.                 ip, ip6, linkip net.IP
  24.             )
  26.             for _, ips := range ipam.LinkLocalIPs {
  27.                 if linkip = net.ParseIP(ips); linkip == nil && ips != "" {
  28.                     return nil, errors.Errorf("Invalid link-local IP address: %s", ipam.LinkLocalIPs)
  29.                 }
  30.                 ipList = append(ipList, linkip)
  32.             }
  34.             if ip = net.ParseIP(ipam.IPv4Address); ip == nil && ipam.IPv4Address != "" {
  35.                 return nil, errors.Errorf("Invalid IPv4 address: %s)", ipam.IPv4Address)
  36.             }
  38.             if ip6 = net.ParseIP(ipam.IPv6Address); ip6 == nil && ipam.IPv6Address != "" {
  39.                 return nil, errors.Errorf("Invalid IPv6 address: %s)", ipam.IPv6Address)
  40.             }
  42.             createOptions = append(createOptions,
  43.                 libnetwork.CreateOptionIpam(ip, ip6, ipList, nil))
  45.         }
  47.         for _, alias := range epConfig.Aliases {
  48.             createOptions = append(createOptions, libnetwork.CreateOptionMyAlias(alias))
  49.         }
  50.         for k, v := range epConfig.DriverOpts {
  51.             createOptions = append(createOptions, libnetwork.EndpointOptionGeneric(options.Generic{k: v}))
  52.         }
  53.     }
  55.     if c.NetworkSettings.Service != nil {
  56.         svcCfg := c.NetworkSettings.Service
  58.         var vip string
  59.         if svcCfg.VirtualAddresses[n.ID()] != nil {
  60.             vip = svcCfg.VirtualAddresses[n.ID()].IPv4
  61.         }
  63.         var portConfigs []*libnetwork.PortConfig
  64.         for _, portConfig := range svcCfg.ExposedPorts {
  65.             portConfigs = append(portConfigs, &libnetwork.PortConfig{
  66.                 Name:          portConfig.Name,
  67.                 Protocol:      libnetwork.PortConfig_Protocol(portConfig.Protocol),
  68.                 TargetPort:    portConfig.TargetPort,
  69.                 PublishedPort: portConfig.PublishedPort,
  70.             })
  71.         }
  73.         createOptions = append(createOptions, libnetwork.CreateOptionService(svcCfg.Name, svcCfg.ID, net.ParseIP(vip), portConfigs, svcCfg.Aliases[n.ID()]))
  74.     }
  76.     if !containertypes.NetworkMode(n.Name()).IsUserDefined() {
  77.         createOptions = append(createOptions, libnetwork.CreateOptionDisableResolution())
  78.     }
  80.     // configs that are applicable only for the endpoint in the network
  81.     // to which container was connected to on docker run.
  82.     // Ideally all these network-specific endpoint configurations must be moved under
  83.     // container.NetworkSettings.Networks[n.Name()]
  84.     if n.Name() == c.HostConfig.NetworkMode.NetworkName() ||
  85.         (n.Name() == defaultNetName && c.HostConfig.NetworkMode.IsDefault()) {
  86.         if c.Config.MacAddress != "" {
  87.             mac, err := net.ParseMAC(c.Config.MacAddress)
  88.             if err != nil {
  89.                 return nil, err
  90.             }
  92.             genericOption := options.Generic{
  93.                 netlabel.MacAddress: mac,
  94.             }
  96.             createOptions = append(createOptions, libnetwork.EndpointOptionGeneric(genericOption))
  97.         }
  99.     }
  101.     // Port-mapping rules belong to the container & applicable only to non-internal networks
  102.     portmaps := getSandboxPortMapInfo(sb)
  103.     if n.Info().Internal() || len(portmaps) > 0 {
  104.         return createOptions, nil
  105.     }
  107.     if c.HostConfig.PortBindings != nil {
  108.         for p, b := range c.HostConfig.PortBindings {
  109.             bindings[p] = []nat.PortBinding{}
  110.             for _, bb := range b {
  111.                 bindings[p] = append(bindings[p], nat.PortBinding{
  112.                     HostIP:   bb.HostIP,
  113.                     HostPort: bb.HostPort,
  114.                 })
  115.             }
  116.         }
  117.     }
  119.     portSpecs := c.Config.ExposedPorts
  120.     ports := make([]nat.Port, len(portSpecs))
  121.     var i int
  122.     for p := range portSpecs {
  123.         ports[i] = p
  124.         i++
  125.     }
  126.     nat.SortPortMap(ports, bindings)
  127.     for _, port := range ports {
  128.         expose := networktypes.TransportPort{}
  129.         expose.Proto = networktypes.ParseProtocol(port.Proto())
  130.         expose.Port = uint16(port.Int())
  131.         exposeList = append(exposeList, expose)
  133.         pb := networktypes.PortBinding{Port: expose.Port, Proto: expose.Proto}
  134.         binding := bindings[port]
  135.         for i := 0; i < len(binding); i++ {
  136.             pbCopy := pb.GetCopy()
  137.             newP, err := nat.NewPort(nat.SplitProtoPort(binding[i].HostPort))
  138.             var portStart, portEnd int
  139.             if err == nil {
  140.                 portStart, portEnd, err = newP.Range()
  141.             }
  142.             if err != nil {
  143.                 return nil, errors.Wrapf(err, "Error parsing HostPort value (%s)", binding[i].HostPort)
  144.             }
  145.             pbCopy.HostPort = uint16(portStart)
  146.             pbCopy.HostPortEnd = uint16(portEnd)
  147.             pbCopy.HostIP = net.ParseIP(binding[i].HostIP)
  148.             pbList = append(pbList, pbCopy)
  149.         }
  151.         if c.HostConfig.PublishAllPorts && len(binding) == 0 {
  152.             pbList = append(pbList, pb)
  153.         }
  154.     }
  156.     var dns []string
  158.     if len(c.HostConfig.DNS) > 0 {
  159.         dns = c.HostConfig.DNS
  160.     } else if len(daemonDNS) > 0 {
  161.         dns = daemonDNS
  162.     }
  164.     if len(dns) > 0 {
  165.         createOptions = append(createOptions,
  166.             libnetwork.CreateOptionDNS(dns))
  167.     }
  169.     createOptions = append(createOptions,
  170.         libnetwork.CreateOptionPortMapping(pbList),
  171.         libnetwork.CreateOptionExposedPorts(exposeList))
  173.     return createOptions, nil
  174. }

——1.2.1.1.2.4) libnetwork/network.go#network.CreateEndpoint

  1. func (n *network) CreateEndpoint(name string, options ...EndpointOption) (Endpoint, error) {
  2.     var err error
  3.     if !config.IsValidName(name) {
  4.         return nil, ErrInvalidName(name)
  5.     }
  7.     if n.ConfigOnly() {
  8.         return nil, types.ForbiddenErrorf("cannot create endpoint on configuration-only network")
  9.     }
  11.     if _, err = n.EndpointByName(name); err == nil {
  12.         return nil, types.ForbiddenErrorf("endpoint with name %s already exists in network %s", name, n.Name())
  13.     }
  15.     n.ctrlr.networkLocker.Lock(n.id)
  16.     defer n.ctrlr.networkLocker.Unlock(n.id)
  17.     // ********************************** NOTICE ********************************** //
  18.     return n.createEndpoint(name, options...)
  19.   // ********************************** NOTICE ********************************** //
  21. }
  23. func (n *network) createEndpoint(name string, options ...EndpointOption) (Endpoint, error) {
  24.     var err error
  26.     ep := &endpoint{name: name, generic: make(map[string]interface{}), iface: &endpointInterface{}}
  27.     ep.id = stringid.GenerateRandomID()
  29.     // Initialize ep.network with a possibly stale copy of n. We need this to get network from
  30.     // store. But once we get it from store we will have the most uptodate copy possibly.
  31.     ep.network = n
  32.     ep.locator = n.getController().clusterHostID()
  33.     ep.network, err = ep.getNetworkFromStore()
  34.     if err != nil {
  35.         return nil, fmt.Errorf("failed to get network during CreateEndpoint: %v", err)
  36.     }
  37.     n = ep.network
  39.     ep.processOptions(options...)
  41.     for _, llIPNet := range ep.Iface().LinkLocalAddresses() {
  42.         if !llIPNet.IP.IsLinkLocalUnicast() {
  43.             return nil, types.BadRequestErrorf("invalid link local IP address: %v", llIPNet.IP)
  44.         }
  45.     }
  47.     if opt, ok := ep.generic[netlabel.MacAddress]; ok {
  48.         if mac, ok := opt.(net.HardwareAddr); ok {
  49.             ep.iface.mac = mac
  50.         }
  51.     }
  53.     ipam, cap, err := n.getController().getIPAMDriver(n.ipamType)
  54.     if err != nil {
  55.         return nil, err
  56.     }
  58.     if cap.RequiresMACAddress {
  59.         if ep.iface.mac == nil {
  60.             ep.iface.mac = netutils.GenerateRandomMAC()
  61.         }
  62.         if ep.ipamOptions == nil {
  63.             ep.ipamOptions = make(map[string]string)
  64.         }
  65.         ep.ipamOptions[netlabel.MacAddress] = ep.iface.mac.String()
  66.     }
  67.     // ********************************** NOTICE ********************************** //
  68.     if err = ep.assignAddress(ipam, true, n.enableIPv6 && !n.postIPv6); err != nil {
  69.     // ********************************** NOTICE ********************************** //
  70.         return nil, err
  71.     }
  72.     defer func() {
  73.         if err != nil {
  74.             ep.releaseAddress()
  75.         }
  76.     }()
  78.   // ********************************** NOTICE ********************************** //
  79.     if err = n.addEndpoint(ep); err != nil {
  80.     // ********************************** NOTICE ********************************** //
  81.         return nil, err
  82.     }
  83.     defer func() {
  84.         if err != nil {
  85.             if e := ep.deleteEndpoint(false); e != nil {
  86.                 logrus.Warnf("cleaning up endpoint failed %s : %v", name, e)
  87.             }
  88.         }
  89.     }()
  91.     // We should perform updateToStore call right after addEndpoint
  92.     // in order to have iface properly configured
  93.     if err = n.getController().updateToStore(ep); err != nil {
  94.         return nil, err
  95.     }
  96.     defer func() {
  97.         if err != nil {
  98.             if e := n.getController().deleteFromStore(ep); e != nil {
  99.                 logrus.Warnf("error rolling back endpoint %s from store: %v", name, e)
  100.             }
  101.         }
  102.     }()
  104.     if err = ep.assignAddress(ipam, false, n.enableIPv6 && n.postIPv6); err != nil {
  105.         return nil, err
  106.     }
  108.     // Watch for service records
  109.     n.getController().watchSvcRecord(ep)
  110.     defer func() {
  111.         if err != nil {
  112.             n.getController().unWatchSvcRecord(ep)
  113.         }
  114.     }()
  116.     // Increment endpoint count to indicate completion of endpoint addition
  117.     if err = n.getEpCnt().IncEndpointCnt(); err != nil {
  118.         return nil, err
  119.     }
  121.     return ep, nil
  122. }
  124. func (n *network) addEndpoint(ep *endpoint) error {
  125.     d, err := n.driver(true)
  126.     if err != nil {
  127.         return fmt.Errorf("failed to add endpoint: %v", err)
  128.     }
  129.     // ********************************** NOTICE ********************************** //
  130.     err = d.CreateEndpoint(n.id, ep.id, ep.Interface(), ep.generic)
  131.   // ********************************** NOTICE ********************************** //
  132.     if err != nil {
  133.         return types.InternalErrorf("failed to create endpoint %s on network %s: %v",
  134.             ep.Name(), n.Name(), err)
  135.     }
  137.     return nil
  138. }

BridgeDriver

——— 1.2.1.1.2.4.1) libnetwork/drivers/bridge/bridge.go#driver.CreateEndpoint

  1. func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo, epOptions map[string]interface{}) error {
  2.     defer osl.InitOSContext()()
  4.     if ifInfo == nil {
  5.         return errors.New("invalid interface info passed")
  6.     }
  8.     // Get the network handler and make sure it exists
  9.     d.Lock()
  10.     n, ok := d.networks[nid]
  11.     dconfig := d.config
  12.     d.Unlock()
  14.     if !ok {
  15.         return types.NotFoundErrorf("network %s does not exist", nid)
  16.     }
  17.     if n == nil {
  18.         return driverapi.ErrNoNetwork(nid)
  19.     }
  21.     // Sanity check
  22.     n.Lock()
  23.     if n.id != nid {
  24.         n.Unlock()
  25.         return InvalidNetworkIDError(nid)
  26.     }
  27.     n.Unlock()
  29.     // Check if endpoint id is good and retrieve correspondent endpoint
  30.     ep, err := n.getEndpoint(eid)
  31.     if err != nil {
  32.         return err
  33.     }
  35.     // Endpoint with that id exists either on desired or other sandbox
  36.     if ep != nil {
  37.         return driverapi.ErrEndpointExists(eid)
  38.     }
  40.     // Try to convert the options to endpoint configuration
  41.     epConfig, err := parseEndpointOptions(epOptions)
  42.     if err != nil {
  43.         return err
  44.     }
  46.     // Create and add the endpoint
  47.     n.Lock()
  48.     endpoint := &bridgeEndpoint{id: eid, nid: nid, config: epConfig}
  49.     n.endpoints[eid] = endpoint
  50.     n.Unlock()
  52.     // On failure make sure to remove the endpoint
  53.     defer func() {
  54.         if err != nil {
  55.             n.Lock()
  56.             delete(n.endpoints, eid)
  57.             n.Unlock()
  58.         }
  59.     }()
  61.     // Generate a name for what will be the host side pipe interface
  62.     hostIfName, err := netutils.GenerateIfaceName(d.nlh, vethPrefix, vethLen)
  63.     if err != nil {
  64.         return err
  65.     }
  67.     // Generate a name for what will be the sandbox side pipe interface
  68.     containerIfName, err := netutils.GenerateIfaceName(d.nlh, vethPrefix, vethLen)
  69.     if err != nil {
  70.         return err
  71.     }
  73.     // Generate and add the interface pipe host <-> sandbox
  74.   // ********************************** NOTICE ********************************** //
  75.     veth := &netlink.Veth{
  76.         LinkAttrs: netlink.LinkAttrs{Name: hostIfName, TxQLen: 0},
  77.         PeerName:  containerIfName}
  78.     if err = d.nlh.LinkAdd(veth); err != nil {
  79.     // ********************************** NOTICE ********************************** //
  80.         return types.InternalErrorf("failed to add the host (%s) <=> sandbox (%s) pair interfaces: %v", hostIfName, containerIfName, err)
  81.     }
  83.     // Get the host side pipe interface handler
  84.     host, err := d.nlh.LinkByName(hostIfName)
  85.     if err != nil {
  86.         return types.InternalErrorf("failed to find host side interface %s: %v", hostIfName, err)
  87.     }
  88.     defer func() {
  89.         if err != nil {
  90.             if err := d.nlh.LinkDel(host); err != nil {
  91.                 logrus.WithError(err).Warnf("Failed to delete host side interface (%s)'s link", hostIfName)
  92.             }
  93.         }
  94.     }()
  96.     // Get the sandbox side pipe interface handler
  97.     sbox, err := d.nlh.LinkByName(containerIfName)
  98.     if err != nil {
  99.         return types.InternalErrorf("failed to find sandbox side interface %s: %v", containerIfName, err)
  100.     }
  101.     defer func() {
  102.         if err != nil {
  103.             if err := d.nlh.LinkDel(sbox); err != nil {
  104.                 logrus.WithError(err).Warnf("Failed to delete sandbox side interface (%s)'s link", containerIfName)
  105.             }
  106.         }
  107.     }()
  109.     n.Lock()
  110.     config := n.config
  111.     n.Unlock()
  113.     // Add bridge inherited attributes to pipe interfaces
  114.     if config.Mtu != 0 {
  115.         err = d.nlh.LinkSetMTU(host, config.Mtu)
  116.         if err != nil {
  117.             return types.InternalErrorf("failed to set MTU on host interface %s: %v", hostIfName, err)
  118.         }
  119.         err = d.nlh.LinkSetMTU(sbox, config.Mtu)
  120.         if err != nil {
  121.             return types.InternalErrorf("failed to set MTU on sandbox interface %s: %v", containerIfName, err)
  122.         }
  123.     }
  125.     // Attach host side pipe interface into the bridge
  126.   // ********************************** NOTICE ********************************** //
  127.     if err = addToBridge(d.nlh, hostIfName, config.BridgeName); err != nil {
  128.     // ********************************** NOTICE ********************************** //
  129.         return fmt.Errorf("adding interface %s to bridge %s failed: %v", hostIfName, config.BridgeName, err)
  130.     }
  132.     if !dconfig.EnableUserlandProxy {
  133.         err = setHairpinMode(d.nlh, host, true)
  134.         if err != nil {
  135.             return err
  136.         }
  137.     }
  139.     // Store the sandbox side pipe interface parameters
  140.     endpoint.srcName = containerIfName
  141.     endpoint.macAddress = ifInfo.MacAddress()
  142.     endpoint.addr = ifInfo.Address()
  143.     endpoint.addrv6 = ifInfo.AddressIPv6()
  145.     // Set the sbox's MAC if not provided. If specified, use the one configured by user, otherwise generate one based on IP.
  146.     if endpoint.macAddress == nil {
  147.         endpoint.macAddress = electMacAddress(epConfig, endpoint.addr.IP)
  148.         if err = ifInfo.SetMacAddress(endpoint.macAddress); err != nil {
  149.             return err
  150.         }
  151.     }
  153.     // Up the host interface after finishing all netlink configuration
  154.     if err = d.nlh.LinkSetUp(host); err != nil {
  155.         return fmt.Errorf("could not set link up for host interface %s: %v", hostIfName, err)
  156.     }
  158.     if endpoint.addrv6 == nil && config.EnableIPv6 {
  159.         var ip6 net.IP
  160.         network := n.bridge.bridgeIPv6
  161.         if config.AddressIPv6 != nil {
  162.             network = config.AddressIPv6
  163.         }
  165.         ones, _ := network.Mask.Size()
  166.         if ones > 80 {
  167.             err = types.ForbiddenErrorf("Cannot self generate an IPv6 address on network %v: At least 48 host bits are needed.", network)
  168.             return err
  169.         }
  171.         ip6 = make(net.IP, len(network.IP))
  172.         copy(ip6, network.IP)
  173.         for i, h := range endpoint.macAddress {
  174.             ip6[i+10] = h
  175.         }
  177.         endpoint.addrv6 = &net.IPNet{IP: ip6, Mask: network.Mask}
  178.         if err = ifInfo.SetIPAddress(endpoint.addrv6); err != nil {
  179.             return err
  180.         }
  181.     }
  183.     if err = d.storeUpdate(endpoint); err != nil {
  184.         return fmt.Errorf("failed to save bridge endpoint %.7s to store: %v", endpoint.id, err)
  185.     }
  187.     return nil
  188. }
  190. func addToBridge(nlh *netlink.Handle, ifaceName, bridgeName string) error {
  191.     link, err := nlh.LinkByName(ifaceName)
  192.     if err != nil {
  193.         return fmt.Errorf("could not find interface %s: %v", ifaceName, err)
  194.     }
  195.     if err = nlh.LinkSetMaster(link,
  196.         &netlink.Bridge{LinkAttrs: netlink.LinkAttrs{Name: bridgeName}}); err != nil {
  197.         logrus.Debugf("Failed to add %s to bridge via netlink.Trying ioctl: %v", ifaceName, err)
  198.         iface, err := net.InterfaceByName(ifaceName)
  199.         if err != nil {
  200.             return fmt.Errorf("could not find network interface %s: %v", ifaceName, err)
  201.         }
  203.         master, err := net.InterfaceByName(bridgeName)
  204.         if err != nil {
  205.             return fmt.Errorf("could not find bridge %s: %v", bridgeName, err)
  206.         }
  208.         return ioctlAddToBridge(iface, master)
  209.     }
  210.     return nil
  211. }

——1.2.1.1.2.5) daemon/container_operations.go#Daemon.updateEndpointNetworkSettings

  1. func (daemon *Daemon) updateEndpointNetworkSettings(container *container.Container, n libnetwork.Network, ep libnetwork.Endpoint) error {
  2.     if err := buildEndpointInfo(container.NetworkSettings, n, ep); err != nil {
  3.         return err
  4.     }
  6.     if container.HostConfig.NetworkMode == runconfig.DefaultDaemonNetworkMode() {
  7.         container.NetworkSettings.Bridge = daemon.configStore.BridgeConfig.Iface
  8.     }
  10.     return nil
  11. }

——1.2.1.1.2.6) daemon/container_operations.go#Daemon.buildSandboxOptions

  1. func (daemon *Daemon) buildSandboxOptions(container *container.Container) ([]libnetwork.SandboxOption, error) {
  2.     var (
  3.         sboxOptions []libnetwork.SandboxOption
  4.         err         error
  5.         dns         []string
  6.         dnsOptions  []string
  7.         bindings    = make(nat.PortMap)
  8.         pbList      []types.PortBinding
  9.         exposeList  []types.TransportPort
  10.     )
  12.     defaultNetName := runconfig.DefaultDaemonNetworkMode().NetworkName()
  13.     sboxOptions = append(sboxOptions, libnetwork.OptionHostname(container.Config.Hostname),
  14.         libnetwork.OptionDomainname(container.Config.Domainname))
  16.     if container.HostConfig.NetworkMode.IsHost() {
  17.         sboxOptions = append(sboxOptions, libnetwork.OptionUseDefaultSandbox())
  18.     } else {
  19.         // OptionUseExternalKey is mandatory for userns support.
  20.         // But optional for non-userns support
  21.         sboxOptions = append(sboxOptions, libnetwork.OptionUseExternalKey())
  22.     }
  24.     if err = daemon.setupPathsAndSandboxOptions(container, &sboxOptions); err != nil {
  25.         return nil, err
  26.     }
  28.     if len(container.HostConfig.DNS) > 0 {
  29.         dns = container.HostConfig.DNS
  30.     } else if len(daemon.configStore.DNS) > 0 {
  31.         dns = daemon.configStore.DNS
  32.     }
  34.     for _, d := range dns {
  35.         sboxOptions = append(sboxOptions, libnetwork.OptionDNS(d))
  36.     }
  38.     dnsSearch := daemon.getDNSSearchSettings(container)
  40.     for _, ds := range dnsSearch {
  41.         sboxOptions = append(sboxOptions, libnetwork.OptionDNSSearch(ds))
  42.     }
  44.     if len(container.HostConfig.DNSOptions) > 0 {
  45.         dnsOptions = container.HostConfig.DNSOptions
  46.     } else if len(daemon.configStore.DNSOptions) > 0 {
  47.         dnsOptions = daemon.configStore.DNSOptions
  48.     }
  50.     for _, ds := range dnsOptions {
  51.         sboxOptions = append(sboxOptions, libnetwork.OptionDNSOptions(ds))
  52.     }
  54.     if container.NetworkSettings.SecondaryIPAddresses != nil {
  55.         name := container.Config.Hostname
  56.         if container.Config.Domainname != "" {
  57.             name = name + "." + container.Config.Domainname
  58.         }
  60.         for _, a := range container.NetworkSettings.SecondaryIPAddresses {
  61.             sboxOptions = append(sboxOptions, libnetwork.OptionExtraHost(name, a.Addr))
  62.         }
  63.     }
  65.     for _, extraHost := range container.HostConfig.ExtraHosts {
  66.         // allow IPv6 addresses in extra hosts; only split on first ":"
  67.         if _, err := opts.ValidateExtraHost(extraHost); err != nil {
  68.             return nil, err
  69.         }
  70.         parts := strings.SplitN(extraHost, ":", 2)
  71.         sboxOptions = append(sboxOptions, libnetwork.OptionExtraHost(parts[0], parts[1]))
  72.     }
  74.     if container.HostConfig.PortBindings != nil {
  75.         for p, b := range container.HostConfig.PortBindings {
  76.             bindings[p] = []nat.PortBinding{}
  77.             for _, bb := range b {
  78.                 bindings[p] = append(bindings[p], nat.PortBinding{
  79.                     HostIP:   bb.HostIP,
  80.                     HostPort: bb.HostPort,
  81.                 })
  82.             }
  83.         }
  84.     }
  86.     portSpecs := container.Config.ExposedPorts
  87.     ports := make([]nat.Port, len(portSpecs))
  88.     var i int
  89.     for p := range portSpecs {
  90.         ports[i] = p
  91.         i++
  92.     }
  93.     nat.SortPortMap(ports, bindings)
  94.     for _, port := range ports {
  95.         expose := types.TransportPort{}
  96.         expose.Proto = types.ParseProtocol(port.Proto())
  97.         expose.Port = uint16(port.Int())
  98.         exposeList = append(exposeList, expose)
  100.         pb := types.PortBinding{Port: expose.Port, Proto: expose.Proto}
  101.         binding := bindings[port]
  102.         for i := 0; i < len(binding); i++ {
  103.             pbCopy := pb.GetCopy()
  104.             newP, err := nat.NewPort(nat.SplitProtoPort(binding[i].HostPort))
  105.             var portStart, portEnd int
  106.             if err == nil {
  107.                 portStart, portEnd, err = newP.Range()
  108.             }
  109.             if err != nil {
  110.                 return nil, fmt.Errorf("Error parsing HostPort value(%s):%v", binding[i].HostPort, err)
  111.             }
  112.             pbCopy.HostPort = uint16(portStart)
  113.             pbCopy.HostPortEnd = uint16(portEnd)
  114.             pbCopy.HostIP = net.ParseIP(binding[i].HostIP)
  115.             pbList = append(pbList, pbCopy)
  116.         }
  118.         if container.HostConfig.PublishAllPorts && len(binding) == 0 {
  119.             pbList = append(pbList, pb)
  120.         }
  121.     }
  123.     sboxOptions = append(sboxOptions,
  124.         libnetwork.OptionPortMapping(pbList),
  125.         libnetwork.OptionExposedPorts(exposeList))
  127.     // Legacy Link feature is supported only for the default bridge network.
  128.     // return if this call to build join options is not for default bridge network
  129.     // Legacy Link is only supported by docker run --link
  130.     bridgeSettings, ok := container.NetworkSettings.Networks[defaultNetName]
  131.     if !ok || bridgeSettings.EndpointSettings == nil {
  132.         return sboxOptions, nil
  133.     }
  135.     if bridgeSettings.EndpointID == "" {
  136.         return sboxOptions, nil
  137.     }
  139.     var (
  140.         childEndpoints, parentEndpoints []string
  141.         cEndpointID                     string
  142.     )
  144.     children := daemon.children(container)
  145.     for linkAlias, child := range children {
  146.         if !isLinkable(child) {
  147.             return nil, fmt.Errorf("Cannot link to %s, as it does not belong to the default network", child.Name)
  148.         }
  149.         _, alias := path.Split(linkAlias)
  150.         // allow access to the linked container via the alias, real name, and container hostname
  151.         aliasList := alias + " " + child.Config.Hostname
  152.         // only add the name if alias isn't equal to the name
  153.         if alias != child.Name[1:] {
  154.             aliasList = aliasList + " " + child.Name[1:]
  155.         }
  156.         sboxOptions = append(sboxOptions, libnetwork.OptionExtraHost(aliasList, child.NetworkSettings.Networks[defaultNetName].IPAddress))
  157.         cEndpointID = child.NetworkSettings.Networks[defaultNetName].EndpointID
  158.         if cEndpointID != "" {
  159.             childEndpoints = append(childEndpoints, cEndpointID)
  160.         }
  161.     }
  163.     for alias, parent := range daemon.parents(container) {
  164.         if daemon.configStore.DisableBridge || !container.HostConfig.NetworkMode.IsPrivate() {
  165.             continue
  166.         }
  168.         _, alias = path.Split(alias)
  169.         logrus.Debugf("Update /etc/hosts of %s for alias %s with ip %s", parent.ID, alias, bridgeSettings.IPAddress)
  170.         sboxOptions = append(sboxOptions, libnetwork.OptionParentUpdate(
  171.             parent.ID,
  172.             alias,
  173.             bridgeSettings.IPAddress,
  174.         ))
  175.         if cEndpointID != "" {
  176.             parentEndpoints = append(parentEndpoints, cEndpointID)
  177.         }
  178.     }
  180.     linkOptions := options.Generic{
  181.         netlabel.GenericData: options.Generic{
  182.             "ParentEndpoints": parentEndpoints,
  183.             "ChildEndpoints":  childEndpoints,
  184.         },
  185.     }
  187.     sboxOptions = append(sboxOptions, libnetwork.OptionGeneric(linkOptions))
  188.     return sboxOptions, nil
  189. }

——1.2.1.1.2.7-创建NetNS) libnetwork/controller.go#controller.NewSandbox

  1. // NetworkController provides the interface for controller instance which manages
  2. // networks.
  3. type NetworkController interface {
  4.     // ID provides a unique identity for the controller
  5.     ID() string
  7.     // BuiltinDrivers returns list of builtin drivers
  8.     BuiltinDrivers() []string
  10.     // BuiltinIPAMDrivers returns list of builtin ipam drivers
  11.     BuiltinIPAMDrivers() []string
  13.     // Config method returns the bootup configuration for the controller
  14.     Config() config.Config
  16.     // Create a new network. The options parameter carries network specific options.
  17.     NewNetwork(networkType, name string, id string, options ...NetworkOption) (Network, error)
  19.     // Networks returns the list of Network(s) managed by this controller.
  20.     Networks() []Network
  22.     // WalkNetworks uses the provided function to walk the Network(s) managed by this controller.
  23.     WalkNetworks(walker NetworkWalker)
  25.     // NetworkByName returns the Network which has the passed name. If not found, the error ErrNoSuchNetwork is returned.
  26.     NetworkByName(name string) (Network, error)
  28.     // NetworkByID returns the Network which has the passed id. If not found, the error ErrNoSuchNetwork is returned.
  29.     NetworkByID(id string) (Network, error)
  31.     // NewSandbox creates a new network sandbox for the passed container id
  32.     NewSandbox(containerID string, options ...SandboxOption) (Sandbox, error)
  34.     // Sandboxes returns the list of Sandbox(s) managed by this controller.
  35.     Sandboxes() []Sandbox
  37.     // WalkSandboxes uses the provided function to walk the Sandbox(s) managed by this controller.
  38.     WalkSandboxes(walker SandboxWalker)
  40.     // SandboxByID returns the Sandbox which has the passed id. If not found, a types.NotFoundError is returned.
  41.     SandboxByID(id string) (Sandbox, error)
  43.     // SandboxDestroy destroys a sandbox given a container ID
  44.     SandboxDestroy(id string) error
  46.     // Stop network controller
  47.     Stop()
  49.     // ReloadConfiguration updates the controller configuration
  50.     ReloadConfiguration(cfgOptions ...config.Option) error
  52.     // SetClusterProvider sets cluster provider
  53.     SetClusterProvider(provider cluster.Provider)
  55.     // Wait for agent initialization complete in libnetwork controller
  56.     AgentInitWait()
  58.     // Wait for agent to stop if running
  59.     AgentStopWait()
  61.     // SetKeys configures the encryption key for gossip and overlay data path
  62.     SetKeys(keys []*types.EncryptionKey) error
  64.     // StartDiagnostic start the network diagnostic mode
  65.     StartDiagnostic(port int)
  66.     // StopDiagnostic start the network diagnostic mode
  67.     StopDiagnostic()
  68.     // IsDiagnosticEnabled returns true if the diagnostic is enabled
  69.     IsDiagnosticEnabled() bool
  70. }
  1. // NewSandbox creates a new sandbox for the passed container id
  2. func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (Sandbox, error) {
  3.     if containerID == "" {
  4.         return nil, types.BadRequestErrorf("invalid container ID")
  5.     }
  7.     var sb *sandbox
  8.     c.Lock()
  9.     for _, s := range c.sandboxes {
  10.         if s.containerID == containerID {
  11.             // If not a stub, then we already have a complete sandbox.
  12.             if !s.isStub {
  13.                 sbID := s.ID()
  14.                 c.Unlock()
  15.                 return nil, types.ForbiddenErrorf("container %s is already present in sandbox %s", containerID, sbID)
  16.             }
  18.             // We already have a stub sandbox from the
  19.             // store. Make use of it so that we don't lose
  20.             // the endpoints from store but reset the
  21.             // isStub flag.
  22.             sb = s
  23.             sb.isStub = false
  24.             break
  25.         }
  26.     }
  27.     c.Unlock()
  29.     sandboxID := stringid.GenerateRandomID()
  30.     if runtime.GOOS == "windows" {
  31.         sandboxID = containerID
  32.     }
  34.     // Create sandbox and process options first. Key generation depends on an option
  35.     if sb == nil {
  36.         sb = &sandbox{
  37.             id:                 sandboxID,
  38.             containerID:        containerID,
  39.             endpoints:          []*endpoint{},
  40.             epPriority:         map[string]int{},
  41.             populatedEndpoints: map[string]struct{}{},
  42.             config:             containerConfig{},
  43.             controller:         c,
  44.             extDNS:             []extDNSEntry{},
  45.         }
  46.     }
  48.     sb.processOptions(options...)
  50.     c.Lock()
  51.     if sb.ingress && c.ingressSandbox != nil {
  52.         c.Unlock()
  53.         return nil, types.ForbiddenErrorf("ingress sandbox already present")
  54.     }
  56.     if sb.ingress {
  57.         c.ingressSandbox = sb
  58.         sb.config.hostsPath = filepath.Join(c.cfg.Daemon.DataDir, "/network/files/hosts")
  59.         sb.config.resolvConfPath = filepath.Join(c.cfg.Daemon.DataDir, "/network/files/resolv.conf")
  60.         sb.id = "ingress_sbox"
  61.     } else if sb.loadBalancerNID != "" {
  62.         sb.id = "lb_" + sb.loadBalancerNID
  63.     }
  64.     c.Unlock()
  66.     var err error
  67.     defer func() {
  68.         if err != nil {
  69.             c.Lock()
  70.             if sb.ingress {
  71.                 c.ingressSandbox = nil
  72.             }
  73.             c.Unlock()
  74.         }
  75.     }()
  76.     // ********************************** NOTICE ********************************** //
  77.     if err = sb.setupResolutionFiles(); err != nil {
  78.     // ********************************** NOTICE ********************************** //
  79.         return nil, err
  80.     }
  82.     if sb.config.useDefaultSandBox {
  83.         c.sboxOnce.Do(func() {
  84.             c.defOsSbox, err = osl.NewSandbox(sb.Key(), false, false)
  85.         })
  87.         if err != nil {
  88.             c.sboxOnce = sync.Once{}
  89.             return nil, fmt.Errorf("failed to create default sandbox: %v", err)
  90.         }
  92.         sb.osSbox = c.defOsSbox
  93.     }
  95.     if sb.osSbox == nil && !sb.config.useExternalKey {
  96.     // ********************************** NOTICE ********************************** //
  97.         if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox, false);
  98.     err != nil {
  99.       // ********************************** NOTICE ********************************** //
  100.             return nil, fmt.Errorf("failed to create new osl sandbox: %v", err)
  101.         }
  102.     }
  104.     if sb.osSbox != nil {
  105.         // Apply operating specific knobs on the load balancer sandbox
  106.         sb.osSbox.ApplyOSTweaks(sb.oslTypes)
  107.     }
  109.     c.Lock()
  110.     c.sandboxes[sb.id] = sb
  111.     c.Unlock()
  112.     defer func() {
  113.         if err != nil {
  114.             c.Lock()
  115.             delete(c.sandboxes, sb.id)
  116.             c.Unlock()
  117.         }
  118.     }()
  120.     err = sb.storeUpdate()
  121.     if err != nil {
  122.         return nil, fmt.Errorf("failed to update the store state of sandbox: %v", err)
  123.     }
  125.     return sb, nil
  126. }

———1.2.1.1.2.7.1) libnetwork/sandbox_dns_unix.go#sandbox.setupResolutionFiles

  1. func (sb *sandbox) setupResolutionFiles() error {
  2.   // ********************************** NOTICE ********************************** //
  3.     if err := sb.buildHostsFile(); err != nil {
  4.     // ********************************** NOTICE ********************************** //
  5.         return err
  6.     }
  8.     if err := sb.updateParentHosts(); err != nil {
  9.         return err
  10.     }
  11.     // ********************************** NOTICE ********************************** //
  12.     return sb.setupDNS()
  13.   // ********************************** NOTICE ********************************** //
  14. }

————1.2.1.1.2.7.1.1) libnetwork/sandbox_dns_unix.go#sandbox.buildHostsFile

  2. func (sb *sandbox) buildHostsFile() error {
  3.     if sb.config.hostsPath == "" {
  4.         sb.config.hostsPath = defaultPrefix + "/" + sb.id + "/hosts"
  5.     }
  7.     dir, _ := filepath.Split(sb.config.hostsPath)
  8.     if err := createBasePath(dir); err != nil {
  9.         return err
  10.     }
  12.     // This is for the host mode networking
  13.     if sb.config.useDefaultSandBox && len(sb.config.extraHosts) == 0 {
  14.         // We are working under the assumption that the origin file option had been properly expressed by the upper layer
  15.         // if not here we are going to error out
  16.         if err := copyFile(sb.config.originHostsPath, sb.config.hostsPath); err != nil && !os.IsNotExist(err) {
  17.             return types.InternalErrorf("could not copy source hosts file %s to %s: %v", sb.config.originHostsPath, sb.config.hostsPath, err)
  18.         }
  19.         return nil
  20.     }
  22.     extraContent := make([]etchosts.Record, 0, len(sb.config.extraHosts))
  23.     for _, extraHost := range sb.config.extraHosts {
  24.         extraContent = append(extraContent, etchosts.Record{Hosts: extraHost.name, IP: extraHost.IP})
  25.     }
  27.     return etchosts.Build(sb.config.hostsPath, "", sb.config.hostName, sb.config.domainName, extraContent)
  28. }

————1.2.1.1.2.7.1.2) libnetwork/sandbox_dns_unix.go#sandbox.setupDNS

  1. func (sb *sandbox) setupDNS() error {
  2.     var newRC *resolvconf.File
  4.     if sb.config.resolvConfPath == "" {
  5.         sb.config.resolvConfPath = defaultPrefix + "/" + sb.id + "/resolv.conf"
  6.     }
  8.     sb.config.resolvConfHashFile = sb.config.resolvConfPath + ".hash"
  10.     dir, _ := filepath.Split(sb.config.resolvConfPath)
  11.     if err := createBasePath(dir); err != nil {
  12.         return err
  13.     }
  15.     // When the user specify a conainter in the host namespace and do no have any dns option specified
  16.     // we just copy the host resolv.conf from the host itself
  17.     if sb.config.useDefaultSandBox &&
  18.         len(sb.config.dnsList) == 0 && len(sb.config.dnsSearchList) == 0 && len(sb.config.dnsOptionsList) == 0 {
  20.         // We are working under the assumption that the origin file option had been properly expressed by the upper layer
  21.         // if not here we are going to error out
  22.         if err := copyFile(sb.config.originResolvConfPath, sb.config.resolvConfPath); err != nil {
  23.             if !os.IsNotExist(err) {
  24.                 return fmt.Errorf("could not copy source resolv.conf file %s to %s: %v", sb.config.originResolvConfPath, sb.config.resolvConfPath, err)
  25.             }
  26.             logrus.Infof("%s does not exist, we create an empty resolv.conf for container", sb.config.originResolvConfPath)
  27.             if err := createFile(sb.config.resolvConfPath); err != nil {
  28.                 return err
  29.             }
  30.         }
  31.         return nil
  32.     }
  34.     originResolvConfPath := sb.config.originResolvConfPath
  35.     if originResolvConfPath == "" {
  36.         // if not specified fallback to default /etc/resolv.conf
  37.         originResolvConfPath = resolvconf.DefaultResolvConf
  38.     }
  39.     currRC, err := resolvconf.GetSpecific(originResolvConfPath)
  40.     if err != nil {
  41.         if !os.IsNotExist(err) {
  42.             return err
  43.         }
  44.         // it's ok to continue if /etc/resolv.conf doesn't exist, default resolvers (Google's Public DNS)
  45.         // will be used
  46.         currRC = &resolvconf.File{}
  47.         logrus.Infof("/etc/resolv.conf does not exist")
  48.     }
  50.     if len(sb.config.dnsList) > 0 || len(sb.config.dnsSearchList) > 0 || len(sb.config.dnsOptionsList) > 0 {
  51.         var (
  52.             err            error
  53.             dnsList        = resolvconf.GetNameservers(currRC.Content, types.IP)
  54.             dnsSearchList  = resolvconf.GetSearchDomains(currRC.Content)
  55.             dnsOptionsList = resolvconf.GetOptions(currRC.Content)
  56.         )
  57.         if len(sb.config.dnsList) > 0 {
  58.             dnsList = sb.config.dnsList
  59.         }
  60.         if len(sb.config.dnsSearchList) > 0 {
  61.             dnsSearchList = sb.config.dnsSearchList
  62.         }
  63.         if len(sb.config.dnsOptionsList) > 0 {
  64.             dnsOptionsList = sb.config.dnsOptionsList
  65.         }
  66.         newRC, err = resolvconf.Build(sb.config.resolvConfPath, dnsList, dnsSearchList, dnsOptionsList)
  67.         if err != nil {
  68.             return err
  69.         }
  70.         // After building the resolv.conf from the user config save the
  71.         // external resolvers in the sandbox. Note that --dns 127.0.0.x
  72.         // config refers to the loopback in the container namespace
  73.         sb.setExternalResolvers(newRC.Content, types.IPv4, false)
  74.     } else {
  75.         // If the host resolv.conf file has 127.0.0.x container should
  76.         // use the host resolver for queries. This is supported by the
  77.         // docker embedded DNS server. Hence save the external resolvers
  78.         // before filtering it out.
  79.         sb.setExternalResolvers(currRC.Content, types.IPv4, true)
  81.         // Replace any localhost/127.* (at this point we have no info about ipv6, pass it as true)
  82.         if newRC, err = resolvconf.FilterResolvDNS(currRC.Content, true); err != nil {
  83.             return err
  84.         }
  85.         // No contention on container resolv.conf file at sandbox creation
  86.         if err := ioutil.WriteFile(sb.config.resolvConfPath, newRC.Content, filePerm); err != nil {
  87.             return types.InternalErrorf("failed to write unhaltered resolv.conf file content when setting up dns for sandbox %s: %v", sb.ID(), err)
  88.         }
  89.     }
  91.     // Write hash
  92.     if err := ioutil.WriteFile(sb.config.resolvConfHashFile, []byte(newRC.Hash), filePerm); err != nil {
  93.         return types.InternalErrorf("failed to write resolv.conf hash file when setting up dns for sandbox %s: %v", sb.ID(), err)
  94.     }
  96.     return nil
  97. }

———1.2.1.1.2.7.2) libnetwork/osl/namespace_linux.go#NewSandbox

  1. // NewSandbox provides a new sandbox instance created in an os specific way
  2. // provided a key which uniquely identifies the sandbox
  3. func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
  4.     if !isRestore {
  5.     // ********************************** NOTICE ********************************** //
  6.         err := createNetworkNamespace(key, osCreate)
  7.     // ********************************** NOTICE ********************************** //
  8.         if err != nil {
  9.             return nil, err
  10.         }
  11.     } else {
  12.         once.Do(createBasePath)
  13.     }
  15.     n := &networkNamespace{path: key, isDefault: !osCreate, nextIfIndex: make(map[string]int)}
  17.   // ********************************** NOTICE ********************************** //
  18.     sboxNs, err := netns.GetFromPath(n.path)
  19.   // ********************************** NOTICE ********************************** //
  20.     if err != nil {
  21.         return nil, fmt.Errorf("failed get network namespace %q: %v", n.path, err)
  22.     }
  23.     defer sboxNs.Close()
  25.     n.nlHandle, err = netlink.NewHandleAt(sboxNs, syscall.NETLINK_ROUTE)
  26.     if err != nil {
  27.         return nil, fmt.Errorf("failed to create a netlink handle: %v", err)
  28.     }
  30.     err = n.nlHandle.SetSocketTimeout(ns.NetlinkSocketsTimeout)
  31.     if err != nil {
  32.         logrus.Warnf("Failed to set the timeout on the sandbox netlink handle sockets: %v", err)
  33.     }
  34.     // In live-restore mode, IPV6 entries are getting cleaned up due to below code
  35.     // We should retain IPV6 configurations in live-restore mode when Docker Daemon
  36.     // comes back. It should work as it is on other cases
  37.     // As starting point, disable IPv6 on all interfaces
  38.     if !isRestore && !n.isDefault {
  39.         err = setIPv6(n.path, "all", false)
  40.         if err != nil {
  41.             logrus.Warnf("Failed to disable IPv6 on all interfaces on network namespace %q: %v", n.path, err)
  42.         }
  43.     }
  45.     if err = n.loopbackUp(); err != nil {
  46.         n.nlHandle.Delete()
  47.         return nil, err
  48.     }
  50.     return n, nil
  51. }

————1.2.1.1.2.7.2.1) libnetwork/osl/namespace_linux.go#createNetworkNamespace

  1. func createNetworkNamespace(path string, osCreate bool) error {
  2.   // ********************************** NOTICE ********************************** //
  3.     if err := createNamespaceFile(path); err != nil {
  4.     // ********************************** NOTICE ********************************** //
  5.         return err
  6.     }
  8.     cmd := &exec.Cmd{
  9.         Path:   reexec.Self(),
  10.         Args:   append([]string{"netns-create"}, path),
  11.         Stdout: os.Stdout,
  12.         Stderr: os.Stderr,
  13.     }
  14.     if osCreate {
  15.         cmd.SysProcAttr = &syscall.SysProcAttr{}
  16.         cmd.SysProcAttr.Cloneflags = syscall.CLONE_NEWNET
  17.     }
  18.     if err := cmd.Run(); err != nil {
  19.         return fmt.Errorf("namespace creation reexec command failed: %v", err)
  20.     }
  22.     return nil
  23. }
  25. func createNamespaceFile(path string) (err error) {
  26.     var f *os.File
  28.     once.Do(createBasePath)
  29.     // Remove it from garbage collection list if present
  30.     removeFromGarbagePaths(path)
  32.     // If the path is there unmount it first
  33.     unmountNamespaceFile(path)
  35.     // wait for garbage collection to complete if it is in progress
  36.     // before trying to create the file.
  37.     gpmWg.Wait()
  39.     if f, err = os.Create(path); err == nil {
  40.         f.Close()
  41.     }
  43.     return err
  44. }
  46. func createBasePath() {
  47.     err := os.MkdirAll(basePath(), 0755)
  48.     if err != nil {
  49.         panic("Could not create net namespace path directory")
  50.     }
  52.     // Start the garbage collection go routine
  53.     go removeUnusedPaths()
  54. }
  56. // prefix是/var/run/docker
  57. func basePath() string {
  58.     return filepath.Join(prefix, "netns")
  59. }
  62. func init() {
  63.     reexec.Register("netns-create", reexecCreateNamespace)
  64. }
  66. func reexecCreateNamespace() {
  67.     if len(os.Args) < 2 {
  68.         logrus.Fatal("no namespace path provided")
  69.     }
  70.     if err := mountNetworkNamespace("/proc/self/ns/net", os.Args[1]); err != nil {
  71.         logrus.Fatal(err)
  72.     }
  73. }
  75. func mountNetworkNamespace(basePath string, lnPath string) error {
  76.     return syscall.Mount(basePath, lnPath, "bind", syscall.MS_BIND, "")
  77. }

——1.2.1.1.2.8) daemon/container_operations.go#updateSandboxNetworkSettings

  1. // updateSandboxNetworkSettings updates the sandbox ID and Key.
  2. func updateSandboxNetworkSettings(c *container.Container, sb libnetwork.Sandbox) error {
  3.     c.NetworkSettings.SandboxID = sb.ID()
  4.     c.NetworkSettings.SandboxKey = sb.Key()
  5.     return nil
  6. }

——1.2.1.1.2.9) daemon/network.go#buildJoinOptions

  1. // buildJoinOptions builds endpoint Join options from a given network.
  2. func buildJoinOptions(networkSettings *internalnetwork.Settings, n interface {
  3.     Name() string
  4. }) ([]libnetwork.EndpointOption, error) {
  5.     var joinOptions []libnetwork.EndpointOption
  6.     if epConfig, ok := networkSettings.Networks[n.Name()]; ok {
  7.         for _, str := range epConfig.Links {
  8.             name, alias, err := opts.ParseLink(str)
  9.             if err != nil {
  10.                 return nil, err
  11.             }
  12.             joinOptions = append(joinOptions, libnetwork.CreateOptionAlias(name, alias))
  13.         }
  14.         for k, v := range epConfig.DriverOpts {
  15.             joinOptions = append(joinOptions, libnetwork.EndpointOptionGeneric(options.Generic{k: v}))
  16.         }
  17.     }
  19.     return joinOptions, nil
  20. }

——1.2.1.1.2.10) libnetwork/endpoint.go#endpoint.Join

  1. func (ep *endpoint) Join(sbox Sandbox, options ...EndpointOption) error {
  2.     if sbox == nil {
  3.         return types.BadRequestErrorf("endpoint cannot be joined by nil container")
  4.     }
  6.     sb, ok := sbox.(*sandbox)
  7.     if !ok {
  8.         return types.BadRequestErrorf("not a valid Sandbox interface")
  9.     }
  11.     sb.joinLeaveStart()
  12.     defer sb.joinLeaveEnd()
  13.     // ********************************** NOTICE ********************************** //
  14.     return ep.sbJoin(sb, options...)
  15.   // ********************************** NOTICE ********************************** //
  16. }
  1. func (ep *endpoint) sbJoin(sb *sandbox, options ...EndpointOption) (err error) {
  2.     n, err := ep.getNetworkFromStore()
  3.     if err != nil {
  4.         return fmt.Errorf("failed to get network from store during join: %v", err)
  5.     }
  7.     ep, err = n.getEndpointFromStore(ep.ID())
  8.     if err != nil {
  9.         return fmt.Errorf("failed to get endpoint from store during join: %v", err)
  10.     }
  12.     ep.Lock()
  13.     if ep.sandboxID != "" {
  14.         ep.Unlock()
  15.         return types.ForbiddenErrorf("another container is attached to the same network endpoint")
  16.     }
  17.     ep.network = n
  18.     ep.sandboxID = sb.ID()
  19.     ep.joinInfo = &endpointJoinInfo{}
  20.     epid := ep.id
  21.     ep.Unlock()
  22.     defer func() {
  23.         if err != nil {
  24.             ep.Lock()
  25.             ep.sandboxID = ""
  26.             ep.Unlock()
  27.         }
  28.     }()
  30.     nid := n.ID()
  32.     ep.processOptions(options...)
  34.     d, err := n.driver(true)
  35.     if err != nil {
  36.         return fmt.Errorf("failed to get driver during join: %v", err)
  37.     }
  38.     // ********************************** NOTICE ********************************** //
  39.     err = d.Join(nid, epid, sb.Key(), ep, sb.Labels())
  40.   // ********************************** NOTICE ********************************** //
  41.     if err != nil {
  42.         return err
  43.     }
  44.     defer func() {
  45.         if err != nil {
  46.             if e := d.Leave(nid, epid); e != nil {
  47.                 logrus.Warnf("driver leave failed while rolling back join: %v", e)
  48.             }
  49.         }
  50.     }()
  52.     // Watch for service records
  53.     if !n.getController().isAgent() {
  54.         n.getController().watchSvcRecord(ep)
  55.     }
  57.     if doUpdateHostsFile(n, sb) {
  58.         address := ""
  59.         if ip := ep.getFirstInterfaceAddress(); ip != nil {
  60.             address = ip.String()
  61.         }
  62.         if err = sb.updateHostsFile(address); err != nil {
  63.             return err
  64.         }
  65.     }
  66.     if err = sb.updateDNS(n.enableIPv6); err != nil {
  67.         return err
  68.     }
  70.     // Current endpoint providing external connectivity for the sandbox
  71.     extEp := sb.getGatewayEndpoint()
  72.     // ********************************** NOTICE ********************************** //
  73.     sb.addEndpoint(ep)
  74.   // ********************************** NOTICE ********************************** //
  75.     defer func() {
  76.         if err != nil {
  77.             sb.removeEndpoint(ep)
  78.         }
  79.     }()
  80.     // ********************************** NOTICE ********************************** //
  81.     if err = sb.populateNetworkResources(ep); err != nil {
  82.     // ********************************** NOTICE ********************************** //
  83.         return err
  84.     }
  86.     if err = n.getController().updateToStore(ep); err != nil {
  87.         return err
  88.     }
  90.     if err = ep.addDriverInfoToCluster(); err != nil {
  91.         return err
  92.     }
  94.     defer func() {
  95.         if err != nil {
  96.             if e := ep.deleteDriverInfoFromCluster(); e != nil {
  97.                 logrus.Errorf("Could not delete endpoint state for endpoint %s from cluster on join failure: %v", ep.Name(), e)
  98.             }
  99.         }
  100.     }()
  102.     // Load balancing endpoints should never have a default gateway nor
  103.     // should they alter the status of a network's default gateway
  104.     if ep.loadBalancer && !sb.ingress {
  105.         return nil
  106.     }
  108.     if sb.needDefaultGW() && sb.getEndpointInGWNetwork() == nil {
  109.         return sb.setupDefaultGW()
  110.     }
  112.     moveExtConn := sb.getGatewayEndpoint() != extEp
  114.     if moveExtConn {
  115.         if extEp != nil {
  116.             logrus.Debugf("Revoking external connectivity on endpoint %s (%s)", extEp.Name(), extEp.ID())
  117.             extN, err := extEp.getNetworkFromStore()
  118.             if err != nil {
  119.                 return fmt.Errorf("failed to get network from store for revoking external connectivity during join: %v", err)
  120.             }
  121.             extD, err := extN.driver(true)
  122.             if err != nil {
  123.                 return fmt.Errorf("failed to get driver for revoking external connectivity during join: %v", err)
  124.             }
  125.             if err = extD.RevokeExternalConnectivity(extEp.network.ID(), extEp.ID()); err != nil {
  126.                 return types.InternalErrorf(
  127.                     "driver failed revoking external connectivity on endpoint %s (%s): %v",
  128.                     extEp.Name(), extEp.ID(), err)
  129.             }
  130.             defer func() {
  131.                 if err != nil {
  132.                     if e := extD.ProgramExternalConnectivity(extEp.network.ID(), extEp.ID(), sb.Labels()); e != nil {
  133.                         logrus.Warnf("Failed to roll-back external connectivity on endpoint %s (%s): %v",
  134.                             extEp.Name(), extEp.ID(), e)
  135.                     }
  136.                 }
  137.             }()
  138.         }
  139.         if !n.internal {
  140.             logrus.Debugf("Programming external connectivity on endpoint %s (%s)", ep.Name(), ep.ID())
  141.             if err = d.ProgramExternalConnectivity(n.ID(), ep.ID(), sb.Labels()); err != nil {
  142.                 return types.InternalErrorf(
  143.                     "driver failed programming external connectivity on endpoint %s (%s): %v",
  144.                     ep.Name(), ep.ID(), err)
  145.             }
  146.         }
  148.     }
  150.     if !sb.needDefaultGW() {
  151.         if e := sb.clearDefaultGW(); e != nil {
  152.             logrus.Warnf("Failure while disconnecting sandbox %s (%s) from gateway network: %v",
  153.                 sb.ID(), sb.ContainerID(), e)
  154.         }
  155.     }
  157.     return nil
  158. }

———1.2.1.1.2.10.1) libnetwork/drivers/bridge/bridge.go#driver#Join

  1. // Join method is invoked when a Sandbox is attached to an endpoint.
  2. func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, options map[string]interface{}) error {
  3.     defer osl.InitOSContext()()
  5.     network, err := d.getNetwork(nid)
  6.     if err != nil {
  7.         return err
  8.     }
  10.     endpoint, err := network.getEndpoint(eid)
  11.     if err != nil {
  12.         return err
  13.     }
  15.     if endpoint == nil {
  16.         return EndpointNotFoundError(eid)
  17.     }
  19.     endpoint.containerConfig, err = parseContainerOptions(options)
  20.     if err != nil {
  21.         return err
  22.     }
  24.     iNames := jinfo.InterfaceName()
  25.     containerVethPrefix := defaultContainerVethPrefix
  26.     if network.config.ContainerIfacePrefix != "" {
  27.         containerVethPrefix = network.config.ContainerIfacePrefix
  28.     }
  29.     err = iNames.SetNames(endpoint.srcName, containerVethPrefix)
  30.     if err != nil {
  31.         return err
  32.     }
  34.     err = jinfo.SetGateway(network.bridge.gatewayIPv4)
  35.     if err != nil {
  36.         return err
  37.     }
  39.     err = jinfo.SetGatewayIPv6(network.bridge.gatewayIPv6)
  40.     if err != nil {
  41.         return err
  42.     }
  44.     return nil
  45. }

———1.2.1.1.2.10.2) libnetwork/sandbox.go#sandbox.populateNetworkResources

  1. func (sb *sandbox) populateNetworkResources(ep *endpoint) error {
  2.     sb.Lock()
  3.     if sb.osSbox == nil {
  4.         sb.Unlock()
  5.         return nil
  6.     }
  7.     inDelete := sb.inDelete
  8.     sb.Unlock()
  10.     ep.Lock()
  11.     joinInfo := ep.joinInfo
  12.     i := ep.iface
  13.     lbModeIsDSR := ep.network.loadBalancerMode == loadBalancerModeDSR
  14.     ep.Unlock()
  16.     if ep.needResolver() {
  17.         sb.startResolver(false)
  18.     }
  20.     if i != nil && i.srcName != "" {
  21.         var ifaceOptions []osl.IfaceOption
  23.         ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().Address(i.addr), sb.osSbox.InterfaceOptions().Routes(i.routes))
  24.         if i.addrv6 != nil && i.addrv6.IP.To16() != nil {
  25.             ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().AddressIPv6(i.addrv6))
  26.         }
  27.         if len(i.llAddrs) != 0 {
  28.             ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().LinkLocalAddresses(i.llAddrs))
  29.         }
  30.         if i.mac != nil {
  31.             ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().MacAddress(i.mac))
  32.         }
  34.     // ********************************** NOTICE ********************************** //
  35.         if err := sb.osSbox.AddInterface(i.srcName, i.dstPrefix, ifaceOptions...); err != nil {
  36.       // ********************************** NOTICE ********************************** //
  37.             return fmt.Errorf("failed to add interface %s to sandbox: %v", i.srcName, err)
  38.         }
  40.         if len(ep.virtualIP) > 0 && lbModeIsDSR {
  41.             if sb.loadBalancerNID == "" {
  42.                 if err := sb.osSbox.DisableARPForVIP(i.srcName); err != nil {
  43.                     return fmt.Errorf("failed disable ARP for VIP: %v", err)
  44.                 }
  45.             }
  46.             ipNet := &net.IPNet{IP: ep.virtualIP, Mask: net.CIDRMask(32, 32)}
  47.             if err := sb.osSbox.AddAliasIP(sb.osSbox.GetLoopbackIfaceName(), ipNet); err != nil {
  48.                 return fmt.Errorf("failed to add virtual ip %v to loopback: %v", ipNet, err)
  49.             }
  50.         }
  51.     }
  53.     if joinInfo != nil {
  54.         // Set up non-interface routes.
  55.         for _, r := range joinInfo.StaticRoutes {
  56.             if err := sb.osSbox.AddStaticRoute(r); err != nil {
  57.                 return fmt.Errorf("failed to add static route %s: %v", r.Destination.String(), err)
  58.             }
  59.         }
  60.     }
  62.     if ep == sb.getGatewayEndpoint() {
  63.         if err := sb.updateGateway(ep); err != nil {
  64.             return err
  65.         }
  66.     }
  68.     // Make sure to add the endpoint to the populated endpoint set
  69.     // before populating loadbalancers.
  70.     sb.Lock()
  71.     sb.populatedEndpoints[ep.ID()] = struct{}{}
  72.     sb.Unlock()
  74.     // Populate load balancer only after updating all the other
  75.     // information including gateway and other routes so that
  76.     // loadbalancers are populated all the network state is in
  77.     // place in the sandbox.
  78.     sb.populateLoadBalancers(ep)
  80.     // Only update the store if we did not come here as part of
  81.     // sandbox delete. If we came here as part of delete then do
  82.     // not bother updating the store. The sandbox object will be
  83.     // deleted anyway
  84.     if !inDelete {
  85.         return sb.storeUpdate()
  86.     }
  88.     return nil
  89. }

————1.2.1.1.2.10.2.1) libnetwork/osl/interface_linux.go#networkNamespace#AddInterface

  1. func (n *networkNamespace) AddInterface(srcName, dstPrefix string, options ...IfaceOption) error {
  2.     i := &nwIface{srcName: srcName, dstName: dstPrefix, ns: n}
  3.     i.processInterfaceOptions(options...)
  5.     if i.master != "" {
  6.         i.dstMaster = n.findDst(i.master, true)
  7.         if i.dstMaster == "" {
  8.             return fmt.Errorf("could not find an appropriate master %q for %q",
  9.                 i.master, i.srcName)
  10.         }
  11.     }
  13.     n.Lock()
  14.     if n.isDefault {
  15.         i.dstName = i.srcName
  16.     } else {
  17.         i.dstName = fmt.Sprintf("%s%d", dstPrefix, n.nextIfIndex[dstPrefix])
  18.         n.nextIfIndex[dstPrefix]++
  19.     }
  21.     path := n.path
  22.     isDefault := n.isDefault
  23.     nlh := n.nlHandle
  24.     nlhHost := ns.NlHandle()
  25.     n.Unlock()
  27.     // If it is a bridge interface we have to create the bridge inside
  28.     // the namespace so don't try to lookup the interface using srcName
  29.     if i.bridge {
  30.         link := &netlink.Bridge{
  31.             LinkAttrs: netlink.LinkAttrs{
  32.                 Name: i.srcName,
  33.             },
  34.         }
  35.         if err := nlh.LinkAdd(link); err != nil {
  36.             return fmt.Errorf("failed to create bridge %q: %v", i.srcName, err)
  37.         }
  38.     } else {
  39.         // Find the network interface identified by the SrcName attribute.
  40.         iface, err := nlhHost.LinkByName(i.srcName)
  41.         if err != nil {
  42.             return fmt.Errorf("failed to get link by name %q: %v", i.srcName, err)
  43.         }
  45.         // Move the network interface to the destination
  46.         // namespace only if the namespace is not a default
  47.         // type
  48.         if !isDefault {
  49.             newNs, err := netns.GetFromPath(path)
  50.             if err != nil {
  51.                 return fmt.Errorf("failed get network namespace %q: %v", path, err)
  52.             }
  53.             defer newNs.Close()
  54.             if err := nlhHost.LinkSetNsFd(iface, int(newNs)); err != nil {
  55.                 return fmt.Errorf("failed to set namespace on link %q: %v", i.srcName, err)
  56.             }
  57.         }
  58.     }
  60.     // Find the network interface identified by the SrcName attribute.
  61.     iface, err := nlh.LinkByName(i.srcName)
  62.     if err != nil {
  63.         return fmt.Errorf("failed to get link by name %q: %v", i.srcName, err)
  64.     }
  66.     // Down the interface before configuring
  67.     if err := nlh.LinkSetDown(iface); err != nil {
  68.         return fmt.Errorf("failed to set link down: %v", err)
  69.     }
  71.     // Configure the interface now this is moved in the proper namespace.
  72.     if err := configureInterface(nlh, iface, i); err != nil {
  73.         // If configuring the device fails move it back to the host namespace
  74.         // and change the name back to the source name. This allows the caller
  75.         // to properly cleanup the interface. Its important especially for
  76.         // interfaces with global attributes, ex: vni id for vxlan interfaces.
  77.         if nerr := nlh.LinkSetName(iface, i.SrcName()); nerr != nil {
  78.             logrus.Errorf("renaming interface (%s->%s) failed, %v after config error %v", i.DstName(), i.SrcName(), nerr, err)
  79.         }
  80.         if nerr := nlh.LinkSetNsFd(iface, ns.ParseHandlerInt()); nerr != nil {
  81.             logrus.Errorf("moving interface %s to host ns failed, %v, after config error %v", i.SrcName(), nerr, err)
  82.         }
  83.         return err
  84.     }
  86.     // Up the interface.
  87.     cnt := 0
  88.     for err = nlh.LinkSetUp(iface); err != nil && cnt < 3; cnt++ {
  89.         logrus.Debugf("retrying link setup because of: %v", err)
  90.         time.Sleep(10 * time.Millisecond)
  91.         err = nlh.LinkSetUp(iface)
  92.     }
  93.     if err != nil {
  94.         return fmt.Errorf("failed to set link up: %v", err)
  95.     }
  97.     // Set the routes on the interface. This can only be done when the interface is up.
  98.     if err := setInterfaceRoutes(nlh, iface, i); err != nil {
  99.         return fmt.Errorf("error setting interface %q routes to %q: %v", iface.Attrs().Name, i.Routes(), err)
  100.     }
  102.     n.Lock()
  103.     n.iFaces = append(n.iFaces, i)
  104.     n.Unlock()
  106.     n.checkLoV6()
  108.     return nil
  109. }

—1.2.1.1.3) container/container.go#Container.WriteHostConfig

  1. // WriteHostConfig saves the host configuration on disk for the container,
  2. // and returns a deep copy of the saved object. Callers must hold a Container lock.
  3. func (container *Container) WriteHostConfig() (*containertypes.HostConfig, error) {
  4.     var (
  5.         buf      bytes.Buffer
  6.         deepCopy containertypes.HostConfig
  7.     )
  9.     pth, err := container.HostConfigPath()
  10.     if err != nil {
  11.         return nil, err
  12.     }
  14.     f, err := ioutils.NewAtomicFileWriter(pth, 0644)
  15.     if err != nil {
  16.         return nil, err
  17.     }
  18.     defer f.Close()
  20.     w := io.MultiWriter(&buf, f)
  21.     if err := json.NewEncoder(w).Encode(&container.HostConfig); err != nil {
  22.         return nil, err
  23.     }
  25.     if err := json.NewDecoder(&buf).Decode(&deepCopy); err != nil {
  26.         return nil, err
  27.     }
  28.     return &deepCopy, nil
  29. }

1.2.1.2) container/container_unix.go#Container.BuildHostnameFile

  1. // BuildHostnameFile writes the container's hostname file.
  2. func (container *Container) BuildHostnameFile() error {
  3.     hostnamePath, err := container.GetRootResourcePath("hostname")
  4.     if err != nil {
  5.         return err
  6.     }
  7.     container.HostnamePath = hostnamePath
  8.     return ioutil.WriteFile(container.HostnamePath, []byte(container.Config.Hostname+"\n"), 0644)
  9. }

1.2.2) daemon/oci_linux.go#Daemon.createSpec

  1. func (daemon *Daemon) createSpec(c *container.Container) (retSpec *specs.Spec, err error) {
  2.   // ********************************** NOTICE ********************************** //
  3.     s := oci.DefaultSpec()
  4.   // ********************************** NOTICE ********************************** //
  5.     if err := daemon.populateCommonSpec(&s, c); err != nil {
  6.     // ********************************** NOTICE ********************************** //
  7.         return nil, err
  8.     }
  10.     var cgroupsPath string
  11.     scopePrefix := "docker"
  12.     parent := "/docker"
  13.     useSystemd := UsingSystemd(daemon.configStore)
  14.     if useSystemd {
  15.         parent = "system.slice"
  16.     }
  18.     if c.HostConfig.CgroupParent != "" {
  19.         parent = c.HostConfig.CgroupParent
  20.     } else if daemon.configStore.CgroupParent != "" {
  21.         parent = daemon.configStore.CgroupParent
  22.     }
  24.     if useSystemd {
  25.         cgroupsPath = parent + ":" + scopePrefix + ":" + c.ID
  26.         logrus.Debugf("createSpec: cgroupsPath: %s", cgroupsPath)
  27.     } else {
  28.         cgroupsPath = filepath.Join(parent, c.ID)
  29.     }
  30.     s.Linux.CgroupsPath = cgroupsPath
  32.     if err := setResources(&s, c.HostConfig.Resources); err != nil {
  33.         return nil, fmt.Errorf("linux runtime spec resources: %v", err)
  34.     }
  35.     // We merge the sysctls injected above with the HostConfig (latter takes
  36.     // precedence for backwards-compatibility reasons).
  37.     for k, v := range c.HostConfig.Sysctls {
  38.         s.Linux.Sysctl[k] = v
  39.     }
  41.     p := s.Linux.CgroupsPath
  42.     if useSystemd {
  43.         initPath, err := cgroups.GetInitCgroup("cpu")
  44.         if err != nil {
  45.             return nil, err
  46.         }
  47.         _, err = cgroups.GetOwnCgroup("cpu")
  48.         if err != nil {
  49.             return nil, err
  50.         }
  51.         p = filepath.Join(initPath, s.Linux.CgroupsPath)
  52.     }
  54.     // Clean path to guard against things like ../../../BAD
  55.     parentPath := filepath.Dir(p)
  56.     if !filepath.IsAbs(parentPath) {
  57.         parentPath = filepath.Clean("/" + parentPath)
  58.     }
  60.     if err := daemon.initCgroupsPath(parentPath); err != nil {
  61.         return nil, fmt.Errorf("linux init cgroups path: %v", err)
  62.     }
  63.     if err := setDevices(&s, c); err != nil {
  64.         return nil, fmt.Errorf("linux runtime spec devices: %v", err)
  65.     }
  66.     if err := daemon.setRlimits(&s, c); err != nil {
  67.         return nil, fmt.Errorf("linux runtime spec rlimits: %v", err)
  68.     }
  69.     if err := setUser(&s, c); err != nil {
  70.         return nil, fmt.Errorf("linux spec user: %v", err)
  71.     }
  72.   // ********************************** NOTICE ********************************** //
  73.     if err := setNamespaces(daemon, &s, c); err != nil {
  74.     // ********************************** NOTICE ********************************** //
  75.         return nil, fmt.Errorf("linux spec namespaces: %v", err)
  76.     }
  77.     capabilities, err := caps.TweakCapabilities(oci.DefaultCapabilities(), c.HostConfig.CapAdd, c.HostConfig.CapDrop, c.HostConfig.Capabilities, c.HostConfig.Privileged)
  78.     if err != nil {
  79.         return nil, fmt.Errorf("linux spec capabilities: %v", err)
  80.     }
  81.     if err := oci.SetCapabilities(&s, capabilities); err != nil {
  82.         return nil, fmt.Errorf("linux spec capabilities: %v", err)
  83.     }
  84.     if err := setSeccomp(daemon, &s, c); err != nil {
  85.         return nil, fmt.Errorf("linux seccomp: %v", err)
  86.     }
  88.   // ********************************** NOTICE ********************************** //
  89.     if err := daemon.setupContainerMountsRoot(c); err != nil {
  90.     // ********************************** NOTICE ********************************** //
  91.         return nil, err
  92.     }
  94.     if err := daemon.setupIpcDirs(c); err != nil {
  95.         return nil, err
  96.     }
  98.     defer func() {
  99.         if err != nil {
  100.             daemon.cleanupSecretDir(c)
  101.         }
  102.     }()
  104.     if err := daemon.setupSecretDir(c); err != nil {
  105.         return nil, err
  106.     }
  108.   // ********************************** NOTICE ********************************** //
  109.     ms, err := daemon.setupMounts(c)
  110.   // ********************************** NOTICE ********************************** //
  111.     if err != nil {
  112.         return nil, err
  113.     }
  115.     if !c.HostConfig.IpcMode.IsPrivate() && !c.HostConfig.IpcMode.IsEmpty() {
  116.         ms = append(ms, c.IpcMounts()...)
  117.     }
  119.     tmpfsMounts, err := c.TmpfsMounts()
  120.     if err != nil {
  121.         return nil, err
  122.     }
  123.     ms = append(ms, tmpfsMounts...)
  125.     secretMounts, err := c.SecretMounts()
  126.     if err != nil {
  127.         return nil, err
  128.     }
  129.     ms = append(ms, secretMounts...)
  131.     sort.Sort(mounts(ms))
  132.     if err := setMounts(daemon, &s, c, ms); err != nil {
  133.         return nil, fmt.Errorf("linux mounts: %v", err)
  134.     }
  136.     for _, ns := range s.Linux.Namespaces {
  137.         if ns.Type == "network" && ns.Path == "" && !c.Config.NetworkDisabled {
  138.             target := filepath.Join("/proc", strconv.Itoa(os.Getpid()), "exe")
  139.             s.Hooks = &specs.Hooks{
  140.                 Prestart: []specs.Hook{{
  141.                     Path: target,
  142.                     Args: []string{"libnetwork-setkey", "-exec-root=" + daemon.configStore.GetExecRoot(), c.ID, daemon.netController.ID()},
  143.                 }},
  144.             }
  145.         }
  146.     }
  148.     if apparmor.IsEnabled() {
  149.         var appArmorProfile string
  150.         if c.AppArmorProfile != "" {
  151.             appArmorProfile = c.AppArmorProfile
  152.         } else if c.HostConfig.Privileged {
  153.             appArmorProfile = "unconfined"
  154.         } else {
  155.             appArmorProfile = "docker-default"
  156.         }
  158.         if appArmorProfile == "docker-default" {
  159.             // Unattended upgrades and other fun services can unload AppArmor
  160.             // profiles inadvertently. Since we cannot store our profile in
  161.             // /etc/apparmor.d, nor can we practically add other ways of
  162.             // telling the system to keep our profile loaded, in order to make
  163.             // sure that we keep the default profile enabled we dynamically
  164.             // reload it if necessary.
  165.             if err := ensureDefaultAppArmorProfile(); err != nil {
  166.                 return nil, err
  167.             }
  168.         }
  170.         s.Process.ApparmorProfile = appArmorProfile
  171.     }
  172.     s.Process.SelinuxLabel = c.GetProcessLabel()
  173.     s.Process.NoNewPrivileges = c.NoNewPrivileges
  174.     s.Process.OOMScoreAdj = &c.HostConfig.OomScoreAdj
  175.     s.Linux.MountLabel = c.MountLabel
  177.     // Set the masked and readonly paths with regard to the host config options if they are set.
  178.     if c.HostConfig.MaskedPaths != nil {
  179.         s.Linux.MaskedPaths = c.HostConfig.MaskedPaths
  180.     }
  181.     if c.HostConfig.ReadonlyPaths != nil {
  182.         s.Linux.ReadonlyPaths = c.HostConfig.ReadonlyPaths
  183.     }
  185.     if daemon.configStore.Rootless {
  186.         if err := specconv.ToRootless(&s); err != nil {
  187.             return nil, err
  188.         }
  189.     }
  190.     return &s, nil
  191. }

1.2.2.1) oci/defaults.go#DefaultSpec

  1. // DefaultSpec returns the default spec used by docker for the current Platform
  2. func DefaultSpec() specs.Spec {
  3.     return DefaultOSSpec(runtime.GOOS)
  4. }
  6. // DefaultLinuxSpec create a default spec for running Linux containers
  7. func DefaultLinuxSpec() specs.Spec {
  8.     s := specs.Spec{
  9.         Version: specs.Version,
  10.         Process: &specs.Process{
  11.             Capabilities: &specs.LinuxCapabilities{
  12.                 Bounding:    DefaultCapabilities(),
  13.                 Permitted:   DefaultCapabilities(),
  14.                 Inheritable: DefaultCapabilities(),
  15.                 Effective:   DefaultCapabilities(),
  16.             },
  17.         },
  18.         Root: &specs.Root{},
  19.     }
  20.     s.Mounts = []specs.Mount{
  21.         {
  22.             Destination: "/proc",
  23.             Type:        "proc",
  24.             Source:      "proc",
  25.             Options:     []string{"nosuid", "noexec", "nodev"},
  26.         },
  27.         {
  28.             Destination: "/dev",
  29.             Type:        "tmpfs",
  30.             Source:      "tmpfs",
  31.             Options:     []string{"nosuid", "strictatime", "mode=755", "size=65536k"},
  32.         },
  33.         {
  34.             Destination: "/dev/pts",
  35.             Type:        "devpts",
  36.             Source:      "devpts",
  37.             Options:     []string{"nosuid", "noexec", "newinstance", "ptmxmode=0666", "mode=0620", "gid=5"},
  38.         },
  39.         {
  40.             Destination: "/sys",
  41.             Type:        "sysfs",
  42.             Source:      "sysfs",
  43.             Options:     []string{"nosuid", "noexec", "nodev", "ro"},
  44.         },
  45.         {
  46.             Destination: "/sys/fs/cgroup",
  47.             Type:        "cgroup",
  48.             Source:      "cgroup",
  49.             Options:     []string{"ro", "nosuid", "noexec", "nodev"},
  50.         },
  51.         {
  52.             Destination: "/dev/mqueue",
  53.             Type:        "mqueue",
  54.             Source:      "mqueue",
  55.             Options:     []string{"nosuid", "noexec", "nodev"},
  56.         },
  57.         {
  58.             Destination: "/dev/shm",
  59.             Type:        "tmpfs",
  60.             Source:      "shm",
  61.             Options:     []string{"nosuid", "noexec", "nodev", "mode=1777"},
  62.         },
  63.     }
  65.     s.Linux = &specs.Linux{
  66.         MaskedPaths: []string{
  67.             "/proc/asound",
  68.             "/proc/acpi",
  69.             "/proc/kcore",
  70.             "/proc/keys",
  71.             "/proc/latency_stats",
  72.             "/proc/timer_list",
  73.             "/proc/timer_stats",
  74.             "/proc/sched_debug",
  75.             "/proc/scsi",
  76.             "/sys/firmware",
  77.         },
  78.         ReadonlyPaths: []string{
  79.             "/proc/bus",
  80.             "/proc/fs",
  81.             "/proc/irq",
  82.             "/proc/sys",
  83.             "/proc/sysrq-trigger",
  84.         },
  85.         Namespaces: []specs.LinuxNamespace{
  86.             {Type: "mount"},
  87.             {Type: "network"},
  88.             {Type: "uts"},
  89.             {Type: "pid"},
  90.             {Type: "ipc"},
  91.         },
  92.         // Devices implicitly contains the following devices:
  93.         // null, zero, full, random, urandom, tty, console, and ptmx.
  94.         // ptmx is a bind mount or symlink of the container's ptmx.
  95.         // See also: https://github.com/opencontainers/runtime-spec/blob/master/config-linux.md#default-devices
  96.         Devices: []specs.LinuxDevice{},
  97.         Resources: &specs.LinuxResources{
  98.             Devices: []specs.LinuxDeviceCgroup{
  99.                 {
  100.                     Allow:  false,
  101.                     Access: "rwm",
  102.                 },
  103.                 {
  104.                     Allow:  true,
  105.                     Type:   "c",
  106.                     Major:  iPtr(1),
  107.                     Minor:  iPtr(5),
  108.                     Access: "rwm",
  109.                 },
  110.                 {
  111.                     Allow:  true,
  112.                     Type:   "c",
  113.                     Major:  iPtr(1),
  114.                     Minor:  iPtr(3),
  115.                     Access: "rwm",
  116.                 },
  117.                 {
  118.                     Allow:  true,
  119.                     Type:   "c",
  120.                     Major:  iPtr(1),
  121.                     Minor:  iPtr(9),
  122.                     Access: "rwm",
  123.                 },
  124.                 {
  125.                     Allow:  true,
  126.                     Type:   "c",
  127.                     Major:  iPtr(1),
  128.                     Minor:  iPtr(8),
  129.                     Access: "rwm",
  130.                 },
  131.                 {
  132.                     Allow:  true,
  133.                     Type:   "c",
  134.                     Major:  iPtr(5),
  135.                     Minor:  iPtr(0),
  136.                     Access: "rwm",
  137.                 },
  138.                 {
  139.                     Allow:  true,
  140.                     Type:   "c",
  141.                     Major:  iPtr(5),
  142.                     Minor:  iPtr(1),
  143.                     Access: "rwm",
  144.                 },
  145.                 {
  146.                     Allow:  false,
  147.                     Type:   "c",
  148.                     Major:  iPtr(10),
  149.                     Minor:  iPtr(229),
  150.                     Access: "rwm",
  151.                 },
  152.             },
  153.         },
  154.     }
  156.     // For LCOW support, populate a blank Windows spec
  157.     if runtime.GOOS == "windows" {
  158.         s.Windows = &specs.Windows{}
  159.     }
  161.     return s
  162. }

1.2.2.2-Link实现) daemon/oci_linux.go#Daemon.populateCommonSpec

  1. func (daemon *Daemon) populateCommonSpec(s *specs.Spec, c *container.Container) error {
  2.     if c.BaseFS == nil {
  3.         return errors.New("populateCommonSpec: BaseFS of container " + c.ID + " is unexpectedly nil")
  4.     }
  5.   // ********************************** NOTICE ********************************** //
  6.     linkedEnv, err := daemon.setupLinkedContainers(c)
  7.   // ********************************** NOTICE ********************************** //
  8.     if err != nil {
  9.         return err
  10.     }
  11.     s.Root = &specs.Root{
  12.         Path:     c.BaseFS.Path(),
  13.         Readonly: c.HostConfig.ReadonlyRootfs,
  14.     }
  15.     if err := c.SetupWorkingDirectory(daemon.idMapping.RootPair()); err != nil {
  16.         return err
  17.     }
  18.     cwd := c.Config.WorkingDir
  19.     if len(cwd) == 0 {
  20.         cwd = "/"
  21.     }
  22.     s.Process.Args = append([]string{c.Path}, c.Args...)
  24.     // only add the custom init if it is specified and the container is running in its
  25.     // own private pid namespace.  It does not make sense to add if it is running in the
  26.     // host namespace or another container's pid namespace where we already have an init
  27.     if c.HostConfig.PidMode.IsPrivate() {
  28.         if (c.HostConfig.Init != nil && *c.HostConfig.Init) ||
  29.             (c.HostConfig.Init == nil && daemon.configStore.Init) {
  30.             s.Process.Args = append([]string{inContainerInitPath, "--", c.Path}, c.Args...)
  31.             path := daemon.configStore.InitPath
  32.             if path == "" {
  33.                 path, err = exec.LookPath(daemonconfig.DefaultInitBinary)
  34.                 if err != nil {
  35.                     return err
  36.                 }
  37.             }
  38.             s.Mounts = append(s.Mounts, specs.Mount{
  39.                 Destination: inContainerInitPath,
  40.                 Type:        "bind",
  41.                 Source:      path,
  42.                 Options:     []string{"bind", "ro"},
  43.             })
  44.         }
  45.     }
  46.     s.Process.Cwd = cwd
  47.   // ********************************** NOTICE ********************************** //
  48.     s.Process.Env = c.CreateDaemonEnvironment(c.Config.Tty, linkedEnv)
  49.   // ********************************** NOTICE ********************************** //
  50.     s.Process.Terminal = c.Config.Tty
  52.     s.Hostname = c.Config.Hostname
  53.     // There isn't a field in the OCI for the NIS domainname, but luckily there
  54.     // is a sysctl which has an identical effect to setdomainname(2) so there's
  55.     // no explicit need for runtime support.
  56.     s.Linux.Sysctl = make(map[string]string)
  57.     if c.Config.Domainname != "" {
  58.         s.Linux.Sysctl["kernel.domainname"] = c.Config.Domainname
  59.     }
  61.     return nil
  62. }

1.2.2.3) daemon/oci_linux.go#setNamespaces

  1. func setNamespaces(daemon *Daemon, s *specs.Spec, c *container.Container) error {
  2.     userNS := false
  3.     // user
  4.     if c.HostConfig.UsernsMode.IsPrivate() {
  5.         uidMap := daemon.idMapping.UIDs()
  6.         if uidMap != nil {
  7.             userNS = true
  8.             ns := specs.LinuxNamespace{Type: "user"}
  9.             setNamespace(s, ns)
  10.             s.Linux.UIDMappings = specMapping(uidMap)
  11.             s.Linux.GIDMappings = specMapping(daemon.idMapping.GIDs())
  12.         }
  13.     }
  14.     // network
  15.     if !c.Config.NetworkDisabled {
  16.         ns := specs.LinuxNamespace{Type: "network"}
  17.         parts := strings.SplitN(string(c.HostConfig.NetworkMode), ":", 2)
  18.         if parts[0] == "container" {
  19.             nc, err := daemon.getNetworkedContainer(c.ID, c.HostConfig.NetworkMode.ConnectedContainer())
  20.             if err != nil {
  21.                 return err
  22.             }
  23.             ns.Path = fmt.Sprintf("/proc/%d/ns/net", nc.State.GetPID())
  24.             if userNS {
  25.                 // to share a net namespace, they must also share a user namespace
  26.                 nsUser := specs.LinuxNamespace{Type: "user"}
  27.                 nsUser.Path = fmt.Sprintf("/proc/%d/ns/user", nc.State.GetPID())
  28.                 setNamespace(s, nsUser)
  29.             }
  30.         } else if c.HostConfig.NetworkMode.IsHost() {
  31.             ns.Path = c.NetworkSettings.SandboxKey
  32.         }
  33.     // ********************************** NOTICE ********************************** //
  34.         setNamespace(s, ns)
  35.     // ********************************** NOTICE ********************************** //
  36.     }
  38.     // ipc
  39.     ipcMode := c.HostConfig.IpcMode
  40.     switch {
  41.     case ipcMode.IsContainer():
  42.         ns := specs.LinuxNamespace{Type: "ipc"}
  43.         ic, err := daemon.getIpcContainer(ipcMode.Container())
  44.         if err != nil {
  45.             return err
  46.         }
  47.         ns.Path = fmt.Sprintf("/proc/%d/ns/ipc", ic.State.GetPID())
  48.         setNamespace(s, ns)
  49.         if userNS {
  50.             // to share an IPC namespace, they must also share a user namespace
  51.             nsUser := specs.LinuxNamespace{Type: "user"}
  52.             nsUser.Path = fmt.Sprintf("/proc/%d/ns/user", ic.State.GetPID())
  53.             setNamespace(s, nsUser)
  54.         }
  55.     case ipcMode.IsHost():
  56.         oci.RemoveNamespace(s, specs.LinuxNamespaceType("ipc"))
  57.     case ipcMode.IsEmpty():
  58.         // A container was created by an older version of the daemon.
  59.         // The default behavior used to be what is now called "shareable".
  60.         fallthrough
  61.     case ipcMode.IsPrivate(), ipcMode.IsShareable(), ipcMode.IsNone():
  62.         ns := specs.LinuxNamespace{Type: "ipc"}
  63.         setNamespace(s, ns)
  64.     default:
  65.         return fmt.Errorf("Invalid IPC mode: %v", ipcMode)
  66.     }
  68.     // pid
  69.     if c.HostConfig.PidMode.IsContainer() {
  70.         ns := specs.LinuxNamespace{Type: "pid"}
  71.         pc, err := daemon.getPidContainer(c)
  72.         if err != nil {
  73.             return err
  74.         }
  75.         ns.Path = fmt.Sprintf("/proc/%d/ns/pid", pc.State.GetPID())
  76.         setNamespace(s, ns)
  77.         if userNS {
  78.             // to share a PID namespace, they must also share a user namespace
  79.             nsUser := specs.LinuxNamespace{Type: "user"}
  80.             nsUser.Path = fmt.Sprintf("/proc/%d/ns/user", pc.State.GetPID())
  81.             setNamespace(s, nsUser)
  82.         }
  83.     } else if c.HostConfig.PidMode.IsHost() {
  84.         oci.RemoveNamespace(s, specs.LinuxNamespaceType("pid"))
  85.     } else {
  86.         ns := specs.LinuxNamespace{Type: "pid"}
  87.         setNamespace(s, ns)
  88.     }
  89.     // uts
  90.     if c.HostConfig.UTSMode.IsHost() {
  91.         oci.RemoveNamespace(s, specs.LinuxNamespaceType("uts"))
  92.         s.Hostname = ""
  93.     }
  95.     return nil
  96. }

1.2.2.3.1) daemon/oci_linux.go#setNamespace
  1. func setNamespace(s *specs.Spec, ns specs.LinuxNamespace) {
  2.     for i, n := range s.Linux.Namespaces {
  3.         if n.Type == ns.Type {
  4.             s.Linux.Namespaces[i] = ns
  5.             return
  6.         }
  7.     }
  8.     s.Linux.Namespaces = append(s.Linux.Namespaces, ns)
  9. }

1.2.2.4) daemon/container_operations_unix.go#setupContainerMountsRoot

  1. func (daemon *Daemon) setupContainerMountsRoot(c *container.Container) error {
  2.     // get the root mount path so we can make it unbindable
  3.     p, err := c.MountsResourcePath("")
  4.     if err != nil {
  5.         return err
  6.     }
  7.     return idtools.MkdirAllAndChown(p, 0700, daemon.idMapping.RootPair())
  8. }

1.2.2.5) daemon/volumes_unix.go#setupMounts

  1. // setupMounts iterates through each of the mount points for a container and
  2. // calls Setup() on each. It also looks to see if is a network mount such as
  3. // /etc/resolv.conf, and if it is not, appends it to the array of mounts.
  4. func (daemon *Daemon) setupMounts(c *container.Container) ([]container.Mount, error) {
  5.     var mounts []container.Mount
  6.     // TODO: tmpfs mounts should be part of Mountpoints
  7.     tmpfsMounts := make(map[string]bool)
  8.     tmpfsMountInfo, err := c.TmpfsMounts()
  9.     if err != nil {
  10.         return nil, err
  11.     }
  12.     for _, m := range tmpfsMountInfo {
  13.         tmpfsMounts[m.Destination] = true
  14.     }
  15.     for _, m := range c.MountPoints {
  16.         if tmpfsMounts[m.Destination] {
  17.             continue
  18.         }
  19.     // ********************************** NOTICE ********************************** //
  20.         if err := daemon.lazyInitializeVolume(c.ID, m); err != nil {
  21.       // ********************************** NOTICE ********************************** //
  22.             return nil, err
  23.         }
  24.         // If the daemon is being shutdown, we should not let a container start if it is trying to
  25.         // mount the socket the daemon is listening on. During daemon shutdown, the socket
  26.         // (/var/run/docker.sock by default) doesn't exist anymore causing the call to m.Setup to
  27.         // create at directory instead. This in turn will prevent the daemon to restart.
  28.         checkfunc := func(m *volumemounts.MountPoint) error {
  29.             if _, exist := daemon.hosts[m.Source]; exist && daemon.IsShuttingDown() {
  30.                 return fmt.Errorf("Could not mount %q to container while the daemon is shutting down", m.Source)
  31.             }
  32.             return nil
  33.         }
  35.         path, err := m.Setup(c.MountLabel, daemon.idMapping.RootPair(), checkfunc)
  36.         if err != nil {
  37.             return nil, err
  38.         }
  39.         if !c.TrySetNetworkMount(m.Destination, path) {
  40.             mnt := container.Mount{
  41.                 Source:      path,
  42.                 Destination: m.Destination,
  43.                 Writable:    m.RW,
  44.                 Propagation: string(m.Propagation),
  45.             }
  46.             if m.Spec.Type == mounttypes.TypeBind && m.Spec.BindOptions != nil {
  47.                 mnt.NonRecursive = m.Spec.BindOptions.NonRecursive
  48.             }
  49.             if m.Volume != nil {
  50.                 attributes := map[string]string{
  51.                     "driver":      m.Volume.DriverName(),
  52.                     "container":   c.ID,
  53.                     "destination": m.Destination,
  54.                     "read/write":  strconv.FormatBool(m.RW),
  55.                     "propagation": string(m.Propagation),
  56.                 }
  57.                 daemon.LogVolumeEvent(m.Volume.Name(), "mount", attributes)
  58.             }
  59.             mounts = append(mounts, mnt)
  60.         }
  61.     }
  63.     mounts = sortMounts(mounts)
  64.   // ********************************** NOTICE ********************************** //
  65.     netMounts := c.NetworkMounts()
  66.   // ********************************** NOTICE ********************************** //
  67.     // if we are going to mount any of the network files from container
  68.     // metadata, the ownership must be set properly for potential container
  69.     // remapped root (user namespaces)
  70.     rootIDs := daemon.idMapping.RootPair()
  71.     for _, mount := range netMounts {
  72.         // we should only modify ownership of network files within our own container
  73.         // metadata repository. If the user specifies a mount path external, it is
  74.         // up to the user to make sure the file has proper ownership for userns
  75.         if strings.Index(mount.Source, daemon.repository) == 0 {
  76.             if err := os.Chown(mount.Source, rootIDs.UID, rootIDs.GID); err != nil {
  77.                 return nil, err
  78.             }
  79.         }
  80.     }
  81.     return append(mounts, netMounts...), nil
  82. }

—1.2.2.5.1) container/container_unix.go#Container.NetworkMounts

  1. // NetworkMounts returns the list of network mounts.
  2. func (container *Container) NetworkMounts() []Mount {
  3.     var mounts []Mount
  4.     shared := container.HostConfig.NetworkMode.IsContainer()
  5.     parser := volumemounts.NewParser(container.OS)
  6.     if container.ResolvConfPath != "" {
  7.         if _, err := os.Stat(container.ResolvConfPath); err != nil {
  8.             logrus.Warnf("ResolvConfPath set to %q, but can't stat this filename (err = %v); skipping", container.ResolvConfPath, err)
  9.         } else {
  10.             writable := !container.HostConfig.ReadonlyRootfs
  11.             if m, exists := container.MountPoints["/etc/resolv.conf"]; exists {
  12.                 writable = m.RW
  13.             } else {
  14.                 label.Relabel(container.ResolvConfPath, container.MountLabel, shared)
  15.             }
  16.             mounts = append(mounts, Mount{
  17.                 Source:      container.ResolvConfPath,
  18.                 Destination: "/etc/resolv.conf",
  19.                 Writable:    writable,
  20.                 Propagation: string(parser.DefaultPropagationMode()),
  21.             })
  22.         }
  23.     }
  24.     if container.HostnamePath != "" {
  25.         if _, err := os.Stat(container.HostnamePath); err != nil {
  26.             logrus.Warnf("HostnamePath set to %q, but can't stat this filename (err = %v); skipping", container.HostnamePath, err)
  27.         } else {
  28.             writable := !container.HostConfig.ReadonlyRootfs
  29.             if m, exists := container.MountPoints["/etc/hostname"]; exists {
  30.                 writable = m.RW
  31.             } else {
  32.                 label.Relabel(container.HostnamePath, container.MountLabel, shared)
  33.             }
  34.             mounts = append(mounts, Mount{
  35.                 Source:      container.HostnamePath,
  36.                 Destination: "/etc/hostname",
  37.                 Writable:    writable,
  38.                 Propagation: string(parser.DefaultPropagationMode()),
  39.             })
  40.         }
  41.     }
  42.     if container.HostsPath != "" {
  43.         if _, err := os.Stat(container.HostsPath); err != nil {
  44.             logrus.Warnf("HostsPath set to %q, but can't stat this filename (err = %v); skipping", container.HostsPath, err)
  45.         } else {
  46.             writable := !container.HostConfig.ReadonlyRootfs
  47.             if m, exists := container.MountPoints["/etc/hosts"]; exists {
  48.                 writable = m.RW
  49.             } else {
  50.                 label.Relabel(container.HostsPath, container.MountLabel, shared)
  51.             }
  52.             mounts = append(mounts, Mount{
  53.                 Source:      container.HostsPath,
  54.                 Destination: "/etc/hosts",
  55.                 Writable:    writable,
  56.                 Propagation: string(parser.DefaultPropagationMode()),
  57.             })
  58.         }
  59.     }
  60.     return mounts
  61. }

1.2.3) libcontainerd/client_daemon.go#client.Create

  1. // Client provides access to containerd features.
  2. type Client interface {
  3.     Version(ctx context.Context) (containerd.Version, error)
  5.     Restore(ctx context.Context, containerID string, attachStdio StdioCallback) (alive bool, pid int, err error)
  7.     Create(ctx context.Context, containerID string, spec *specs.Spec, runtimeOptions interface{}) error
  8.     Start(ctx context.Context, containerID, checkpointDir string, withStdin bool, attachStdio StdioCallback) (pid int, err error)
  9.     SignalProcess(ctx context.Context, containerID, processID string, signal int) error
  10.     Exec(ctx context.Context, containerID, processID string, spec *specs.Process, withStdin bool, attachStdio StdioCallback) (int, error)
  11.     ResizeTerminal(ctx context.Context, containerID, processID string, width, height int) error
  12.     CloseStdin(ctx context.Context, containerID, processID string) error
  13.     Pause(ctx context.Context, containerID string) error
  14.     Resume(ctx context.Context, containerID string) error
  15.     Stats(ctx context.Context, containerID string) (*Stats, error)
  16.     ListPids(ctx context.Context, containerID string) ([]uint32, error)
  17.     Summary(ctx context.Context, containerID string) ([]Summary, error)
  18.     DeleteTask(ctx context.Context, containerID string) (uint32, time.Time, error)
  19.     Delete(ctx context.Context, containerID string) error
  20.     Status(ctx context.Context, containerID string) (Status, error)
  22.     UpdateResources(ctx context.Context, containerID string, resources *Resources) error
  23.     CreateCheckpoint(ctx context.Context, containerID, checkpointDir string, exit bool) error
  24. }
  26. type client struct {
  27.     sync.RWMutex // protects containers map
  29.     client   *containerd.Client
  30.     stateDir string
  31.     logger   *logrus.Entry
  32.     ns       string
  34.     backend    Backend
  35.     eventQ     queue
  36.   // 容器缓存，放在内存里的
  37.     containers map[string]*container
  38. }
  40. type container struct {
  41.     mu sync.Mutex
  43.     bundleDir string
  44.     ctr       containerd.Container
  45.     task      containerd.Task
  46.     execs     map[string]containerd.Process
  47.     oomKilled bool
  48. }
  1. func (c *client) Create(ctx context.Context, id string, ociSpec *specs.Spec, runtimeOptions interface{}) error {
  2.     if ctr := c.getContainer(id); ctr != nil {
  3.         return errors.WithStack(newConflictError("id already in use"))
  4.     }
  5.     // ********************************** NOTICE ********************************** //
  6.     bdir, err := prepareBundleDir(filepath.Join(c.stateDir, id), ociSpec)
  7.   // ********************************** NOTICE ********************************** //
  8.     if err != nil {
  9.         return errdefs.System(errors.Wrap(err, "prepare bundle dir failed"))
  10.     }
  12.     c.logger.WithField("bundle", bdir).WithField("root", ociSpec.Root.Path).Debug("bundle dir created")
  14.   // ********************************** NOTICE ********************************** //
  15.     cdCtr, err := c.client.NewContainer(ctx, id,
  16.         containerd.WithSpec(ociSpec),
  17.         // TODO(mlaventure): when containerd support lcow, revisit runtime value
  18.         containerd.WithRuntime(fmt.Sprintf("io.containerd.runtime.v1.%s", runtime.GOOS), runtimeOptions))
  19.   // ********************************** NOTICE ********************************** //
  20.     if err != nil {
  21.         return wrapError(err)
  22.     }
  24.     c.Lock()
  25.     c.containers[id] = &container{
  26.         bundleDir: b
  27.     dir,
  28.         ctr:       cdCtr,
  29.     }
  30.     c.Unlock()
  31.     return nil
  32. }

1.2.4) libcontainerd/client_daemon.go#client.Start

  1. // Start create and start a task for the specified containerd id
  2. func (c *client) Start(ctx context.Context, id, checkpointDir string, withStdin bool, attachStdio StdioCallback) (int, error) {
  3.     ctr := c.getContainer(id)
  4.     if ctr == nil {
  5.         return -1, errors.WithStack(newNotFoundError("no such container"))
  6.     }
  7.   // ********************************** NOTICE ********************************** //
  8.     if t := ctr.getTask(); t != nil {
  9.     // ********************************** NOTICE ********************************** //
  10.         return -1, errors.WithStack(newConflictError("container already started"))
  11.     }
  13.     var (
  14.         cp             *types.Descriptor
  15.         t              containerd.Task
  16.         rio            cio.IO
  17.         err            error
  18.         stdinCloseSync = make(chan struct{})
  19.     )
  21.     if checkpointDir != "" {
  22.         // write checkpoint to the content store
  23.         tar := archive.Diff(ctx, "", checkpointDir)
  24.         cp, err = c.writeContent(ctx, images.MediaTypeContainerd1Checkpoint, checkpointDir, tar)
  25.         // remove the checkpoint when we're done
  26.         defer func() {
  27.             if cp != nil {
  28.                 err := c.client.ContentStore().Delete(context.Background(), cp.Digest)
  29.                 if err != nil {
  30.                     c.logger.WithError(err).WithFields(logrus.Fields{
  31.                         "ref":    checkpointDir,
  32.                         "digest": cp.Digest,
  33.                     }).Warnf("failed to delete temporary checkpoint entry")
  34.                 }
  35.             }
  36.         }()
  37.         if err := tar.Close(); err != nil {
  38.             return -1, errors.Wrap(err, "failed to close checkpoint tar stream")
  39.         }
  40.         if err != nil {
  41.             return -1, errors.Wrapf(err, "failed to upload checkpoint to containerd")
  42.         }
  43.     }
  45.     spec, err := ctr.ctr.Spec(ctx)
  46.     if err != nil {
  47.         return -1, errors.Wrap(err, "failed to retrieve spec")
  48.     }
  49.     uid, gid := getSpecUser(spec)
  50.   // ********************************** NOTICE ********************************** //
  51.     t, err = ctr.ctr.NewTask(ctx,
  52.         func(id string) (cio.IO, error) {
  53.             fifos := newFIFOSet(ctr.bundleDir, InitProcessName, withStdin, spec.Process.Terminal)
  55.             rio, err = c.createIO(fifos, id, InitProcessName, stdinCloseSync, attachStdio)
  56.             return rio, err
  57.         },
  58.         func(_ context.Context, _ *containerd.Client, info *containerd.TaskInfo) error {
  59.             info.Checkpoint = cp
  60.             info.Options = &runctypes.CreateOptions{
  61.                 IoUid:       uint32(uid),
  62.                 IoGid:       uint32(gid),
  63.                 NoPivotRoot: os.Getenv("DOCKER_RAMDISK") != "",
  64.             }
  65.             return nil
  66.         })
  67.   // ********************************** NOTICE ********************************** //
  68.     if err != nil {
  69.         close(stdinCloseSync)
  70.         if rio != nil {
  71.             rio.Cancel()
  72.             rio.Close()
  73.         }
  74.         return -1, wrapError(err)
  75.     }
  77.     ctr.setTask(t)
  79.     // Signal c.createIO that it can call CloseIO
  80.     close(stdinCloseSync)
  82.   // ********************************** NOTICE ********************************** //
  83.     if err := t.Start(ctx); err != nil {
  84.     // ********************************** NOTICE ********************************** //
  85.         if _, err := t.Delete(ctx); err != nil {
  86.             c.logger.WithError(err).WithField("container", id).
  87.                 Error("failed to delete task after fail start")
  88.         }
  89.         ctr.setTask(nil)
  90.         return -1, wrapError(err)
  91.     }
  93.     return int(t.Pid()), nil
  94. }