cmd/dockerd/docker.go

func main() {
  // ********************************** NOTICE ********************************** //
    if reexec.Init() {
    // ********************************** NOTICE ********************************** //
        return
    }
    // initial log formatting; this setting is updated after the daemon configuration is loaded.
    logrus.SetFormatter(&logrus.TextFormatter{
        TimestampFormat: jsonmessage.RFC3339NanoFixed,
        FullTimestamp:   true,
    })
    // Set terminal emulation based on platform as required.
    _, stdout, stderr := term.StdStreams()
    // @jhowardmsft - maybe there is a historic reason why on non-Windows, stderr is used
    // here. However, on Windows it makes no sense and there is no need.
    if runtime.GOOS == "windows" {
        logrus.SetOutput(stdout)
    } else {
        logrus.SetOutput(stderr)
    }
    onError := func(err error) {
        fmt.Fprintf(stderr, "%s\n", err)
        os.Exit(1)
    }
    // ********************************** NOTICE ********************************** //
    cmd, err := newDaemonCommand()
  // ********************************** NOTICE ********************************** //
    if err != nil {
        onError(err)
    }
    cmd.SetOutput(stdout)
    if err := cmd.Execute(); err != nil {
        onError(err)
    }
}

1) pkg/reexec/reexec.go#Init

var registeredInitializers = make(map[string]func())
// Register adds an initialization func under the specified name
func Register(name string, initializer func()) {
    if _, exists := registeredInitializers[name]; exists {
        panic(fmt.Sprintf("reexec func already registered under name %q", name))
    }
    registeredInitializers[name] = initializer
}
// Init is called as the first part of the exec process and returns true if an
// initialization function was called.
func Init() bool {
    initializer, exists := registeredInitializers[os.Args[0]]
    if exists {
        initializer()
        return true
    }
    return false
}

2) cmd/dockerd/docker.go#newDaemonCommand

func newDaemonCommand() (*cobra.Command, error) {
    opts := newDaemonOptions(config.New())
    cmd := &cobra.Command{
        Use:           "dockerd [OPTIONS]",
        Short:         "A self-sufficient runtime for containers.",
        SilenceUsage:  true,
        SilenceErrors: true,
        Args:          cli.NoArgs,
        RunE: func(cmd *cobra.Command, args []string) error {
            opts.flags = cmd.Flags()
      // ********************************** NOTICE ********************************** //
            return runDaemon(opts)
      // ********************************** NOTICE ********************************** //
        },
        DisableFlagsInUseLine: true,
        Version:               fmt.Sprintf("%s, build %s", dockerversion.Version, dockerversion.GitCommit),
    }
    cli.SetupRootCommand(cmd)
    flags := cmd.Flags()
    flags.BoolP("version", "v", false, "Print version information and quit")
    defaultDaemonConfigFile, err := getDefaultDaemonConfigFile()
    if err != nil {
        return nil, err
    }
    flags.StringVar(&opts.configFile, "config-file", defaultDaemonConfigFile, "Daemon configuration file")
    opts.InstallFlags(flags)
    if err := installConfigFlags(opts.daemonConfig, flags); err != nil {
        return nil, err
    }
    installServiceFlags(flags)
    return cmd, nil
}

2.1) cmd/dockerd/docker_unix.go#runDaemon

func runDaemon(opts *daemonOptions) error {
    daemonCli := NewDaemonCli()
    return daemonCli.start(opts)
}
// NewDaemonCli returns a daemon CLI
func NewDaemonCli() *DaemonCli {
    return &DaemonCli{}
}

2.1.1) cmd/dockerd/daemon.go#DaemonCli.start

func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
    stopc := make(chan bool)
    defer close(stopc)
    // warn from uuid package when running the daemon
    uuid.Loggerf = logrus.Warnf
    opts.SetDefaultOptions(opts.flags)
    if cli.Config, err = loadDaemonCliConfig(opts); err != nil {
        return err
    }
    if err := configureDaemonLogs(cli.Config); err != nil {
        return err
    }
    cli.configFile = &opts.configFile
    cli.flags = opts.flags
    if cli.Config.Debug {
        debug.Enable()
    }
    if cli.Config.Experimental {
        logrus.Warn("Running experimental build")
        if cli.Config.IsRootless() {
            logrus.Warn("Running in rootless mode. Cgroups, AppArmor, and CRIU are disabled.")
        }
    } else {
        if cli.Config.IsRootless() {
            return fmt.Errorf("rootless mode is supported only when running in experimental mode")
        }
    }
    // return human-friendly error before creating files
    if runtime.GOOS == "linux" && os.Geteuid() != 0 {
        return fmt.Errorf("dockerd needs to be started with root. To see how to run dockerd in rootless mode with unprivileged user, see the documentation")
    }
    system.InitLCOW(cli.Config.Experimental)
    if err := setDefaultUmask(); err != nil {
        return err
    }
    // Create the daemon root before we create ANY other files (PID, or migrate keys)
    // to ensure the appropriate ACL is set (particularly relevant on Windows)
  // ********************************** NOTICE ********************************** //
    if err := daemon.CreateDaemonRoot(cli.Config); err != nil {
    // ********************************** NOTICE ********************************** //
        return err
    }
    if err := system.MkdirAll(cli.Config.ExecRoot, 0700, ""); err != nil {
        return err
    }
    potentiallyUnderRuntimeDir := []string{cli.Config.ExecRoot}
    if cli.Pidfile != "" {
        pf, err := pidfile.New(cli.Pidfile)
        if err != nil {
            return errors.Wrap(err, "failed to start daemon")
        }
        potentiallyUnderRuntimeDir = append(potentiallyUnderRuntimeDir, cli.Pidfile)
        defer func() {
            if err := pf.Remove(); err != nil {
                logrus.Error(err)
            }
        }()
    }
    // Set sticky bit if XDG_RUNTIME_DIR is set && the file is actually under XDG_RUNTIME_DIR
    if _, err := homedir.StickRuntimeDirContents(potentiallyUnderRuntimeDir); err != nil {
        // StickRuntimeDirContents returns nil error if XDG_RUNTIME_DIR is just unset
        logrus.WithError(err).Warn("cannot set sticky bit on files under XDG_RUNTIME_DIR")
    }
    // ********************************** NOTICE ********************************** //
    serverConfig, err := newAPIServerConfig(cli)
  // ********************************** NOTICE ********************************** //
    if err != nil {
        return errors.Wrap(err, "failed to create API server")
    }
  // ********************************** NOTICE ********************************** //
    cli.api = apiserver.New(serverConfig)
  // ********************************** NOTICE ********************************** //
    hosts, err := loadListeners(cli, serverConfig)
    if err != nil {
        return errors.Wrap(err, "failed to load listeners")
    }
    ctx, cancel := context.WithCancel(context.Background())
    if cli.Config.ContainerdAddr == "" && runtime.GOOS != "windows" {
        systemContainerdAddr, ok, err := systemContainerdRunning(cli.Config.IsRootless())
        if err != nil {
            cancel()
            return errors.Wrap(err, "could not determine whether the system containerd is running")
        }
        if !ok {
            opts, err := cli.getContainerdDaemonOpts()
            if err != nil {
                cancel()
                return errors.Wrap(err, "failed to generate containerd options")
            }
            r, err := supervisor.Start(ctx, filepath.Join(cli.Config.Root, "containerd"), filepath.Join(cli.Config.ExecRoot, "containerd"), opts...)
            if err != nil {
                cancel()
                return errors.Wrap(err, "failed to start containerd")
            }
            cli.Config.ContainerdAddr = r.Address()
            // Try to wait for containerd to shutdown
            defer r.WaitTimeout(10 * time.Second)
        } else {
            cli.Config.ContainerdAddr = systemContainerdAddr
        }
    }
    defer cancel()
    signal.Trap(func() {
        cli.stop()
        <-stopc // wait for daemonCli.start() to return
    }, logrus.StandardLogger())
    // Notify that the API is active, but before daemon is set up.
    preNotifySystem()
    pluginStore := plugin.NewStore()
  // ********************************** NOTICE ********************************** //
    if err := cli.initMiddlewares(cli.api, serverConfig, pluginStore); err != nil {
    // ********************************** NOTICE ********************************** //
        logrus.Fatalf("Error creating middlewares: %v", err)
    }
  // ********************************** NOTICE ********************************** //
    d, err := daemon.NewDaemon(ctx, cli.Config, pluginStore)
  // ********************************** NOTICE ********************************** //
    if err != nil {
        return errors.Wrap(err, "failed to start daemon")
    }
    d.StoreHosts(hosts)
    // validate after NewDaemon has restored enabled plugins. Don't change order.
    if err := validateAuthzPlugins(cli.Config.AuthorizationPlugins, pluginStore); err != nil {
        return errors.Wrap(err, "failed to validate authorization plugin")
    }
    // TODO: move into startMetricsServer()
    if cli.Config.MetricsAddress != "" {
        if !d.HasExperimental() {
            return errors.Wrap(err, "metrics-addr is only supported when experimental is enabled")
        }
        if err := startMetricsServer(cli.Config.MetricsAddress); err != nil {
            return err
        }
    }
    c, err := createAndStartCluster(cli, d)
    if err != nil {
        logrus.Fatalf("Error starting cluster component: %v", err)
    }
    // Restart all autostart containers which has a swarm endpoint
    // and is not yet running now that we have successfully
    // initialized the cluster.
    d.RestartSwarmContainers()
    logrus.Info("Daemon has completed initialization")
    cli.d = d
    routerOptions, err := newRouterOptions(cli.Config, d)
    if err != nil {
        return err
    }
    routerOptions.api = cli.api
    routerOptions.cluster = c
  // ********************************** NOTICE ********************************** //
    initRouter(routerOptions)
  // ********************************** NOTICE ********************************** //
    go d.ProcessClusterNotifications(ctx, c.GetWatchStream())
    cli.setupConfigReloadTrap()
    // The serve API routine never exits unless an error occurs
    // We need to start it as a goroutine and wait on it so
    // daemon doesn't exit
    serveAPIWait := make(chan error)
    go cli.api.Wait(serveAPIWait)
    // after the daemon is done setting up we can notify systemd api
    notifySystem()
    // Daemon is fully initialized and handling API traffic
    // Wait for serve API to complete
  // ********************************** NOTICE ********************************** //
    errAPI := <-serveAPIWait
  // ********************************** NOTICE ********************************** //
    c.Cleanup()
    shutdownDaemon(d)
    // Stop notification processing and any background processes
    cancel()
    if errAPI != nil {
        return errors.Wrap(errAPI, "shutting down due to ServeAPI error")
    }
    return nil
}

2.1.1.1) api/server/server.go#New（返回Server）

// New returns a new instance of the server based on the specified configuration.
// It allocates resources which will be needed for ServeAPI(ports, unix-sockets).
func New(cfg *Config) *Server {
    return &Server{
        cfg: cfg,
    }
}
// Server contains instance details for the server
type Server struct {
    cfg           *Config
    servers       []*HTTPServer
    routers       []router.Router
    routerSwapper *routerSwapper
    middlewares   []middleware.Middleware
}

2.1.1.2) daemon/daemon.go#NewDaemon

// NewDaemon sets up everything for the daemon to be able to service
// requests from the webserver.
func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.Store) (daemon *Daemon, err error) {
    setDefaultMtu(config)
    registryService, err := registry.NewService(config.ServiceOptions)
    if err != nil {
        return nil, err
    }
    // Ensure that we have a correct root key limit for launching containers.
    if err := ModifyRootKeyLimit(); err != nil {
        logrus.Warnf("unable to modify root key limit, number of containers could be limited by this quota: %v", err)
    }
    // Ensure we have compatible and valid configuration options
    if err := verifyDaemonSettings(config); err != nil {
        return nil, err
    }
    // Do we have a disabled network?
    config.DisableBridge = isBridgeNetworkDisabled(config)
    // Setup the resolv.conf
    setupResolvConf(config)
    // Verify the platform is supported as a daemon
    if !platformSupported {
        return nil, errSystemNotSupported
    }
    // Validate platform-specific requirements
    if err := checkSystem(); err != nil {
        return nil, err
    }
    idMapping, err := setupRemappedRoot(config)
    if err != nil {
        return nil, err
    }
    rootIDs := idMapping.RootPair()
    if err := setupDaemonProcess(config); err != nil {
        return nil, err
    }
    // set up the tmpDir to use a canonical path
    tmp, err := prepareTempDir(config.Root, rootIDs)
    if err != nil {
        return nil, fmt.Errorf("Unable to get the TempDir under %s: %s", config.Root, err)
    }
    realTmp, err := getRealPath(tmp)
    if err != nil {
        return nil, fmt.Errorf("Unable to get the full path to the TempDir (%s): %s", tmp, err)
    }
    if runtime.GOOS == "windows" {
        if _, err := os.Stat(realTmp); err != nil && os.IsNotExist(err) {
            if err := system.MkdirAll(realTmp, 0700, ""); err != nil {
                return nil, fmt.Errorf("Unable to create the TempDir (%s): %s", realTmp, err)
            }
        }
        os.Setenv("TEMP", realTmp)
        os.Setenv("TMP", realTmp)
    } else {
        os.Setenv("TMPDIR", realTmp)
    }
  // ********************************** NOTICE ********************************** //
    d := &Daemon{
        configStore: config,
        PluginStore: pluginStore,
        startupDone: make(chan struct{}),
    }
  // ********************************** NOTICE ********************************** //
    // Ensure the daemon is properly shutdown if there is a failure during
    // initialization
    defer func() {
        if err != nil {
            if err := d.Shutdown(); err != nil {
                logrus.Error(err)
            }
        }
    }()
    if err := d.setGenericResources(config); err != nil {
        return nil, err
    }
    // set up SIGUSR1 handler on Unix-like systems, or a Win32 global event
    // on Windows to dump Go routine stacks
    stackDumpDir := config.Root
    if execRoot := config.GetExecRoot(); execRoot != "" {
        stackDumpDir = execRoot
    }
    d.setupDumpStackTrap(stackDumpDir)
    if err := d.setupSeccompProfile(); err != nil {
        return nil, err
    }
    // Set the default isolation mode (only applicable on Windows)
    if err := d.setDefaultIsolation(); err != nil {
        return nil, fmt.Errorf("error setting default isolation mode: %v", err)
    }
    if err := configureMaxThreads(config); err != nil {
        logrus.Warnf("Failed to configure golang's threads limit: %v", err)
    }
    // ensureDefaultAppArmorProfile does nothing if apparmor is disabled
    if err := ensureDefaultAppArmorProfile(); err != nil {
        logrus.Errorf(err.Error())
    }
    daemonRepo := filepath.Join(config.Root, "containers")
    if err := idtools.MkdirAllAndChown(daemonRepo, 0700, rootIDs); err != nil {
        return nil, err
    }
    // Create the directory where we'll store the runtime scripts (i.e. in
    // order to support runtimeArgs)
    daemonRuntimes := filepath.Join(config.Root, "runtimes")
    if err := system.MkdirAll(daemonRuntimes, 0700, ""); err != nil {
        return nil, err
    }
    if err := d.loadRuntimes(); err != nil {
        return nil, err
    }
    if runtime.GOOS == "windows" {
        if err := system.MkdirAll(filepath.Join(config.Root, "credentialspecs"), 0, ""); err != nil {
            return nil, err
        }
    }
    // On Windows we don't support the environment variable, or a user supplied graphdriver
    // as Windows has no choice in terms of which graphdrivers to use. It's a case of
    // running Windows containers on Windows - windowsfilter, running Linux containers on Windows,
    // lcow. Unix platforms however run a single graphdriver for all containers, and it can
    // be set through an environment variable, a daemon start parameter, or chosen through
    // initialization of the layerstore through driver priority order for example.
    d.graphDrivers = make(map[string]string)
    layerStores := make(map[string]layer.Store)
    if runtime.GOOS == "windows" {
        d.graphDrivers[runtime.GOOS] = "windowsfilter"
        if system.LCOWSupported() {
            d.graphDrivers["linux"] = "lcow"
        }
    } else {
        driverName := os.Getenv("DOCKER_DRIVER")
        if driverName == "" {
            driverName = config.GraphDriver
        } else {
            logrus.Infof("Setting the storage driver from the $DOCKER_DRIVER environment variable (%s)", driverName)
        }
        d.graphDrivers[runtime.GOOS] = driverName // May still be empty. Layerstore init determines instead.
    }
    d.RegistryService = registryService
    logger.RegisterPluginGetter(d.PluginStore)
    metricsSockPath, err := d.listenMetricsSock()
    if err != nil {
        return nil, err
    }
    registerMetricsPluginCallback(d.PluginStore, metricsSockPath)
    gopts := []grpc.DialOption{
        grpc.WithInsecure(),
        grpc.WithBackoffMaxDelay(3 * time.Second),
        grpc.WithDialer(dialer.Dialer),
        // TODO(stevvooe): We may need to allow configuration of this on the client.
        grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)),
        grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)),
    }
    if config.ContainerdAddr != "" {
        d.containerdCli, err = containerd.New(config.ContainerdAddr, containerd.WithDefaultNamespace(ContainersNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second))
        if err != nil {
            return nil, errors.Wrapf(err, "failed to dial %q", config.ContainerdAddr)
        }
    }
  // ********************************** NOTICE ********************************** //
    createPluginExec := func(m *plugin.Manager) (plugin.Executor, error) {
        var pluginCli *containerd.Client
        // Windows is not currently using containerd, keep the
        // client as nil
        if config.ContainerdAddr != "" {
            pluginCli, err = containerd.New(config.ContainerdAddr, containerd.WithDefaultNamespace(pluginexec.PluginNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second))
            if err != nil {
                return nil, errors.Wrapf(err, "failed to dial %q", config.ContainerdAddr)
            }
        }
        return pluginexec.New(ctx, getPluginExecRoot(config.Root), pluginCli, m)
    }
  // ********************************** NOTICE ********************************** //
    // Plugin system initialization should happen before restore. Do not change order.
    d.pluginManager, err = plugin.NewManager(plugin.ManagerConfig{
        Root:               filepath.Join(config.Root, "plugins"),
        ExecRoot:           getPluginExecRoot(config.Root),
        Store:              d.PluginStore,
        CreateExecutor:     createPluginExec,
        RegistryService:    registryService,
        LiveRestoreEnabled: config.LiveRestoreEnabled,
        LogPluginEvent:     d.LogPluginEvent, // todo: make private
        AuthzMiddleware:    config.AuthzMiddleware,
    })
    if err != nil {
        return nil, errors.Wrap(err, "couldn't create plugin manager")
    }
    if err := d.setupDefaultLogConfig(); err != nil {
        return nil, err
    }
    for operatingSystem, gd := range d.graphDrivers {
        layerStores[operatingSystem], err = layer.NewStoreFromOptions(layer.StoreOptions{
            Root:                      config.Root,
            MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s", "layerdb"),
            GraphDriver:               gd,
            GraphDriverOptions:        config.GraphOptions,
            IDMapping:                 idMapping,
            PluginGetter:              d.PluginStore,
            ExperimentalEnabled:       config.Experimental,
            OS:                        operatingSystem,
        })
        if err != nil {
            return nil, err
        }
    }
    // As layerstore initialization may set the driver
    for os := range d.graphDrivers {
        d.graphDrivers[os] = layerStores[os].DriverName()
    }
    // Configure and validate the kernels security support. Note this is a Linux/FreeBSD
    // operation only, so it is safe to pass *just* the runtime OS graphdriver.
    if err := configureKernelSecuritySupport(config, d.graphDrivers[runtime.GOOS]); err != nil {
        return nil, err
    }
    imageRoot := filepath.Join(config.Root, "image", d.graphDrivers[runtime.GOOS])
    ifs, err := image.NewFSStoreBackend(filepath.Join(imageRoot, "imagedb"))
    if err != nil {
        return nil, err
    }
    lgrMap := make(map[string]image.LayerGetReleaser)
    for os, ls := range layerStores {
        lgrMap[os] = ls
    }
  // ********************************** NOTICE ********************************** //
    imageStore, err := image.NewImageStore(ifs, lgrMap)
  // ********************************** NOTICE ********************************** //
    if err != nil {
        return nil, err
    }
    // ********************************** NOTICE ********************************** //
    d.volumes, err = volumesservice.NewVolumeService(config.Root, d.PluginStore, rootIDs, d)
  // ********************************** NOTICE ********************************** //
    if err != nil {
        return nil, err
    }
    trustKey, err := loadOrCreateTrustKey(config.TrustKeyPath)
    if err != nil {
        return nil, err
    }
    trustDir := filepath.Join(config.Root, "trust")
    if err := system.MkdirAll(trustDir, 0700, ""); err != nil {
        return nil, err
    }
    // We have a single tag/reference store for the daemon globally. However, it's
    // stored under the graphdriver. On host platforms which only support a single
    // container OS, but multiple selectable graphdrivers, this means depending on which
    // graphdriver is chosen, the global reference store is under there. For
    // platforms which support multiple container operating systems, this is slightly
    // more problematic as where does the global ref store get located? Fortunately,
    // for Windows, which is currently the only daemon supporting multiple container
    // operating systems, the list of graphdrivers available isn't user configurable.
    // For backwards compatibility, we just put it under the windowsfilter
    // directory regardless.
    refStoreLocation := filepath.Join(imageRoot, `repositories.json`)
    rs, err := refstore.NewReferenceStore(refStoreLocation)
    if err != nil {
        return nil, fmt.Errorf("Couldn't create reference store repository: %s", err)
    }
    distributionMetadataStore, err := dmetadata.NewFSMetadataStore(filepath.Join(imageRoot, "distribution"))
    if err != nil {
        return nil, err
    }
    // Discovery is only enabled when the daemon is launched with an address to advertise.  When
    // initialized, the daemon is registered and we can store the discovery backend as it's read-only
    if err := d.initDiscovery(config); err != nil {
        return nil, err
    }
    sysInfo := sysinfo.New(false)
    // Check if Devices cgroup is mounted, it is hard requirement for container security,
    // on Linux.
    if runtime.GOOS == "linux" && !sysInfo.CgroupDevicesEnabled {
        return nil, errors.New("Devices cgroup isn't mounted")
    }
    d.ID = trustKey.PublicKey().KeyID()
    d.repository = daemonRepo
    d.containers = container.NewMemoryStore()
    if d.containersReplica, err = container.NewViewDB(); err != nil {
        return nil, err
    }
    d.execCommands = exec.NewStore()
    d.idIndex = truncindex.NewTruncIndex([]string{})
    d.statsCollector = d.newStatsCollector(1 * time.Second)
    d.EventsService = events.New()
    d.root = config.Root
    d.idMapping = idMapping
    d.seccompEnabled = sysInfo.Seccomp
    d.apparmorEnabled = sysInfo.AppArmor
    d.linkIndex = newLinkIndex()
    // TODO: imageStore, distributionMetadataStore, and ReferenceStore are only
    // used above to run migration. They could be initialized in ImageService
    // if migration is called from daemon/images. layerStore might move as well.
  // ********************************** NOTICE ********************************** //
    d.imageService = images.NewImageService(images.ImageServiceConfig{
        ContainerStore:            d.containers,
        DistributionMetadataStore: distributionMetadataStore,
        EventsService:             d.EventsService,
        ImageStore:                imageStore,
        LayerStores:               layerStores,
        MaxConcurrentDownloads:    *config.MaxConcurrentDownloads,
        MaxConcurrentUploads:      *config.MaxConcurrentUploads,
        ReferenceStore:            rs,
        RegistryService:           registryService,
        TrustKey:                  trustKey,
    })
  // ********************************** NOTICE ********************************** //
    go d.execCommandGC()
  // ********************************** NOTICE ********************************** //
    d.containerd, err = libcontainerd.NewClient(ctx, d.containerdCli, filepath.Join(config.ExecRoot, "containerd"), ContainersNamespace, d)
  // ********************************** NOTICE ********************************** //
    if err != nil {
        return nil, err
    }
    // ********************************** NOTICE ********************************** //
    if err := d.restore(); err != nil {
    // ********************************** NOTICE ********************************** //
        return nil, err
    }
    close(d.startupDone)
    // FIXME: this method never returns an error
    info, _ := d.SystemInfo()
    engineInfo.WithValues(
        dockerversion.Version,
        dockerversion.GitCommit,
        info.Architecture,
        info.Driver,
        info.KernelVersion,
        info.OperatingSystem,
        info.OSType,
        info.ID,
    ).Set(1)
    engineCpus.Set(float64(info.NCPU))
    engineMemory.Set(float64(info.MemTotal))
    gd := ""
    for os, driver := range d.graphDrivers {
        if len(gd) > 0 {
            gd += ", "
        }
        gd += driver
        if len(d.graphDrivers) > 1 {
            gd = fmt.Sprintf("%s (%s)", gd, os)
        }
    }
    logrus.WithFields(logrus.Fields{
        "version":        dockerversion.Version,
        "commit":         dockerversion.GitCommit,
        "graphdriver(s)": gd,
    }).Info("Docker daemon")
    return d, nil
}

2.1.1.2.1) daemon/daemon.go#Daemon.restore

func (daemon *Daemon) restore() error {
    var mapLock sync.Mutex
    containers := make(map[string]*container.Container)
    logrus.Info("Loading containers: start.")
  // ********************************** NOTICE ********************************** //
    dir, err := ioutil.ReadDir(daemon.repository)
  // ********************************** NOTICE ********************************** //
    if err != nil {
        return err
    }
    // parallelLimit is the maximum number of parallel startup jobs that we
    // allow (this is the limited used for all startup semaphores). The multipler
    // (128) was chosen after some fairly significant benchmarking -- don't change
    // it unless you've tested it significantly (this value is adjusted if
    // RLIMIT_NOFILE is small to avoid EMFILE).
    parallelLimit := adjustParallelLimit(len(dir), 128*runtime.NumCPU())
    // Re-used for all parallel startup jobs.
    var group sync.WaitGroup
    sem := semaphore.NewWeighted(int64(parallelLimit))
    for _, v := range dir {
        group.Add(1)
        go func(id string) {
            defer group.Done()
            _ = sem.Acquire(context.Background(), 1)
            defer sem.Release(1)
            // ********************************** NOTICE ********************************** //
            container, err := daemon.load(id)
      // ********************************** NOTICE ********************************** //
            if err != nil {
                logrus.Errorf("Failed to load container %v: %v", id, err)
                return
            }
            if !system.IsOSSupported(container.OS) {
                logrus.Errorf("Failed to load container %v: %s (%q)", id, system.ErrNotSupportedOperatingSystem, container.OS)
                return
            }
            // Ignore the container if it does not support the current driver being used by the graph
            currentDriverForContainerOS := daemon.graphDrivers[container.OS]
            if (container.Driver == "" && currentDriverForContainerOS == "aufs") || container.Driver == currentDriverForContainerOS {
                rwlayer, err := daemon.imageService.GetLayerByID(container.ID, container.OS)
                if err != nil {
                    logrus.Errorf("Failed to load container mount %v: %v", id, err)
                    return
                }
                container.RWLayer = rwlayer
                logrus.Debugf("Loaded container %v, isRunning: %v", container.ID, container.IsRunning())
                mapLock.Lock()
                containers[container.ID] = container
                mapLock.Unlock()
            } else {
                logrus.Debugf("Cannot load container %s because it was created with another graph driver.", container.ID)
            }
        }(v.Name())
    }
    group.Wait()
    removeContainers := make(map[string]*container.Container)
    restartContainers := make(map[*container.Container]chan struct{})
    activeSandboxes := make(map[string]interface{})
    for _, c := range containers {
        group.Add(1)
        go func(c *container.Container) {
            defer group.Done()
            _ = sem.Acquire(context.Background(), 1)
            defer sem.Release(1)
            if err := daemon.registerName(c); err != nil {
                logrus.Errorf("Failed to register container name %s: %s", c.ID, err)
                mapLock.Lock()
                delete(containers, c.ID)
                mapLock.Unlock()
                return
            }
            if err := daemon.Register(c); err != nil {
                logrus.Errorf("Failed to register container %s: %s", c.ID, err)
                mapLock.Lock()
                delete(containers, c.ID)
                mapLock.Unlock()
                return
            }
            // The LogConfig.Type is empty if the container was created before docker 1.12 with default log driver.
            // We should rewrite it to use the daemon defaults.
            // Fixes https://github.com/docker/docker/issues/22536
            if c.HostConfig.LogConfig.Type == "" {
                if err := daemon.mergeAndVerifyLogConfig(&c.HostConfig.LogConfig); err != nil {
                    logrus.Errorf("Failed to verify log config for container %s: %q", c.ID, err)
                }
            }
        }(c)
    }
    group.Wait()
    for _, c := range containers {
        group.Add(1)
        go func(c *container.Container) {
            defer group.Done()
            _ = sem.Acquire(context.Background(), 1)
            defer sem.Release(1)
            daemon.backportMountSpec(c)
            if err := daemon.checkpointAndSave(c); err != nil {
                logrus.WithError(err).WithField("container", c.ID).Error("error saving backported mountspec to disk")
            }
            daemon.setStateCounter(c)
            logrus.WithFields(logrus.Fields{
                "container": c.ID,
                "running":   c.IsRunning(),
                "paused":    c.IsPaused(),
            }).Debug("restoring container")
            var (
                err      error
                alive    bool
                ec       uint32
                exitedAt time.Time
            )
            alive, _, err = daemon.containerd.Restore(context.Background(), c.ID, c.InitializeStdio)
            if err != nil && !errdefs.IsNotFound(err) {
                logrus.Errorf("Failed to restore container %s with containerd: %s", c.ID, err)
                return
            }
            if !alive {
                ec, exitedAt, err = daemon.containerd.DeleteTask(context.Background(), c.ID)
                if err != nil && !errdefs.IsNotFound(err) {
                    logrus.WithError(err).Errorf("Failed to delete container %s from containerd", c.ID)
                    return
                }
            } else if !daemon.configStore.LiveRestoreEnabled {
                if err := daemon.kill(c, c.StopSignal()); err != nil && !errdefs.IsNotFound(err) {
                    logrus.WithError(err).WithField("container", c.ID).Error("error shutting down container")
                    return
                }
            }
            if c.IsRunning() || c.IsPaused() {
                c.RestartManager().Cancel() // manually start containers because some need to wait for swarm networking
                if c.IsPaused() && alive {
                    s, err := daemon.containerd.Status(context.Background(), c.ID)
                    if err != nil {
                        logrus.WithError(err).WithField("container", c.ID).
                            Errorf("Failed to get container status")
                    } else {
                        logrus.WithField("container", c.ID).WithField("state", s).
                            Info("restored container paused")
                        switch s {
                        case libcontainerd.StatusPaused, libcontainerd.StatusPausing:
                            // nothing to do
                        case libcontainerd.StatusStopped:
                            alive = false
                        case libcontainerd.StatusUnknown:
                            logrus.WithField("container", c.ID).
                                Error("Unknown status for container during restore")
                        default:
                            // running
                            c.Lock()
                            c.Paused = false
                            daemon.setStateCounter(c)
                            if err := c.CheckpointTo(daemon.containersReplica); err != nil {
                                logrus.WithError(err).WithField("container", c.ID).
                                    Error("Failed to update stopped container state")
                            }
                            c.Unlock()
                        }
                    }
                }
                if !alive {
                    c.Lock()
                    c.SetStopped(&container.ExitStatus{ExitCode: int(ec), ExitedAt: exitedAt})
                    daemon.Cleanup(c)
                    if err := c.CheckpointTo(daemon.containersReplica); err != nil {
                        logrus.Errorf("Failed to update stopped container %s state: %v", c.ID, err)
                    }
                    c.Unlock()
                }
                // we call Mount and then Unmount to get BaseFs of the container
                if err := daemon.Mount(c); err != nil {
                    // The mount is unlikely to fail. However, in case mount fails
                    // the container should be allowed to restore here. Some functionalities
                    // (like docker exec -u user) might be missing but container is able to be
                    // stopped/restarted/removed.
                    // See #29365 for related information.
                    // The error is only logged here.
                    logrus.Warnf("Failed to mount container on getting BaseFs path %v: %v", c.ID, err)
                } else {
                    if err := daemon.Unmount(c); err != nil {
                        logrus.Warnf("Failed to umount container on getting BaseFs path %v: %v", c.ID, err)
                    }
                }
                c.ResetRestartManager(false)
                if !c.HostConfig.NetworkMode.IsContainer() && c.IsRunning() {
                    options, err := daemon.buildSandboxOptions(c)
                    if err != nil {
                        logrus.Warnf("Failed build sandbox option to restore container %s: %v", c.ID, err)
                    }
                    mapLock.Lock()
                    activeSandboxes[c.NetworkSettings.SandboxID] = options
                    mapLock.Unlock()
                }
            }
            // get list of containers we need to restart
            // Do not autostart containers which
            // has endpoints in a swarm scope
            // network yet since the cluster is
            // not initialized yet. We will start
            // it after the cluster is
            // initialized.
            if daemon.configStore.AutoRestart && c.ShouldRestart() && !c.NetworkSettings.HasSwarmEndpoint && c.HasBeenStartedBefore {
                mapLock.Lock()
                restartContainers[c] = make(chan struct{})
                mapLock.Unlock()
            } else if c.HostConfig != nil && c.HostConfig.AutoRemove {
                mapLock.Lock()
                removeContainers[c.ID] = c
                mapLock.Unlock()
            }
            c.Lock()
            if c.RemovalInProgress {
                // We probably crashed in the middle of a removal, reset
                // the flag.
                //
                // We DO NOT remove the container here as we do not
                // know if the user had requested for either the
                // associated volumes, network links or both to also
                // be removed. So we put the container in the "dead"
                // state and leave further processing up to them.
                logrus.Debugf("Resetting RemovalInProgress flag from %v", c.ID)
                c.RemovalInProgress = false
                c.Dead = true
                if err := c.CheckpointTo(daemon.containersReplica); err != nil {
                    logrus.Errorf("Failed to update RemovalInProgress container %s state: %v", c.ID, err)
                }
            }
            c.Unlock()
        }(c)
    }
    group.Wait()
    // ********************************** NOTICE ********************************** //
    daemon.netController, err = daemon.initNetworkController(daemon.configStore, activeSandboxes)
  // ********************************** NOTICE ********************************** //
    if err != nil {
        return fmt.Errorf("Error initializing network controller: %v", err)
    }
    // Now that all the containers are registered, register the links
    for _, c := range containers {
        group.Add(1)
        go func(c *container.Container) {
            _ = sem.Acquire(context.Background(), 1)
            if err := daemon.registerLinks(c, c.HostConfig); err != nil {
                logrus.Errorf("failed to register link for container %s: %v", c.ID, err)
            }
            sem.Release(1)
            group.Done()
        }(c)
    }
    group.Wait()
    for c, notifier := range restartContainers {
        group.Add(1)
        go func(c *container.Container, chNotify chan struct{}) {
            _ = sem.Acquire(context.Background(), 1)
            logrus.Debugf("Starting container %s", c.ID)
            // ignore errors here as this is a best effort to wait for children to be
            //   running before we try to start the container
            children := daemon.children(c)
            timeout := time.After(5 * time.Second)
            for _, child := range children {
                if notifier, exists := restartContainers[child]; exists {
                    select {
                    case <-notifier:
                    case <-timeout:
                    }
                }
            }
            // Make sure networks are available before starting
            daemon.waitForNetworks(c)
            if err := daemon.containerStart(c, "", "", true); err != nil {
                logrus.Errorf("Failed to start container %s: %s", c.ID, err)
            }
            close(chNotify)
            sem.Release(1)
            group.Done()
        }(c, notifier)
    }
    group.Wait()
    for id := range removeContainers {
        group.Add(1)
        go func(cid string) {
            _ = sem.Acquire(context.Background(), 1)
            if err := daemon.ContainerRm(cid, &types.ContainerRmConfig{ForceRemove: true, RemoveVolume: true}); err != nil {
                logrus.Errorf("Failed to remove container %s: %s", cid, err)
            }
            sem.Release(1)
            group.Done()
        }(id)
    }
    group.Wait()
    // any containers that were started above would already have had this done,
    // however we need to now prepare the mountpoints for the rest of the containers as well.
    // This shouldn't cause any issue running on the containers that already had this run.
    // This must be run after any containers with a restart policy so that containerized plugins
    // can have a chance to be running before we try to initialize them.
    for _, c := range containers {
        // if the container has restart policy, do not
        // prepare the mountpoints since it has been done on restarting.
        // This is to speed up the daemon start when a restart container
        // has a volume and the volume driver is not available.
        if _, ok := restartContainers[c]; ok {
            continue
        } else if _, ok := removeContainers[c.ID]; ok {
            // container is automatically removed, skip it.
            continue
        }
        group.Add(1)
        go func(c *container.Container) {
            _ = sem.Acquire(context.Background(), 1)
            if err := daemon.prepareMountPoints(c); err != nil {
                logrus.Error(err)
            }
            sem.Release(1)
            group.Done()
        }(c)
    }
    group.Wait()
    logrus.Info("Loading containers: done.")
    return nil
}