From 90d4b8ec8008ea59a69d17948202321d5d453eb8 Mon Sep 17 00:00:00 2001 From: zhangwei Date: Sat, 22 Oct 2022 20:36:16 +0800 Subject: [PATCH] CAPK supports to install docker --- api/v1beta1/containermanager_types.go | 12 +- api/v1beta1/kkmachine_webhook.go | 3 + ...tructure.cluster.x-k8s.io_kkinstances.yaml | 4 + ...structure.cluster.x-k8s.io_kkmachines.yaml | 4 + ...e.cluster.x-k8s.io_kkmachinetemplates.yaml | 4 + pkg/service/containermanager/containerd.go | 2 +- pkg/service/containermanager/docker.go | 331 ++++++++++++++++++ pkg/service/containermanager/service.go | 2 + .../templates/cri-docker.service | 39 +++ .../templates/cri-docker.socket | 12 + .../containermanager/templates/daemon.json | 13 + .../containermanager/templates/docker.service | 47 +++ .../file/checksum/internal_checksum.go | 9 + pkg/service/operation/file/docker.go | 48 ++- test/e2e/config/e2e_conf.yaml | 3 + .../v1beta1/bases/cluster-with-kcp.yaml | 7 +- .../v1beta1/bases/md.yaml | 4 +- 17 files changed, 536 insertions(+), 8 deletions(-) create mode 100644 pkg/service/containermanager/docker.go create mode 100644 pkg/service/containermanager/templates/cri-docker.service create mode 100644 pkg/service/containermanager/templates/cri-docker.socket create mode 100644 pkg/service/containermanager/templates/daemon.json create mode 100644 pkg/service/containermanager/templates/docker.service diff --git a/api/v1beta1/containermanager_types.go b/api/v1beta1/containermanager_types.go index 6f8cb40b..50ed23e8 100644 --- a/api/v1beta1/containermanager_types.go +++ b/api/v1beta1/containermanager_types.go @@ -18,9 +18,10 @@ package v1beta1 // Default values. const ( - DockerType = "docker" - DefaultDockerVersion = "20.10.8" - DefaultDockerCRISocket = "" + DockerType = "docker" + DefaultDockerVersion = "20.10.8" + DefaultCRIDockerdVersion = "0.2.6" + DefaultDockerCRISocket = "unix:///run/cri-dockerd.sock" ContainerdType = "containerd" DefaultContainerdVersion = "1.6.4" @@ -39,4 +40,9 @@ type ContainerManager struct { // Version defines the version of ContainerManager. Version string `json:"version,omitempty"` + + // CRIDockerdVersion defines the version of cri-dockerd, available only when Type is docker. + // https://github.com/Mirantis/cri-dockerd + // +optional + CRIDockerdVersion string `json:"criDockerdVersion,omitempty"` } diff --git a/api/v1beta1/kkmachine_webhook.go b/api/v1beta1/kkmachine_webhook.go index 6e6d5336..9f9da625 100644 --- a/api/v1beta1/kkmachine_webhook.go +++ b/api/v1beta1/kkmachine_webhook.go @@ -64,6 +64,9 @@ func defaultContainerManager(spec *KKMachineSpec) { if spec.ContainerManager.Version == "" { spec.ContainerManager.Version = DefaultDockerVersion } + if spec.ContainerManager.CRIDockerdVersion == "" { + spec.ContainerManager.CRIDockerdVersion = DefaultCRIDockerdVersion + } spec.ContainerManager.CRISocket = DefaultDockerCRISocket } } diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkinstances.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_kkinstances.yaml index ae4c8d58..10820115 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkinstances.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_kkinstances.yaml @@ -85,6 +85,10 @@ spec: description: ContainerManager is the container manager config of this machine. properties: + criDockerdVersion: + description: CRIDockerdVersion defines the version of cri-dockerd, + available only when Type is docker. https://github.com/Mirantis/cri-dockerd + type: string criSocket: description: CRISocket is used to connect an existing CRIClient. type: string diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_kkmachines.yaml index b25ff368..f58d0916 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkmachines.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_kkmachines.yaml @@ -60,6 +60,10 @@ spec: description: ContainerManager is the container manager config of this machine. properties: + criDockerdVersion: + description: CRIDockerdVersion defines the version of cri-dockerd, + available only when Type is docker. https://github.com/Mirantis/cri-dockerd + type: string criSocket: description: CRISocket is used to connect an existing CRIClient. type: string diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkmachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_kkmachinetemplates.yaml index 78323305..3c850494 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkmachinetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_kkmachinetemplates.yaml @@ -68,6 +68,10 @@ spec: description: ContainerManager is the container manager config of this machine. properties: + criDockerdVersion: + description: CRIDockerdVersion defines the version of + cri-dockerd, available only when Type is docker. https://github.com/Mirantis/cri-dockerd + type: string criSocket: description: CRISocket is used to connect an existing CRIClient. diff --git a/pkg/service/containermanager/containerd.go b/pkg/service/containermanager/containerd.go index ef6f6a8e..1a10aef6 100644 --- a/pkg/service/containermanager/containerd.go +++ b/pkg/service/containermanager/containerd.go @@ -72,7 +72,7 @@ func (s *ContainerdService) getContainerdService(sshClient ssh.Interface, versio } func (s *ContainerdService) getCrictlService(sshClient ssh.Interface, version, arch string) (operation.Binary, error) { - if s.containerdFactory != nil { + if s.crictlFactory != nil { return s.crictlFactory(sshClient, version, arch) } return file.NewCrictl(sshClient, s.scope.RootFs(), version, arch) diff --git a/pkg/service/containermanager/docker.go b/pkg/service/containermanager/docker.go new file mode 100644 index 00000000..55d18d7e --- /dev/null +++ b/pkg/service/containermanager/docker.go @@ -0,0 +1,331 @@ +/* + Copyright 2022 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package containermanager + +import ( + "fmt" + "path/filepath" + "strings" + "text/template" + "time" + + infrav1 "github.com/kubesphere/kubekey/api/v1beta1" + "github.com/kubesphere/kubekey/pkg/clients/ssh" + "github.com/kubesphere/kubekey/pkg/scope" + "github.com/kubesphere/kubekey/pkg/service/operation" + "github.com/kubesphere/kubekey/pkg/service/operation/directory" + "github.com/kubesphere/kubekey/pkg/service/operation/file" + "github.com/kubesphere/kubekey/pkg/service/util" +) + +// DockerService is a ContainerManager service implementation for docker. +type DockerService struct { + sshClient ssh.Interface + scope scope.KKInstanceScope + instanceScope *scope.InstanceScope + + templateFactory func(sshClient ssh.Interface, template *template.Template, data file.Data, dst string) (operation.Template, error) + dockerFactory func(sshClient ssh.Interface, version, arch string) (operation.Binary, error) + criDockerdFactory func(sshClient ssh.Interface, version, arch string) (operation.Binary, error) + crictlFactory func(sshClient ssh.Interface, version, arch string) (operation.Binary, error) +} + +// NewDockerService returns a new DockerService given the remote instance container manager client. +func NewDockerService(sshClient ssh.Interface, scope scope.KKInstanceScope, instanceScope *scope.InstanceScope) *DockerService { + return &DockerService{ + sshClient: sshClient, + scope: scope, + instanceScope: instanceScope, + } +} + +func (d *DockerService) getTemplateService(template *template.Template, data file.Data, dst string) (operation.Template, error) { + if d.templateFactory != nil { + return d.templateFactory(d.sshClient, template, data, dst) + } + return file.NewTemplate(d.sshClient, d.scope.RootFs(), template, data, dst) +} + +func (d *DockerService) getDockerService(sshClient ssh.Interface, version, arch string) (operation.Binary, error) { + if d.dockerFactory != nil { + return d.dockerFactory(sshClient, version, arch) + } + return file.NewDocker(sshClient, d.scope.RootFs(), version, arch) +} + +func (d *DockerService) getCRIDockerdService(sshClient ssh.Interface, version, arch string) (operation.Binary, error) { + if d.criDockerdFactory != nil { + return d.criDockerdFactory(sshClient, version, arch) + } + return file.NewCRIDockerd(sshClient, d.scope.RootFs(), version, arch) +} + +func (d *DockerService) getCrictlService(sshClient ssh.Interface, version, arch string) (operation.Binary, error) { + if d.crictlFactory != nil { + return d.crictlFactory(sshClient, version, arch) + } + return file.NewCrictl(sshClient, d.scope.RootFs(), version, arch) +} + +// Type returns the type docker of the container manager. +func (d *DockerService) Type() string { + return file.DockerID +} + +// Version returns the version of the container manager. +func (d *DockerService) Version() string { + return d.instanceScope.KKInstance.Spec.ContainerManager.Version +} + +// CRIDockerdVersion returns the version of the cri-dockerd. +func (d *DockerService) CRIDockerdVersion() string { + return d.instanceScope.KKInstance.Spec.ContainerManager.CRIDockerdVersion +} + +// IsExist returns true if the container manager is installed. +func (d *DockerService) IsExist() bool { + res, err := d.sshClient.SudoCmd( + "if [ -z $(which docker) ] || [ ! -e /run/docker.sock ]; " + + "then echo 'not exist'; " + + "fi") + if err != nil { + return false + } + if strings.Contains(res, "not exist") { + return false + } + return true +} + +// Get gets the binary of docker and related components and copy them to the remote instance. +func (d *DockerService) Get(timeout time.Duration) error { + docker, err := d.getDockerService(d.sshClient, d.Version(), d.instanceScope.Arch()) + if err != nil { + return err + } + criDockerd, err := d.getCRIDockerdService(d.sshClient, d.CRIDockerdVersion(), d.instanceScope.Arch()) + if err != nil { + return err + } + crictl, err := d.getCrictlService(d.sshClient, getFirstMinorVersion(d.instanceScope.KubernetesVersion()), d.instanceScope.Arch()) + if err != nil { + return err + } + + binaries := []operation.Binary{ + docker, + criDockerd, + crictl, + } + + zone := d.scope.ComponentZone() + host := d.scope.ComponentHost() + overrideMap := make(map[string]infrav1.Override) + for _, o := range d.scope.ComponentOverrides() { + overrideMap[o.ID+o.Version+o.Arch] = o + } + + for _, b := range binaries { + d.instanceScope.V(4).Info("download binary", "binary", b.Name(), "version", b.Version(), + "url", b.URL().String()) + + override := overrideMap[b.ID()+b.Version()+b.Arch()] + if err := util.DownloadAndCopy(b, zone, host, override.Path, override.URL, override.Checksum.Value, timeout); err != nil { + return err + } + } + + // /usr/local + dir := filepath.Dir(filepath.Dir(docker.RemotePath())) + if _, err = d.sshClient.SudoCmdf("tar Cxzvf %s %s && mv %s/docker/* %s", dir, docker.RemotePath(), dir, directory.BinDir); err != nil { + return err + } + dir = filepath.Dir(filepath.Dir(criDockerd.RemotePath())) + if _, err = d.sshClient.SudoCmdf("tar Cxzvf %s %s && mv %s/cri-dockerd/* %s", dir, criDockerd.RemotePath(), dir, directory.BinDir); err != nil { + return err + } + if _, err = d.sshClient.SudoCmdf("tar Cxzvf %s %s", filepath.Dir(crictl.RemotePath()), crictl.RemotePath()); err != nil { + return err + } + return nil +} + +// Install installs the container manager and related components. +func (d *DockerService) Install() error { + if err := d.installDocker(); err != nil { + return err + } + if err := d.installCRIDockerd(); err != nil { + return err + } + if err := d.installCrictl(); err != nil { + return err + } + return nil +} + +func (d *DockerService) installDocker() error { + if err := d.generateDockerService(); err != nil { + return err + } + if err := d.generateDockerConfig(); err != nil { + return err + } + if _, err := d.sshClient.SudoCmd("systemctl daemon-reload && systemctl enable docker && systemctl start docker"); err != nil { + return err + } + return nil +} + +func (d *DockerService) installCRIDockerd() error { + if err := d.generateCRIDockerdService(); err != nil { + return err + } + if err := d.generateCRIDockerdSocket(); err != nil { + return err + } + if _, err := d.sshClient.SudoCmd("systemctl daemon-reload && systemctl enable cri-docker && systemctl enable --now cri-docker.socket && systemctl start cri-docker"); err != nil { + return err + } + return nil +} + +func (d *DockerService) installCrictl() error { + temp, err := template.ParseFS(f, "templates/crictl.yaml") + if err != nil { + return err + } + + svc, err := d.getTemplateService( + temp, + file.Data{ + "Endpoint": d.instanceScope.ContainerManager().CRISocket, + }, + filepath.Join("/etc/", temp.Name())) + if err != nil { + return err + } + if err := svc.RenderToLocal(); err != nil { + return err + } + if err := svc.Copy(true); err != nil { + return err + } + return nil +} + +func (d *DockerService) generateDockerService() error { + temp, err := template.ParseFS(f, "templates/docker.service") + if err != nil { + return err + } + svc, err := d.getTemplateService(temp, nil, filepath.Join(file.SystemdDir, temp.Name())) + if err != nil { + return err + } + if err = svc.RenderToLocal(); err != nil { + return err + } + if err = svc.Copy(true); err != nil { + return err + } + return nil +} + +func (d *DockerService) generateDockerConfig() error { + temp, err := template.ParseFS(f, "templates/daemon.json") + if err != nil { + return err + } + svc, err := d.getTemplateService( + temp, + file.Data{ + "Mirrors": d.mirrors(), + "InsecureRegistries": d.insecureRegistry(), + }, + filepath.Join("/etc/docker", temp.Name())) + if err != nil { + return err + } + if err = svc.RenderToLocal(); err != nil { + return err + } + if err = svc.Copy(true); err != nil { + return err + } + return nil +} + +func (d *DockerService) generateCRIDockerdService() error { + temp, err := template.ParseFS(f, "templates/cri-docker.service") + if err != nil { + return err + } + svc, err := d.getTemplateService(temp, nil, filepath.Join(file.SystemdDir, temp.Name())) + if err != nil { + return err + } + if err = svc.RenderToLocal(); err != nil { + return err + } + if err = svc.Copy(true); err != nil { + return err + } + return nil +} + +func (d *DockerService) generateCRIDockerdSocket() error { + temp, err := template.ParseFS(f, "templates/cri-docker.socket") + if err != nil { + return err + } + svc, err := d.getTemplateService(temp, nil, filepath.Join(file.SystemdDir, temp.Name())) + if err != nil { + return err + } + if err = svc.RenderToLocal(); err != nil { + return err + } + if err = svc.Copy(true); err != nil { + return err + } + return nil +} + +func (d *DockerService) mirrors() string { + var m string + if d.scope.GlobalRegistry() != nil { + var mirrorsArr []string + for _, mirror := range d.scope.GlobalRegistry().RegistryMirrors { + mirrorsArr = append(mirrorsArr, fmt.Sprintf("%q", mirror)) + } + m = strings.Join(mirrorsArr, ", ") + } + return m +} + +func (d *DockerService) insecureRegistry() string { + var insecureRegistries string + if d.scope.GlobalRegistry() != nil { + var registriesArr []string + for _, repo := range d.scope.GlobalRegistry().InsecureRegistries { + registriesArr = append(registriesArr, fmt.Sprintf("%q", repo)) + } + insecureRegistries = strings.Join(registriesArr, ", ") + } + return insecureRegistries +} diff --git a/pkg/service/containermanager/service.go b/pkg/service/containermanager/service.go index 0a52c536..e61dde1c 100644 --- a/pkg/service/containermanager/service.go +++ b/pkg/service/containermanager/service.go @@ -43,6 +43,8 @@ func NewService(sshClient ssh.Interface, scope scope.KKInstanceScope, instanceSc switch instanceScope.ContainerManager().Type { case file.ContainerdID: return NewContainerdService(sshClient, scope, instanceScope) + case file.DockerID: + return NewDockerService(sshClient, scope, instanceScope) default: return NewContainerdService(sshClient, scope, instanceScope) } diff --git a/pkg/service/containermanager/templates/cri-docker.service b/pkg/service/containermanager/templates/cri-docker.service new file mode 100644 index 00000000..0842c128 --- /dev/null +++ b/pkg/service/containermanager/templates/cri-docker.service @@ -0,0 +1,39 @@ +[Unit] +Description=CRI Interface for Docker Application Container Engine +Documentation=https://docs.mirantis.com +After=network-online.target firewalld.service docker.service +Wants=network-online.target +Requires=cri-docker.socket + +[Service] +Type=notify +ExecStart=/usr/local/bin/cri-dockerd --container-runtime-endpoint fd:// +ExecReload=/bin/kill -s HUP $MAINPID +TimeoutSec=0 +RestartSec=2 +Restart=always + +# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229. +# Both the old, and new location are accepted by systemd 229 and up, so using the old location +# to make them work for either version of systemd. +StartLimitBurst=3 + +# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230. +# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make +# this option work for either version of systemd. +StartLimitInterval=60s + +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity + +# Comment TasksMax if your systemd version does not support it. +# Only systemd 226 and above support this option. +TasksMax=infinity +Delegate=yes +KillMode=process + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/service/containermanager/templates/cri-docker.socket b/pkg/service/containermanager/templates/cri-docker.socket new file mode 100644 index 00000000..ddba2f5b --- /dev/null +++ b/pkg/service/containermanager/templates/cri-docker.socket @@ -0,0 +1,12 @@ +[Unit] +Description=CRI Docker Socket for the API +PartOf=cri-docker.service + +[Socket] +ListenStream=%t/cri-dockerd.sock +SocketMode=0660 +SocketUser=root +SocketGroup=root + +[Install] +WantedBy=sockets.target \ No newline at end of file diff --git a/pkg/service/containermanager/templates/daemon.json b/pkg/service/containermanager/templates/daemon.json new file mode 100644 index 00000000..88fb65d5 --- /dev/null +++ b/pkg/service/containermanager/templates/daemon.json @@ -0,0 +1,13 @@ +{ + "log-opts": { + "max-size": "5m", + "max-file":"3" + }, +{{- if .Mirrors }} +"registry-mirrors": [{{ .Mirrors }}], +{{- end}} +{{- if .InsecureRegistries }} +"insecure-registries": [{{ .InsecureRegistries }}], +{{- end}} +"exec-opts": ["native.cgroupdriver=systemd"] +} \ No newline at end of file diff --git a/pkg/service/containermanager/templates/docker.service b/pkg/service/containermanager/templates/docker.service new file mode 100644 index 00000000..6a98627e --- /dev/null +++ b/pkg/service/containermanager/templates/docker.service @@ -0,0 +1,47 @@ +[Unit] +Description=Docker Application Container Engine +Documentation=https://docs.docker.com +# After=network-online.target firewalld.service containerd.service +# Wants=network-online.target +# Requires=docker.socket containerd.service + +[Service] +Type=notify +# the default is not to use systemd for cgroups because the delegate issues still +# exists and systemd currently does not support the cgroup feature set required +# for containers run by docker +ExecStart=/usr/local/bin/dockerd +ExecReload=/bin/kill -s HUP $MAINPID +TimeoutSec=0 +RestartSec=2 +Restart=always + +# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229. +# Both the old, and new location are accepted by systemd 229 and up, so using the old location +# to make them work for either version of systemd. +StartLimitBurst=3 + +# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230. +# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make +# this option work for either version of systemd. +StartLimitInterval=60s + +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity + +# Comment TasksMax if your systemd version does not support it. +# Only systemd 226 and above support this option. +TasksMax=infinity + +# set delegate yes so that systemd does not reset the cgroups of docker containers +Delegate=yes + +# kill only the docker process, not all processes in the cgroup +KillMode=process +OOMScoreAdjust=-500 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/service/operation/file/checksum/internal_checksum.go b/pkg/service/operation/file/checksum/internal_checksum.go index 0824e75b..ff8e7943 100644 --- a/pkg/service/operation/file/checksum/internal_checksum.go +++ b/pkg/service/operation/file/checksum/internal_checksum.go @@ -31,6 +31,7 @@ const ( arm64 = "arm64" k3s = "k3s" docker = "docker" + cridockerd = "cri-dockerd" crictl = "crictl" registry = "registry" harbor = "harbor" @@ -439,6 +440,14 @@ var ( "20.10.17": "249244024b507a6599084522cc73e73993349d13264505b387593f2b2ed603e6", }, }, + cridockerd: { + amd64: { + "0.2.6": "5d57b160d5a1f75333149823bec3e291a1a0960383ddc9ddd6e4ff177382c755", + }, + arm64: { + "0.2.6": "90122641e45e8ff81dbdd4d84c06fd9744b807b87bff5d0db7f826ded326a9fd", + }, + }, containerd: { amd64: { "1.6.2": "3d94f887de5f284b0d6ee61fa17ba413a7d60b4bb27d756a402b713a53685c6a", diff --git a/pkg/service/operation/file/docker.go b/pkg/service/operation/file/docker.go index 248ed241..5464f186 100644 --- a/pkg/service/operation/file/docker.go +++ b/pkg/service/operation/file/docker.go @@ -23,6 +23,7 @@ import ( "github.com/kubesphere/kubekey/pkg/clients/ssh" "github.com/kubesphere/kubekey/pkg/rootfs" + "github.com/kubesphere/kubekey/pkg/service/operation/directory" "github.com/kubesphere/kubekey/pkg/util" ) @@ -37,6 +38,15 @@ const ( DockerDefaultVersion = "20.10.8" ) +// CRI-Dockerd Info +const ( + CRIDockerdName = "cri-dockerd-%s.%s.tgz" + CRIDockerdID = "cri-dockerd" + CRIDockerdURL = "https://github.com/Mirantis/cri-dockerd/releases/download" + CRIDockerdURLPathTmpl = "/v%s/cri-dockerd-%s.%s.tgz" + DefaultCRIDockerdVersion = "0.2.6" +) + // Docker is a Binary for docker. type Docker struct { *Binary @@ -51,7 +61,7 @@ func NewDocker(sshClient ssh.Interface, rootFs rootfs.Interface, version, arch s Type: FileBinary, Name: fileName, LocalFullPath: filepath.Join(rootFs.ClusterRootFsDir(), fileName), - RemoteFullPath: filepath.Join(BinDir, fileName), + RemoteFullPath: filepath.Join(directory.BinDir, fileName), }) if err != nil { return nil, err @@ -76,3 +86,39 @@ func (d *Docker) SetZone(zone string) { d.SetPath(fmt.Sprintf(DockerURLPathTmplCN, util.ArchAlias(d.arch), d.version)) } } + +// CRIDockerd is a Binary for cri-dockerd. +type CRIDockerd struct { + *Binary +} + +// NewCRIDockerd returns a new CRIDockerd. +func NewCRIDockerd(sshClient ssh.Interface, rootFs rootfs.Interface, version, arch string) (*CRIDockerd, error) { + fileName := fmt.Sprintf(CRIDockerdName, version, arch) + file, err := NewFile(Params{ + SSHClient: sshClient, + RootFs: rootFs, + Type: FileBinary, + Name: fileName, + LocalFullPath: filepath.Join(rootFs.ClusterRootFsDir(), fileName), + RemoteFullPath: filepath.Join(directory.BinDir, fileName), + }) + if err != nil { + return nil, err + } + + u := parseURL(CRIDockerdURL, fmt.Sprintf(CRIDockerdURLPathTmpl, version, version, arch)) + binary := NewBinary(BinaryParams{ + File: file, + ID: CRIDockerdID, + Version: version, + Arch: arch, + URL: u, + }) + return &CRIDockerd{binary}, nil +} + +// SetZone override Binary's SetZone method. +func (d *CRIDockerd) SetZone(zone string) { + // TODO set zone +} diff --git a/test/e2e/config/e2e_conf.yaml b/test/e2e/config/e2e_conf.yaml index 4150929a..b05b6263 100644 --- a/test/e2e/config/e2e_conf.yaml +++ b/test/e2e/config/e2e_conf.yaml @@ -93,6 +93,7 @@ variables: # This avoids building node images in the default case which improves the test duration significantly. KUBERNETES_VERSION_MANAGEMENT: "v1.24.0" KUBERNETES_VERSION: "v1.24.0" + IMAGE_REPOSITORY: "k8s.gcr.io" CNI: "./data/cni/calico.yaml" EVENT_BRIDGE_INSTANCE_STATE: "true" EXP_CLUSTER_RESOURCE_SET: "true" @@ -105,6 +106,8 @@ variables: PASSWORD: "Qcloud@123" INSTANCES: "[{address: 192.168.100.3}, {address: 192.168.100.4}]" CONTROL_PLANE_ENDPOINT_IP: "192.168.100.100" + CONTAINER_MANAGER_TYPE: containerd + CRI_SOCKET: unix:///var/run/containerd/containerd.sock intervals: default/wait-controllers: [ "5m", "10s" ] diff --git a/test/e2e/data/infrastructure-kubekey/v1beta1/bases/cluster-with-kcp.yaml b/test/e2e/data/infrastructure-kubekey/v1beta1/bases/cluster-with-kcp.yaml index 4493a1e4..efdd6789 100644 --- a/test/e2e/data/infrastructure-kubekey/v1beta1/bases/cluster-with-kcp.yaml +++ b/test/e2e/data/infrastructure-kubekey/v1beta1/bases/cluster-with-kcp.yaml @@ -47,6 +47,8 @@ metadata: spec: template: spec: + containerManager: + type: ${CONTAINER_MANAGER_TYPE} roles: - control-plane repository: @@ -72,6 +74,7 @@ spec: clusterConfiguration: controllerManager: extraArgs: {enable-hostpath-provisioner: 'true'} + imageRepository: ${IMAGE_REPOSITORY} apiServer: # host.docker.internal is required by kubetest when running on macOS because of the way ports are proxied. certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal] @@ -145,12 +148,12 @@ spec: path: /etc/kubernetes/manifests/kube-vip.yaml initConfiguration: nodeRegistration: - criSocket: unix:///var/run/containerd/containerd.sock + criSocket: ${CRI_SOCKET} kubeletExtraArgs: eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' joinConfiguration: nodeRegistration: - criSocket: unix:///var/run/containerd/containerd.sock + criSocket: ${CRI_SOCKET} kubeletExtraArgs: eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' version: "${KUBERNETES_VERSION}" diff --git a/test/e2e/data/infrastructure-kubekey/v1beta1/bases/md.yaml b/test/e2e/data/infrastructure-kubekey/v1beta1/bases/md.yaml index 16206bd5..f6aeffaf 100644 --- a/test/e2e/data/infrastructure-kubekey/v1beta1/bases/md.yaml +++ b/test/e2e/data/infrastructure-kubekey/v1beta1/bases/md.yaml @@ -6,6 +6,8 @@ metadata: spec: template: spec: + containerManager: + type: ${CONTAINER_MANAGER_TYPE} roles: - control-plane - worker @@ -23,7 +25,7 @@ spec: spec: joinConfiguration: nodeRegistration: - criSocket: unix:///var/run/containerd/containerd.sock + criSocket: ${CRI_SOCKET} kubeletExtraArgs: eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' ---