From 973d3db18f4e5e1c738178ee77acaf05c3555f54 Mon Sep 17 00:00:00 2001 From: guofutan Date: Wed, 5 Jan 2022 15:20:52 +0800 Subject: [PATCH 1/5] fix: no matches for kind "ClusterConfiguration" in version "installer.kubesphere.io/v1alpha1" --- pkg/kubesphere/modules.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/kubesphere/modules.go b/pkg/kubesphere/modules.go index 787a1eb2..1ce40344 100644 --- a/pkg/kubesphere/modules.go +++ b/pkg/kubesphere/modules.go @@ -111,6 +111,8 @@ func (d *DeployModule) Init() { d.Tasks = []task.Interface{ generateManifests, + // apply crd installer.kubesphere.io/v1alpha1 + apply, addConfig, createNamespace, setup, From da7538946715a8e4bb70be7104b7e247df02d1e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Jan 2022 16:24:45 +0000 Subject: [PATCH 2/5] build(deps): bump github.com/containerd/containerd from 1.5.8 to 1.5.9 Bumps [github.com/containerd/containerd](https://github.com/containerd/containerd) from 1.5.8 to 1.5.9. - [Release notes](https://github.com/containerd/containerd/releases) - [Changelog](https://github.com/containerd/containerd/blob/main/RELEASES.md) - [Commits](https://github.com/containerd/containerd/compare/v1.5.8...v1.5.9) --- updated-dependencies: - dependency-name: github.com/containerd/containerd dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b0908378..544101a5 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/kubesphere/kubekey go 1.17 require ( - github.com/containerd/containerd v1.5.8 + github.com/containerd/containerd v1.5.9 github.com/deckarep/golang-set v1.7.1 github.com/dominodatalab/os-release v0.0.0-20190522011736-bcdb4a3e3c2f github.com/go-logr/logr v1.2.2 diff --git a/go.sum b/go.sum index 04c0add0..5d799943 100644 --- a/go.sum +++ b/go.sum @@ -232,8 +232,8 @@ github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoT github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= github.com/containerd/containerd v1.5.2/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= -github.com/containerd/containerd v1.5.8 h1:NmkCC1/QxyZFBny8JogwLpOy2f+VEbO/f6bV2Mqtwuw= -github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s= +github.com/containerd/containerd v1.5.9 h1:rs6Xg1gtIxaeyG+Smsb/0xaSDu1VgFhOCKBXxMxbsF4= +github.com/containerd/containerd v1.5.9/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= From f3e52fb7c2597d5fbfaf642803c92025bb9ddb9c Mon Sep 17 00:00:00 2001 From: 24sama Date: Fri, 7 Jan 2022 11:41:39 +0800 Subject: [PATCH 3/5] Add a md5 check in UnArchiveModule Signed-off-by: 24sama --- pkg/artifact/module.go | 22 +++++++++++++++++--- pkg/artifact/prepares.go | 18 ++++++++++++++++ pkg/artifact/tasks.go | 45 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 3 deletions(-) diff --git a/pkg/artifact/module.go b/pkg/artifact/module.go index 3c43740d..24fa8906 100644 --- a/pkg/artifact/module.go +++ b/pkg/artifact/module.go @@ -120,13 +120,29 @@ func (u *UnArchiveModule) Init() { u.Name = "UnArchiveArtifactModule" u.Desc = "UnArchive the KubeKey artifact" + md5Check := &task.LocalTask{ + Name: "CheckArtifactMd5", + Desc: "Check the KubeKey artifact md5 value", + Action: new(Md5Check), + } + unArchive := &task.LocalTask{ - Name: "UnArchiveArtifact", - Desc: "UnArchive the KubeKey artifact", - Action: new(UnArchive), + Name: "UnArchiveArtifact", + Desc: "UnArchive the KubeKey artifact", + Prepare: &Md5AreEqual{Not: true}, + Action: new(UnArchive), + } + + createMd5File := &task.LocalTask{ + Name: "CreateArtifactMd5File", + Desc: "Create the KubeKey artifact Md5 file", + Prepare: &Md5AreEqual{Not: true}, + Action: new(CreateMd5File), } u.Tasks = []task.Interface{ + md5Check, unArchive, + createMd5File, } } diff --git a/pkg/artifact/prepares.go b/pkg/artifact/prepares.go index b686b52c..0cc376de 100644 --- a/pkg/artifact/prepares.go +++ b/pkg/artifact/prepares.go @@ -17,6 +17,7 @@ package artifact import ( + "fmt" "github.com/kubesphere/kubekey/pkg/common" "github.com/kubesphere/kubekey/pkg/core/connector" ) @@ -33,3 +34,20 @@ func (e *EnableDownload) PreCheck(_ connector.Runtime) (bool, error) { } return false, nil } + +type Md5AreEqual struct { + common.KubePrepare + Not bool +} + +func (m *Md5AreEqual) PreCheck(_ connector.Runtime) (bool, error) { + equal, ok := m.ModuleCache.GetMustBool("md5AreEqual") + if !ok { + return false, fmt.Errorf("get md5 equal value from module cache failed") + } + + if equal { + return !m.Not, nil + } + return m.Not, nil +} diff --git a/pkg/artifact/tasks.go b/pkg/artifact/tasks.go index 20964ed3..682082de 100644 --- a/pkg/artifact/tasks.go +++ b/pkg/artifact/tasks.go @@ -32,6 +32,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "io" + "io/ioutil" "os" "os/exec" "path/filepath" @@ -274,3 +275,47 @@ func (u *UnArchive) Execute(runtime connector.Runtime) error { } return nil } + +type Md5Check struct { + common.KubeAction +} + +func (m *Md5Check) Execute(runtime connector.Runtime) error { + m.ModuleCache.Set("md5AreEqual", false) + + // check if there is a md5.sum file. This file's content contains the last artifact md5 value. + oldFile := filepath.Join(runtime.GetWorkDir(), "md5.sum") + if exist := coreutil.IsExist(oldFile); !exist { + return nil + } + + oldMd5, err := ioutil.ReadFile(oldFile) + if err != nil { + return errors.Wrapf(errors.WithStack(err), "read old md5 file %s failed", oldFile) + } + + newMd5 := coreutil.LocalMd5Sum(m.KubeConf.Arg.Artifact) + + if string(oldMd5) == newMd5 { + m.ModuleCache.Set("md5AreEqual", true) + } + return nil +} + +type CreateMd5File struct { + common.KubeAction +} + +func (c *CreateMd5File) Execute(runtime connector.Runtime) error { + oldFile := filepath.Join(runtime.GetWorkDir(), "md5.sum") + newMd5 := coreutil.LocalMd5Sum(c.KubeConf.Arg.Artifact) + f, err := os.Create(oldFile) + if err != nil { + return errors.Wrapf(errors.WithStack(err), "create md5 fild %s failed", oldFile) + } + + if _, err := io.Copy(f, strings.NewReader(newMd5)); err != nil { + return errors.Wrapf(errors.WithStack(err), "write md5 value to file %s failed", oldFile) + } + return nil +} From 941c26b7e756070b610ea0fc73a3fff8fb137921 Mon Sep 17 00:00:00 2001 From: 24sama Date: Fri, 7 Jan 2022 15:38:02 +0800 Subject: [PATCH 4/5] fix: kk will use the wrong binary's path in the multi-arch environment Signed-off-by: 24sama --- pkg/binaries/k3s.go | 4 ++-- pkg/binaries/kubernetes.go | 11 ++++------- pkg/binaries/tasks.go | 2 +- pkg/container/containerd.go | 2 +- pkg/container/docker.go | 2 +- pkg/k3s/tasks.go | 2 +- pkg/kubernetes/tasks.go | 2 +- 7 files changed, 11 insertions(+), 14 deletions(-) diff --git a/pkg/binaries/k3s.go b/pkg/binaries/k3s.go index 1dd44465..2ecbf297 100644 --- a/pkg/binaries/k3s.go +++ b/pkg/binaries/k3s.go @@ -70,7 +70,7 @@ func K3sFilesDownloadHTTP(kubeConf *common.KubeConf, filepath, version, arch str binaries := []files.KubeBinary{k3s, helm, kubecni, etcd} binariesMap := make(map[string]files.KubeBinary) for _, binary := range binaries { - logger.Log.Messagef(common.LocalHost, "downloading %s ...", binary.Name) + logger.Log.Messagef(common.LocalHost, "downloading %s %s ...", arch, binary.Name) binariesMap[binary.Name] = binary if util.IsExist(binary.Path) { @@ -122,7 +122,7 @@ func K3sFilesDownloadHTTP(kubeConf *common.KubeConf, filepath, version, arch str } } - pipelineCache.Set(common.KubeBinaries, binariesMap) + pipelineCache.Set(common.KubeBinaries+"-"+arch, binariesMap) return nil } diff --git a/pkg/binaries/kubernetes.go b/pkg/binaries/kubernetes.go index d2e64444..95e0b198 100644 --- a/pkg/binaries/kubernetes.go +++ b/pkg/binaries/kubernetes.go @@ -88,7 +88,7 @@ func K8sFilesDownloadHTTP(kubeConf *common.KubeConf, filepath, version, arch str binaries := []files.KubeBinary{kubeadm, kubelet, kubectl, helm, kubecni, docker, crictl, etcd} binariesMap := make(map[string]files.KubeBinary) for _, binary := range binaries { - logger.Log.Messagef(common.LocalHost, "downloading %s ...", binary.Name) + logger.Log.Messagef(common.LocalHost, "downloading %s %s ...", arch, binary.Name) binariesMap[binary.Name] = binary if util.IsExist(binary.Path) { @@ -151,7 +151,7 @@ func K8sFilesDownloadHTTP(kubeConf *common.KubeConf, filepath, version, arch str } } - pipelineCache.Set(common.KubeBinaries, binariesMap) + pipelineCache.Set(common.KubeBinaries+"-"+arch, binariesMap) return nil } @@ -185,7 +185,7 @@ func sha256sum(path string) (string, error) { return fmt.Sprintf("%x", sha256.Sum256(data)), nil } -func KubernetesArtifactBinariesDownload(manifest *common.ArtifactManifest, path, arch string, pipelineCache *cache.Cache) error { +func KubernetesArtifactBinariesDownload(manifest *common.ArtifactManifest, path, arch string) error { kkzone := os.Getenv("KKZONE") m := manifest.Spec @@ -220,11 +220,9 @@ func KubernetesArtifactBinariesDownload(manifest *common.ArtifactManifest, path, binaries = append(binaries, crictl) } - binariesMap := make(map[string]files.KubeBinary) for _, binary := range binaries { - logger.Log.Messagef(common.LocalHost, "downloading %s ...", binary.Name) + logger.Log.Messagef(common.LocalHost, "downloading %s %s ...", arch, binary.Name) - binariesMap[binary.Name] = binary if util.IsExist(binary.Path) { // download it again if it's incorrect if err := SHA256Check(binary); err != nil { @@ -274,6 +272,5 @@ func KubernetesArtifactBinariesDownload(manifest *common.ArtifactManifest, path, } } - pipelineCache.Set(common.KubeBinaries, binariesMap) return nil } diff --git a/pkg/binaries/tasks.go b/pkg/binaries/tasks.go index afd6ab0c..6c3c73e1 100644 --- a/pkg/binaries/tasks.go +++ b/pkg/binaries/tasks.go @@ -129,7 +129,7 @@ func (a *ArtifactDownload) Execute(runtime connector.Runtime) error { return errors.Wrap(err, "Failed to create download target dir") } - if err := KubernetesArtifactBinariesDownload(a.Manifest, binariesDir, arch, a.PipelineCache); err != nil { + if err := KubernetesArtifactBinariesDownload(a.Manifest, binariesDir, arch); err != nil { return err } } diff --git a/pkg/container/containerd.go b/pkg/container/containerd.go index c7bfe40e..747d143a 100644 --- a/pkg/container/containerd.go +++ b/pkg/container/containerd.go @@ -36,7 +36,7 @@ func (s *SyncCrictlBinaries) Execute(runtime connector.Runtime) error { return err } - binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries) + binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) if !ok { return errors.New("get KubeBinary by pipeline cache failed") } diff --git a/pkg/container/docker.go b/pkg/container/docker.go index 977a67ce..f16b1c3b 100644 --- a/pkg/container/docker.go +++ b/pkg/container/docker.go @@ -38,7 +38,7 @@ func (s *SyncDockerBinaries) Execute(runtime connector.Runtime) error { return err } - binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries) + binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) if !ok { return errors.New("get KubeBinary by pipeline cache failed") } diff --git a/pkg/k3s/tasks.go b/pkg/k3s/tasks.go index 15c2815d..954dcc82 100644 --- a/pkg/k3s/tasks.go +++ b/pkg/k3s/tasks.go @@ -88,7 +88,7 @@ type SyncKubeBinary struct { } func (s *SyncKubeBinary) Execute(runtime connector.Runtime) error { - binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries) + binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) if !ok { return errors.New("get KubeBinary by pipeline cache failed") } diff --git a/pkg/kubernetes/tasks.go b/pkg/kubernetes/tasks.go index 80386091..d46da70f 100644 --- a/pkg/kubernetes/tasks.go +++ b/pkg/kubernetes/tasks.go @@ -101,7 +101,7 @@ type SyncKubeBinary struct { } func (i *SyncKubeBinary) Execute(runtime connector.Runtime) error { - binariesMapObj, ok := i.PipelineCache.Get(common.KubeBinaries) + binariesMapObj, ok := i.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) if !ok { return errors.New("get KubeBinary by pipeline cache failed") } From e598c919dc217b0c85385e8cb77dcf36c67c3226 Mon Sep 17 00:00:00 2001 From: pixiake Date: Wed, 29 Dec 2021 15:12:58 +0800 Subject: [PATCH 5/5] Creating container images registry is supported Signed-off-by: pixiake --- apis/kubekey/v1alpha2/cluster_types.go | 71 +++- apis/kubekey/v1alpha2/default.go | 97 +++--- .../kubekey/v1alpha2/zz_generated.deepcopy.go | 12 + cmd/ctl/init/init.go | 1 + cmd/ctl/init/init_registry.go | 67 ++++ .../bases/kubekey.kubesphere.io_clusters.yaml | 8 +- pkg/binaries/module.go | 25 ++ pkg/binaries/registry.go | 131 ++++++++ pkg/binaries/tasks.go | 18 + pkg/bootstrap/os/templates/init_script.go | 5 + pkg/bootstrap/registry/certs.go | 142 ++++++++ pkg/bootstrap/registry/module.go | 311 ++++++++++++++++++ pkg/bootstrap/registry/prepares.go | 34 ++ pkg/bootstrap/registry/tasks.go | 209 ++++++++++++ pkg/bootstrap/registry/templates/harbor.go | 119 +++++++ pkg/bootstrap/registry/templates/registry.go | 52 +++ pkg/common/common.go | 15 +- pkg/common/kube_runtime.go | 3 + pkg/common/loader.go | 7 +- pkg/config/templates/cluster.go | 1 + pkg/container/templates/containerd_config.go | 1 - pkg/etcd/certs.go | 2 +- pkg/files/file.go | 40 ++- pkg/pipelines/add_nodes.go | 2 + pkg/pipelines/init_registry.go | 72 ++++ 25 files changed, 1357 insertions(+), 88 deletions(-) create mode 100644 cmd/ctl/init/init_registry.go create mode 100644 pkg/binaries/registry.go create mode 100644 pkg/bootstrap/registry/certs.go create mode 100644 pkg/bootstrap/registry/module.go create mode 100644 pkg/bootstrap/registry/prepares.go create mode 100644 pkg/bootstrap/registry/tasks.go create mode 100644 pkg/bootstrap/registry/templates/harbor.go create mode 100644 pkg/bootstrap/registry/templates/registry.go create mode 100644 pkg/pipelines/init_registry.go diff --git a/apis/kubekey/v1alpha2/cluster_types.go b/apis/kubekey/v1alpha2/cluster_types.go index d77151f8..acb44f82 100644 --- a/apis/kubekey/v1alpha2/cluster_types.go +++ b/apis/kubekey/v1alpha2/cluster_types.go @@ -147,28 +147,31 @@ type HostCfg struct { PrivateKeyPath string `yaml:"privateKeyPath,omitempty" json:"privateKeyPath,omitempty"` Arch string `yaml:"arch,omitempty" json:"arch,omitempty"` - Labels map[string]string `yaml:"labels,omitempty" json:"labels,omitempty"` - ID string `yaml:"id,omitempty" json:"id,omitempty"` - Index int `json:"-"` - IsEtcd bool `json:"-"` - IsMaster bool `json:"-"` - IsWorker bool `json:"-"` + Labels map[string]string `yaml:"labels,omitempty" json:"labels,omitempty"` + ID string `yaml:"id,omitempty" json:"id,omitempty"` + Index int `json:"-"` + IsEtcd bool `json:"-"` + IsMaster bool `json:"-"` + IsWorker bool `json:"-"` + IsRegistry bool `json:"-"` } -// RoleGroups defines the grouping of role for hosts (etcd / master / worker). +// RoleGroups defines the grouping of role for hosts (etcd / master / worker / registry). type RoleGroups struct { - Etcd []string `yaml:"etcd" json:"etcd,omitempty"` - Master []string `yaml:"master" json:"master,omitempty"` - Worker []string `yaml:"worker" json:"worker,omitempty"` + Etcd []string `yaml:"etcd" json:"etcd,omitempty"` + Master []string `yaml:"master" json:"master,omitempty"` + Worker []string `yaml:"worker" json:"worker,omitempty"` + Registry []string `yaml:"registry" json:"registry,omitempty"` } // HostGroups defines the grouping of hosts for cluster (all / etcd / master / worker / k8s). type HostGroups struct { - All []HostCfg - Etcd []HostCfg - Master []HostCfg - Worker []HostCfg - K8s []HostCfg + All []HostCfg + Etcd []HostCfg + Master []HostCfg + Worker []HostCfg + K8s []HostCfg + Registry []HostCfg } // ControlPlaneEndpoint defines the control plane endpoint information for cluster. @@ -181,6 +184,7 @@ type ControlPlaneEndpoint struct { // RegistryConfig defines the configuration information of the image's repository. type RegistryConfig struct { + Type string `yaml:"type" json:"type,omitempty"` RegistryMirrors []string `yaml:"registryMirrors" json:"registryMirrors,omitempty"` InsecureRegistries []string `yaml:"insecureRegistries" json:"insecureRegistries,omitempty"` PrivateRegistry string `yaml:"privateRegistry" json:"privateRegistry,omitempty"` @@ -242,7 +246,7 @@ func (cfg *ClusterSpec) GroupHosts() (*HostGroups, error) { hostList[host.Name] = host.Name } - etcdGroup, masterGroup, workerGroup, err := cfg.ParseRolesList(hostList) + etcdGroup, masterGroup, workerGroup, registryGroup, err := cfg.ParseRolesList(hostList) if err != nil { return nil, err } @@ -275,6 +279,14 @@ func (cfg *ClusterSpec) GroupHosts() (*HostGroups, error) { } } + if len(registryGroup) > 0 { + for _, hostName := range registryGroup { + if hostName != "" && host.Name == hostName { + host.IsRegistry = true + } + } + } + if host.IsEtcd { clusterHostsGroups.Etcd = append(clusterHostsGroups.Etcd, host) } @@ -287,6 +299,10 @@ func (cfg *ClusterSpec) GroupHosts() (*HostGroups, error) { if host.IsMaster || host.IsWorker { clusterHostsGroups.K8s = append(clusterHostsGroups.K8s, host) } + if host.IsRegistry { + clusterHostsGroups.Registry = append(clusterHostsGroups.Registry, host) + } + clusterHostsGroups.All = append(clusterHostsGroups.All, host) } @@ -297,6 +313,9 @@ func (cfg *ClusterSpec) GroupHosts() (*HostGroups, error) { if len(etcdGroup) == 0 { logger.Log.Fatal(errors.New("The number of etcd cannot be 0")) } + if len(registryGroup) > 1 { + logger.Log.Fatal(errors.New("The number of registry node cannot be greater than 1.")) + } if len(masterGroup) != len(clusterHostsGroups.Master) { return nil, errors.New("Incorrect nodeName under roleGroups/master in the configuration file") @@ -307,6 +326,9 @@ func (cfg *ClusterSpec) GroupHosts() (*HostGroups, error) { if len(workerGroup) != len(clusterHostsGroups.Worker) { return nil, errors.New("Incorrect nodeName under roleGroups/work in the configuration file") } + if len(registryGroup) != len(clusterHostsGroups.Registry) { + return nil, errors.New("Incorrect nodeName under roleGroups/registry in the configuration file") + } return &clusterHostsGroups, nil } @@ -331,10 +353,11 @@ func (cfg *ClusterSpec) ClusterDNS() string { } // ParseRolesList is used to parse the host grouping list. -func (cfg *ClusterSpec) ParseRolesList(hostList map[string]string) ([]string, []string, []string, error) { +func (cfg *ClusterSpec) ParseRolesList(hostList map[string]string) ([]string, []string, []string, []string, error) { etcdGroupList := make([]string, 0) masterGroupList := make([]string, 0) workerGroupList := make([]string, 0) + registryGroupList := make([]string, 0) for _, host := range cfg.RoleGroups.Etcd { if strings.Contains(host, "[") && strings.Contains(host, "]") && strings.Contains(host, ":") { @@ -368,7 +391,19 @@ func (cfg *ClusterSpec) ParseRolesList(hostList map[string]string) ([]string, [] workerGroupList = append(workerGroupList, host) } } - return etcdGroupList, masterGroupList, workerGroupList, nil + + for _, host := range cfg.RoleGroups.Registry { + if strings.Contains(host, "[") && strings.Contains(host, "]") && strings.Contains(host, ":") { + registryGroupList = append(registryGroupList, getHostsRange(host, hostList, "registry")...) + } else { + if err := hostVerify(hostList, host, "registry"); err != nil { + logger.Log.Fatal(err) + } + registryGroupList = append(registryGroupList, host) + } + } + + return etcdGroupList, masterGroupList, workerGroupList, registryGroupList, nil } func getHostsRange(rangeStr string, hostList map[string]string, group string) []string { diff --git a/apis/kubekey/v1alpha2/default.go b/apis/kubekey/v1alpha2/default.go index 984c95b6..122b1d86 100644 --- a/apis/kubekey/v1alpha2/default.go +++ b/apis/kubekey/v1alpha2/default.go @@ -24,53 +24,56 @@ import ( ) const ( - DefaultPreDir = "kubekey" - DefaultTmpDir = "/tmp/kubekey" - DefaultSSHPort = 22 - DefaultLBPort = 6443 - DefaultLBDomain = "lb.kubesphere.local" - DefaultNetworkPlugin = "calico" - DefaultPodsCIDR = "10.233.64.0/18" - DefaultServiceCIDR = "10.233.0.0/18" - DefaultKubeImageNamespace = "kubesphere" - DefaultClusterName = "cluster.local" - DefaultArch = "amd64" - DefaultEtcdVersion = "v3.4.13" - DefaultEtcdPort = "2379" - DefaultDockerVersion = "20.10.8" - DefaultCrictlVersion = "v1.22.0" - DefaultKubeVersion = "v1.21.5" - DefaultCalicoVersion = "v3.20.0" - DefaultFlannelVersion = "v0.12.0" - DefaultCniVersion = "v0.9.1" - DefaultCiliumVersion = "v1.8.3" - DefaultKubeovnVersion = "v1.5.0" - DefalutMultusVersion = "v3.8" - DefaultHelmVersion = "v3.6.3" - DefaultMaxPods = 110 - DefaultNodeCidrMaskSize = 24 - DefaultIPIPMode = "Always" - DefaultVXLANMode = "Never" - DefaultVethMTU = 1440 - DefaultBackendMode = "vxlan" - DefaultProxyMode = "ipvs" - DefaultCrioEndpoint = "unix:///var/run/crio/crio.sock" - DefaultContainerdEndpoint = "unix:///run/containerd/containerd.sock" - DefaultIsulaEndpoint = "unix:///var/run/isulad.sock" - Etcd = "etcd" - Master = "master" - Worker = "worker" - K8s = "k8s" - DefaultEtcdBackupDir = "/var/backups/kube_etcd" - DefaultEtcdBackupPeriod = 30 - DefaultKeepBackNumber = 5 - DefaultEtcdBackupScriptDir = "/usr/local/bin/kube-scripts" - DefaultJoinCIDR = "100.64.0.0/16" - DefaultNetworkType = "geneve" - DefaultVlanID = "100" - DefaultOvnLabel = "node-role.kubernetes.io/master" - DefaultDPDKVersion = "19.11" - DefaultDNSAddress = "114.114.114.114" + DefaultPreDir = "kubekey" + DefaultTmpDir = "/tmp/kubekey" + DefaultSSHPort = 22 + DefaultLBPort = 6443 + DefaultLBDomain = "lb.kubesphere.local" + DefaultNetworkPlugin = "calico" + DefaultPodsCIDR = "10.233.64.0/18" + DefaultServiceCIDR = "10.233.0.0/18" + DefaultKubeImageNamespace = "kubesphere" + DefaultClusterName = "cluster.local" + DefaultArch = "amd64" + DefaultEtcdVersion = "v3.4.13" + DefaultEtcdPort = "2379" + DefaultDockerVersion = "20.10.8" + DefaultCrictlVersion = "v1.22.0" + DefaultKubeVersion = "v1.21.5" + DefaultCalicoVersion = "v3.20.0" + DefaultFlannelVersion = "v0.12.0" + DefaultCniVersion = "v0.9.1" + DefaultCiliumVersion = "v1.8.3" + DefaultKubeovnVersion = "v1.5.0" + DefalutMultusVersion = "v3.8" + DefaultHelmVersion = "v3.6.3" + DefaultDockerComposeVersion = "v2.2.2" + DefaultRegistryVersion = "2" + DefaultHarborVersion = "v2.4.1" + DefaultMaxPods = 110 + DefaultNodeCidrMaskSize = 24 + DefaultIPIPMode = "Always" + DefaultVXLANMode = "Never" + DefaultVethMTU = 1440 + DefaultBackendMode = "vxlan" + DefaultProxyMode = "ipvs" + DefaultCrioEndpoint = "unix:///var/run/crio/crio.sock" + DefaultContainerdEndpoint = "unix:///run/containerd/containerd.sock" + DefaultIsulaEndpoint = "unix:///var/run/isulad.sock" + Etcd = "etcd" + Master = "master" + Worker = "worker" + K8s = "k8s" + DefaultEtcdBackupDir = "/var/backups/kube_etcd" + DefaultEtcdBackupPeriod = 30 + DefaultKeepBackNumber = 5 + DefaultEtcdBackupScriptDir = "/usr/local/bin/kube-scripts" + DefaultJoinCIDR = "100.64.0.0/16" + DefaultNetworkType = "geneve" + DefaultVlanID = "100" + DefaultOvnLabel = "node-role.kubernetes.io/master" + DefaultDPDKVersion = "19.11" + DefaultDNSAddress = "114.114.114.114" Docker = "docker" Conatinerd = "containerd" diff --git a/apis/kubekey/v1alpha2/zz_generated.deepcopy.go b/apis/kubekey/v1alpha2/zz_generated.deepcopy.go index f6e6a01e..a95a5130 100644 --- a/apis/kubekey/v1alpha2/zz_generated.deepcopy.go +++ b/apis/kubekey/v1alpha2/zz_generated.deepcopy.go @@ -479,6 +479,13 @@ func (in *HostGroups) DeepCopyInto(out *HostGroups) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Registry != nil { + in, out := &in.Registry, &out.Registry + *out = make([]HostCfg, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostGroups. @@ -889,6 +896,11 @@ func (in *RoleGroups) DeepCopyInto(out *RoleGroups) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.Registry != nil { + in, out := &in.Registry, &out.Registry + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RoleGroups. diff --git a/cmd/ctl/init/init.go b/cmd/ctl/init/init.go index 7918ad7b..13c190d5 100644 --- a/cmd/ctl/init/init.go +++ b/cmd/ctl/init/init.go @@ -40,5 +40,6 @@ func NewCmdInit() *cobra.Command { } o.CommonOptions.AddCommonFlag(cmd) cmd.AddCommand(NewCmdInitOs()) + cmd.AddCommand(NewCmdInitRegistry()) return cmd } diff --git a/cmd/ctl/init/init_registry.go b/cmd/ctl/init/init_registry.go new file mode 100644 index 00000000..c338ddc3 --- /dev/null +++ b/cmd/ctl/init/init_registry.go @@ -0,0 +1,67 @@ +/* +Copyright 2022 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package init + +import ( + "github.com/kubesphere/kubekey/cmd/ctl/options" + "github.com/kubesphere/kubekey/cmd/ctl/util" + "github.com/kubesphere/kubekey/pkg/common" + "github.com/kubesphere/kubekey/pkg/pipelines" + "github.com/spf13/cobra" +) + +type InitRegistryOptions struct { + CommonOptions *options.CommonOptions + ClusterCfgFile string + DownloadCmd string +} + +func NewInitRegistryOptions() *InitRegistryOptions { + return &InitRegistryOptions{ + CommonOptions: options.NewCommonOptions(), + } +} + +// NewCmdInitRegistry creates a new init os command +func NewCmdInitRegistry() *cobra.Command { + o := NewInitRegistryOptions() + cmd := &cobra.Command{ + Use: "registry", + Short: "Init a local image registry", + Run: func(cmd *cobra.Command, args []string) { + util.CheckErr(o.Run()) + }, + } + + o.CommonOptions.AddCommonFlag(cmd) + o.AddFlags(cmd) + return cmd +} + +func (o *InitRegistryOptions) Run() error { + arg := common.Argument{ + FilePath: o.ClusterCfgFile, + Debug: o.CommonOptions.Verbose, + } + return pipelines.InitRegistry(arg, o.DownloadCmd) +} + +func (o *InitRegistryOptions) AddFlags(cmd *cobra.Command) { + cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") + cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s", + `The user defined command to download the necessary files. The first param '%s' is output path, the second param '%s', is the URL`) +} diff --git a/config/crd/bases/kubekey.kubesphere.io_clusters.yaml b/config/crd/bases/kubekey.kubesphere.io_clusters.yaml index 4301ea59..bb02553e 100644 --- a/config/crd/bases/kubekey.kubesphere.io_clusters.yaml +++ b/config/crd/bases/kubekey.kubesphere.io_clusters.yaml @@ -608,10 +608,12 @@ spec: items: type: string type: array + type: + type: string type: object roleGroups: description: RoleGroups defines the grouping of role for hosts (etcd - / master / worker). + / master / worker / registry). properties: etcd: items: @@ -621,6 +623,10 @@ spec: items: type: string type: array + registry: + items: + type: string + type: array worker: items: type: string diff --git a/pkg/binaries/module.go b/pkg/binaries/module.go index 96c53378..2ca5e59d 100644 --- a/pkg/binaries/module.go +++ b/pkg/binaries/module.go @@ -18,7 +18,9 @@ package binaries import ( "github.com/kubesphere/kubekey/pkg/common" + "github.com/kubesphere/kubekey/pkg/core/logger" "github.com/kubesphere/kubekey/pkg/core/task" + "github.com/pkg/errors" ) type NodeBinariesModule struct { @@ -77,3 +79,26 @@ func (a *ArtifactBinariesModule) Init() { download, } } + +type RegistryPackageModule struct { + common.KubeModule +} + +func (n *RegistryPackageModule) Init() { + n.Name = "RegistryPackageModule" + n.Desc = "Download registry package" + + if len(n.Runtime.GetHostsByRole(common.Registry)) == 0 { + logger.Log.Fatal(errors.New("[registry] node not found in the roleGroups of the configuration file")) + } + + download := &task.LocalTask{ + Name: "DownloadRegistryPackage", + Desc: "Download registry package", + Action: new(RegistryPackageDownload), + } + + n.Tasks = []task.Interface{ + download, + } +} diff --git a/pkg/binaries/registry.go b/pkg/binaries/registry.go new file mode 100644 index 00000000..8b365d86 --- /dev/null +++ b/pkg/binaries/registry.go @@ -0,0 +1,131 @@ +/* + Copyright 2022 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package binaries + +import ( + "fmt" + kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/apis/kubekey/v1alpha2" + "github.com/kubesphere/kubekey/pkg/common" + "github.com/kubesphere/kubekey/pkg/core/cache" + "github.com/kubesphere/kubekey/pkg/core/logger" + "github.com/kubesphere/kubekey/pkg/core/util" + "github.com/kubesphere/kubekey/pkg/files" + "github.com/pkg/errors" + "io" + "os" + "os/exec" +) + +// RegistryPackageDownloadHTTP defines the kubernetes' binaries that need to be downloaded in advance and downloads them. +func RegistryPackageDownloadHTTP(kubeConf *common.KubeConf, filepath, arch string, pipelineCache *cache.Cache) error { + kkzone := os.Getenv("KKZONE") + + binaries := []files.KubeBinary{} + + switch kubeConf.Cluster.Registry.Type { + case common.Harbor: + harbor := files.KubeBinary{Name: "harbor", Arch: arch, Version: kubekeyapiv1alpha2.DefaultHarborVersion} + harbor.Path = fmt.Sprintf("%s/harbor-offline-installer-%s.tgz", filepath, kubekeyapiv1alpha2.DefaultHarborVersion) + // TODO: Harbor only supports amd64, so there is no need to consider other architectures at present. + docker := files.KubeBinary{Name: "docker", Arch: arch, Version: kubekeyapiv1alpha2.DefaultDockerVersion} + docker.Path = fmt.Sprintf("%s/docker-%s.tgz", filepath, kubekeyapiv1alpha2.DefaultDockerVersion) + compose := files.KubeBinary{Name: "compose", Arch: arch, Version: kubekeyapiv1alpha2.DefaultDockerComposeVersion} + compose.Path = fmt.Sprintf("%s/docker-compose-linux-x86_64", filepath) + + if kkzone == "cn" { + harbor.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/harbor/releases/download/%s/harbor-offline-installer-%s.tgz", harbor.Version, harbor.Version) + docker.Url = fmt.Sprintf("https://mirrors.aliyun.com/docker-ce/linux/static/stable/x86_64/docker-%s.tgz", docker.Version) + compose.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/docker/compose/releases/download/%s/docker-compose-linux-x86_64", compose.Version) + } else { + harbor.Url = fmt.Sprintf("https://github.com/goharbor/harbor/releases/download/%s/harbor-offline-installer-%s.tgz", harbor.Version, harbor.Version) + docker.Url = fmt.Sprintf("https://download.docker.com/linux/static/stable/x86_64/docker-%s.tgz", docker.Version) + compose.Url = fmt.Sprintf("https://github.com/docker/compose/releases/download/%s/docker-compose-linux-x86_64", compose.Version) + } + + harbor.GetCmd = kubeConf.Arg.DownloadCommand(harbor.Path, harbor.Url) + docker.GetCmd = kubeConf.Arg.DownloadCommand(docker.Path, docker.Url) + compose.GetCmd = kubeConf.Arg.DownloadCommand(compose.Path, compose.Url) + + binaries = []files.KubeBinary{harbor, docker, compose} + default: + registry := files.KubeBinary{Name: "registry", Arch: arch, Version: kubekeyapiv1alpha2.DefaultRegistryVersion} + registry.Path = fmt.Sprintf("%s/registry-%s-linux-%s.tar.gz", filepath, kubekeyapiv1alpha2.DefaultRegistryVersion, arch) + if kkzone == "cn" { + registry.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/registry/%s/registry-%s-linux-%s.tar.gz", kubekeyapiv1alpha2.DefaultRegistryVersion, kubekeyapiv1alpha2.DefaultRegistryVersion, registry.Arch) + } else { + registry.Url = fmt.Sprintf("https://github.com/kubesphere/kubekey/releases/download/v2.0.0-alpha.1/registry-%s-linux-%s.tar.gz", kubekeyapiv1alpha2.DefaultRegistryVersion, registry.Arch) + } + registry.GetCmd = kubeConf.Arg.DownloadCommand(registry.Path, registry.Url) + binaries = []files.KubeBinary{registry} + } + + binariesMap := make(map[string]files.KubeBinary) + for _, binary := range binaries { + logger.Log.Messagef(common.LocalHost, "downloading %s ...", binary.Name) + binariesMap[binary.Name] = binary + if util.IsExist(binary.Path) { + // download it again if it's incorrect + if err := SHA256Check(binary); err != nil { + _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", binary.Path)).Run() + } else { + continue + } + } + + for i := 5; i > 0; i-- { + cmd := exec.Command("/bin/sh", "-c", binary.GetCmd) + stdout, err := cmd.StdoutPipe() + if err != nil { + return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.Name, binary.GetCmd, err) + } + cmd.Stderr = cmd.Stdout + + if err = cmd.Start(); err != nil { + return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.Name, binary.GetCmd, err) + } + for { + tmp := make([]byte, 1024) + _, err := stdout.Read(tmp) + fmt.Print(string(tmp)) // Get the output from the pipeline in real time and print it to the terminal + if errors.Is(err, io.EOF) { + break + } else if err != nil { + logger.Log.Errorln(err) + break + } + } + if err = cmd.Wait(); err != nil { + if kkzone != "cn" { + logger.Log.Warningln("Having a problem with accessing https://storage.googleapis.com? You can try again after setting environment 'export KKZONE=cn'") + } + return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.Name, binary.GetCmd, err) + } + + if err := SHA256Check(binary); err != nil { + if i == 1 { + return err + } + _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", binary.Path)).Run() + continue + } + break + } + } + + pipelineCache.Set(common.KubeBinaries, binariesMap) + return nil +} diff --git a/pkg/binaries/tasks.go b/pkg/binaries/tasks.go index afd6ab0c..3a39f798 100644 --- a/pkg/binaries/tasks.go +++ b/pkg/binaries/tasks.go @@ -135,3 +135,21 @@ func (a *ArtifactDownload) Execute(runtime connector.Runtime) error { } return nil } + +type RegistryPackageDownload struct { + common.KubeAction +} + +func (k *RegistryPackageDownload) Execute(runtime connector.Runtime) error { + arch := runtime.GetHostsByRole(common.Registry)[0].GetArch() + + packageDir := filepath.Join(runtime.GetWorkDir(), "registry", arch) + if err := util.CreateDir(packageDir); err != nil { + return errors.Wrap(err, "Failed to create download target dir") + } + if err := RegistryPackageDownloadHTTP(k.KubeConf, packageDir, arch, k.PipelineCache); err != nil { + return err + } + + return nil +} diff --git a/pkg/bootstrap/os/templates/init_script.go b/pkg/bootstrap/os/templates/init_script.go index d5f86f48..fa2d9563 100644 --- a/pkg/bootstrap/os/templates/init_script.go +++ b/pkg/bootstrap/os/templates/init_script.go @@ -18,6 +18,7 @@ package templates import ( "fmt" + "github.com/kubesphere/kubekey/pkg/bootstrap/registry" "text/template" "github.com/kubesphere/kubekey/pkg/common" @@ -167,6 +168,10 @@ func GenerateHosts(runtime connector.ModuleRuntime, kubeConf *common.KubeConf) [ } } + if len(runtime.GetHostsByRole(common.Registry)) > 0 { + hostsList = append(hostsList, fmt.Sprintf("%s %s", runtime.GetHostsByRole(common.Registry)[0].GetInternalAddress(), registry.RegistryCertificateBaseName)) + } + hostsList = append(hostsList, lbHost) return hostsList } diff --git a/pkg/bootstrap/registry/certs.go b/pkg/bootstrap/registry/certs.go new file mode 100644 index 00000000..c57caadd --- /dev/null +++ b/pkg/bootstrap/registry/certs.go @@ -0,0 +1,142 @@ +/* + Copyright 2022 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package registry + +import ( + "crypto/x509" + "fmt" + "github.com/kubesphere/kubekey/pkg/common" + "github.com/kubesphere/kubekey/pkg/core/connector" + "github.com/kubesphere/kubekey/pkg/utils/certs" + "github.com/pkg/errors" + "k8s.io/client-go/util/cert" + certutil "k8s.io/client-go/util/cert" + netutils "k8s.io/utils/net" + "net" + "path/filepath" + "strings" +) + +const ( + RegistryCertificateBaseName = "dockerhub.kubekey.local" + LocalCertsDir = "localCertsDir" + CertsFileList = "certsFileList" +) + +// KubekeyCertEtcdCA is the definition of the root CA used by the hosted etcd server. +func KubekeyCertRegistryCA() *certs.KubekeyCert { + return &certs.KubekeyCert{ + Name: "registry-ca", + LongName: "self-signed CA to provision identities for registry", + BaseName: "ca", + Config: certs.CertConfig{ + Config: certutil.Config{ + CommonName: "registry-ca", + }, + }, + } +} + +// KubekeyCertEtcdAdmin is the definition of the cert for etcd admin. +func KubekeyCertRegistryServer(altNames *certutil.AltNames) *certs.KubekeyCert { + return &certs.KubekeyCert{ + Name: "registry-server", + LongName: "certificate for registry server", + BaseName: RegistryCertificateBaseName, + CAName: "registry-ca", + Config: certs.CertConfig{ + Config: certutil.Config{ + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + AltNames: *altNames, + CommonName: RegistryCertificateBaseName, + }, + }, + } +} + +type FetchCerts struct { + common.KubeAction +} + +func (f *FetchCerts) Execute(runtime connector.Runtime) error { + src := "/etc/ssl/registry/ssl" + dst := fmt.Sprintf("%s/pki/registry", runtime.GetWorkDir()) + + certs, err := runtime.GetRunner().SudoCmd("ls /etc/ssl/registry/ssl/ | grep .pem", false) + if err != nil { + return nil + } + + certsList := strings.Split(certs, "\r\n") + if len(certsList) > 0 { + for _, cert := range certsList { + if err := runtime.GetRunner().Fetch(filepath.Join(dst, cert), filepath.Join(src, cert)); err != nil { + return errors.Wrap(err, fmt.Sprintf("Fetch %s failed", filepath.Join(src, cert))) + } + } + } + + return nil +} + +type GenerateCerts struct { + common.KubeAction +} + +func (g *GenerateCerts) Execute(runtime connector.Runtime) error { + var pkiPath string + if g.KubeConf.Arg.CertificatesDir == "" { + pkiPath = fmt.Sprintf("%s/pki/registry", runtime.GetWorkDir()) + } + + var altName cert.AltNames + + dnsList := []string{"localhost", RegistryCertificateBaseName, runtime.GetHostsByRole(common.Registry)[0].GetName()} + ipList := []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback, netutils.ParseIPSloppy(runtime.GetHostsByRole(common.Registry)[0].GetInternalAddress())} + + altName.DNSNames = dnsList + altName.IPs = ipList + + files := []string{"ca.pem", "ca-key.pem", fmt.Sprintf("%s.pem", RegistryCertificateBaseName), fmt.Sprintf("%s-key.pem", RegistryCertificateBaseName)} + + // CA + certsList := []*certs.KubekeyCert{KubekeyCertRegistryCA()} + + // Certs + certsList = append(certsList, KubekeyCertRegistryServer(&altName)) + + var lastCACert *certs.KubekeyCert + for _, c := range certsList { + if c.CAName == "" { + err := certs.GenerateCA(c, pkiPath, g.KubeConf) + if err != nil { + return err + } + lastCACert = c + } else { + err := certs.GenerateCerts(c, lastCACert, pkiPath, g.KubeConf) + if err != nil { + return err + } + } + } + + g.ModuleCache.Set(LocalCertsDir, pkiPath) + g.ModuleCache.Set(CertsFileList, files) + + return nil +} diff --git a/pkg/bootstrap/registry/module.go b/pkg/bootstrap/registry/module.go new file mode 100644 index 00000000..fd4a4c1d --- /dev/null +++ b/pkg/bootstrap/registry/module.go @@ -0,0 +1,311 @@ +/* + Copyright 2022 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package registry + +import ( + "fmt" + "github.com/kubesphere/kubekey/pkg/bootstrap/registry/templates" + "github.com/kubesphere/kubekey/pkg/common" + "github.com/kubesphere/kubekey/pkg/container" + docker_template "github.com/kubesphere/kubekey/pkg/container/templates" + "github.com/kubesphere/kubekey/pkg/core/action" + "github.com/kubesphere/kubekey/pkg/core/prepare" + "github.com/kubesphere/kubekey/pkg/core/task" + "github.com/kubesphere/kubekey/pkg/core/util" + "path/filepath" +) + +type RegistryCertsModule struct { + common.KubeModule + Skip bool +} + +func (p *RegistryCertsModule) IsSkip() bool { + return p.Skip +} + +func (i *RegistryCertsModule) Init() { + i.Name = "InitRegistryModule" + i.Desc = "Init a local registry" + + fetchCerts := &task.RemoteTask{ + Name: "FetchRegistryCerts", + Desc: "Fetcd registry certs", + Hosts: i.Runtime.GetHostsByRole(common.Registry), + Prepare: new(FirstRegistryNode), + Action: new(FetchCerts), + Parallel: false, + } + + generateCerts := &task.LocalTask{ + Name: "GenerateRegistryCerts", + Desc: "Generate registry Certs", + Action: new(GenerateCerts), + } + + syncCertsFile := &task.RemoteTask{ + Name: "SyncCertsFile", + Desc: "Synchronize certs file", + Hosts: i.Runtime.GetHostsByRole(common.Registry), + Action: new(SyncCertsFile), + Parallel: true, + Retry: 1, + } + + syncCertsToAllNodes := &task.RemoteTask{ + Name: "SyncCertsFileToAllNodes", + Desc: "Synchronize certs file to all nodes", + Hosts: i.Runtime.GetHostsByRole(common.K8s), + Action: new(SyncCertsToAllNodes), + Parallel: true, + Retry: 1, + } + + i.Tasks = []task.Interface{ + fetchCerts, + generateCerts, + syncCertsFile, + syncCertsToAllNodes, + } + +} + +type InstallRegistryModule struct { + common.KubeModule +} + +func (i *InstallRegistryModule) Init() { + i.Name = "InstallRegistryModule" + i.Desc = "Install local registry" + + switch i.KubeConf.Cluster.Registry.Type { + case common.Harbor: + i.Tasks = InstallHarbor(i) + default: + i.Tasks = InstallRegistry(i) + } +} + +func InstallRegistry(i *InstallRegistryModule) []task.Interface { + installRegistryBinary := &task.RemoteTask{ + Name: "InstallRegistryBinary", + Desc: "Install local registry", + Hosts: i.Runtime.GetHostsByRole(common.Registry), + Action: new(InstallRegistryBinary), + Parallel: true, + Retry: 1, + } + + generateRegistryService := &task.RemoteTask{ + Name: "GenerateRegistryService", + Desc: "Generate registry service", + Hosts: i.Runtime.GetHostsByRole(common.Registry), + Action: &action.Template{ + Template: templates.RegistryServiceTempl, + Dst: "/etc/systemd/system/registry.service", + }, + Parallel: true, + Retry: 1, + } + + generateRegistryConfig := &task.RemoteTask{ + Name: "GenerateRegistryConfig", + Desc: "Generate registry config", + Hosts: i.Runtime.GetHostsByRole(common.Registry), + Action: &action.Template{ + Template: templates.RegistryConfigTempl, + Dst: "/etc/kubekey/registry/config.yaml", + Data: util.Data{ + "Certificate": fmt.Sprintf("%s.pem", RegistryCertificateBaseName), + "Key": fmt.Sprintf("%s-key.pem", RegistryCertificateBaseName), + }, + }, + Parallel: true, + Retry: 1, + } + + startRgistryService := &task.RemoteTask{ + Name: "StartRegistryService", + Desc: "Start registry service", + Hosts: i.Runtime.GetHostsByRole(common.Registry), + Action: new(StartRegistryService), + Parallel: true, + Retry: 1, + } + + return []task.Interface{ + installRegistryBinary, + generateRegistryService, + generateRegistryConfig, + startRgistryService, + } +} + +func InstallHarbor(i *InstallRegistryModule) []task.Interface { + // Install docker + syncBinaries := &task.RemoteTask{ + Name: "SyncDockerBinaries", + Desc: "Sync docker binaries", + Hosts: i.Runtime.GetHostsByRole(common.Registry), + Prepare: &prepare.PrepareCollection{ + &container.DockerExist{Not: true}, + }, + Action: new(container.SyncDockerBinaries), + Parallel: true, + Retry: 2, + } + + generateContainerdService := &task.RemoteTask{ + Name: "GenerateContainerdService", + Desc: "Generate containerd service", + Hosts: i.Runtime.GetHostsByRole(common.Registry), + Prepare: &prepare.PrepareCollection{ + &container.DockerExist{Not: true}, + }, + Action: &action.Template{ + Template: docker_template.ContainerdService, + Dst: filepath.Join("/etc/systemd/system", docker_template.ContainerdService.Name()), + }, + Parallel: true, + } + + enableContainerd := &task.RemoteTask{ + Name: "EnableContainerd", + Desc: "Enable containerd", + Hosts: i.Runtime.GetHostsByRole(common.Registry), + Prepare: &prepare.PrepareCollection{ + &container.DockerExist{Not: true}, + }, + Action: new(container.EnableContainerd), + Parallel: true, + } + + generateDockerService := &task.RemoteTask{ + Name: "GenerateDockerService", + Desc: "Generate docker service", + Hosts: i.Runtime.GetHostsByRole(common.Registry), + Prepare: &prepare.PrepareCollection{ + &container.DockerExist{Not: true}, + }, + Action: &action.Template{ + Template: docker_template.DockerService, + Dst: filepath.Join("/etc/systemd/system", docker_template.DockerService.Name()), + }, + Parallel: true, + } + + generateDockerConfig := &task.RemoteTask{ + Name: "GenerateDockerConfig", + Desc: "Generate docker config", + Hosts: i.Runtime.GetHostsByRole(common.Registry), + Prepare: &prepare.PrepareCollection{ + &container.DockerExist{Not: true}, + }, + Action: &action.Template{ + Template: docker_template.DockerConfig, + Dst: filepath.Join("/etc/docker/", docker_template.DockerConfig.Name()), + Data: util.Data{ + "Mirrors": docker_template.Mirrors(i.KubeConf), + "InsecureRegistries": docker_template.InsecureRegistries(i.KubeConf), + }, + }, + Parallel: true, + } + + enableDocker := &task.RemoteTask{ + Name: "EnableDocker", + Desc: "Enable docker", + Hosts: i.Runtime.GetHostsByRole(common.Registry), + Prepare: &prepare.PrepareCollection{ + &container.DockerExist{Not: true}, + }, + Action: new(container.EnableDocker), + Parallel: true, + } + + dockerLoginRegistry := &task.RemoteTask{ + Name: "Login PrivateRegistry", + Desc: "Add auths to container runtime", + Hosts: i.Runtime.GetHostsByRole(common.Registry), + Prepare: &prepare.PrepareCollection{ + &container.DockerExist{}, + &container.PrivateRegistryAuth{}, + }, + Action: new(container.DockerLoginRegistry), + Parallel: true, + } + + // Install docker compose + installDockerCompose := &task.RemoteTask{ + Name: "InstallDockerCompose", + Desc: "Install docker compose", + Hosts: i.Runtime.GetHostsByRole(common.Registry), + Action: new(InstallDockerCompose), + Parallel: true, + Retry: 2, + } + + // Install Harbor + syncHarborPackage := &task.RemoteTask{ + Name: "SyncHarborPackage", + Desc: "Sync harbor package", + Hosts: i.Runtime.GetHostsByRole(common.Registry), + Action: new(SyncHarborPackage), + Parallel: true, + Retry: 2, + } + + generateHarborConfig := &task.RemoteTask{ + Name: "GenerateHarborConfig", + Desc: "Generate harbor config", + Hosts: i.Runtime.GetHostsByRole(common.Registry), + Action: &action.Template{ + Template: templates.HarborConfigTempl, + Dst: "/opt/harbor/harbor.yml", + Data: util.Data{ + "Domain": RegistryCertificateBaseName, + "Certificate": fmt.Sprintf("%s.pem", RegistryCertificateBaseName), + "Key": fmt.Sprintf("%s-key.pem", RegistryCertificateBaseName), + }, + }, + Parallel: true, + Retry: 1, + } + + startHarbor := &task.RemoteTask{ + Name: "StartHarbor", + Desc: "start harbor", + Hosts: i.Runtime.GetHostsByRole(common.Registry), + Action: new(StartHarbor), + Parallel: true, + Retry: 2, + } + + return []task.Interface{ + syncBinaries, + generateContainerdService, + enableContainerd, + generateDockerService, + generateDockerConfig, + enableDocker, + dockerLoginRegistry, + installDockerCompose, + syncHarborPackage, + generateHarborConfig, + startHarbor, + } +} diff --git a/pkg/bootstrap/registry/prepares.go b/pkg/bootstrap/registry/prepares.go new file mode 100644 index 00000000..efb5fd46 --- /dev/null +++ b/pkg/bootstrap/registry/prepares.go @@ -0,0 +1,34 @@ +/* + Copyright 2022 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package registry + +import ( + "github.com/kubesphere/kubekey/pkg/common" + "github.com/kubesphere/kubekey/pkg/core/connector" +) + +type FirstRegistryNode struct { + common.KubePrepare + Not bool +} + +func (f *FirstRegistryNode) PreCheck(runtime connector.Runtime) (bool, error) { + if runtime.GetHostsByRole(common.Registry)[0].GetName() == runtime.RemoteHost().GetName() { + return !f.Not, nil + } + return f.Not, nil +} diff --git a/pkg/bootstrap/registry/tasks.go b/pkg/bootstrap/registry/tasks.go new file mode 100644 index 00000000..5c6d9f62 --- /dev/null +++ b/pkg/bootstrap/registry/tasks.go @@ -0,0 +1,209 @@ +/* + Copyright 2022 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package registry + +import ( + "fmt" + kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/apis/kubekey/v1alpha2" + "github.com/kubesphere/kubekey/pkg/common" + "github.com/kubesphere/kubekey/pkg/core/connector" + "github.com/kubesphere/kubekey/pkg/core/logger" + "github.com/kubesphere/kubekey/pkg/utils" + "github.com/pkg/errors" + "path/filepath" + "strings" +) + +type SyncCertsFile struct { + common.KubeAction +} + +func (s *SyncCertsFile) Execute(runtime connector.Runtime) error { + localCertsDir, ok := s.ModuleCache.Get(LocalCertsDir) + if !ok { + return errors.New("get etcd local certs dir by module cache failed") + } + files, ok := s.ModuleCache.Get(CertsFileList) + if !ok { + return errors.New("get etcd certs file list by module cache failed") + } + dir := localCertsDir.(string) + fileList := files.([]string) + + for _, fileName := range fileList { + if err := runtime.GetRunner().SudoScp(filepath.Join(dir, fileName), filepath.Join(common.RegistryCertDir, fileName)); err != nil { + return errors.Wrap(errors.WithStack(err), "scp etcd certs file failed") + } + } + + return nil +} + +type SyncCertsToAllNodes struct { + common.KubeAction +} + +func (s *SyncCertsToAllNodes) Execute(runtime connector.Runtime) error { + localCertsDir, ok := s.ModuleCache.Get(LocalCertsDir) + if !ok { + return errors.New("get etcd local certs dir by module cache failed") + } + files, ok := s.ModuleCache.Get(CertsFileList) + if !ok { + return errors.New("get etcd certs file list by module cache failed") + } + dir := localCertsDir.(string) + fileList := files.([]string) + + var dstDir string + switch s.KubeConf.Cluster.Kubernetes.ContainerManager { + case common.Docker: + dstDir = fmt.Sprintf("/etc/docker/certs.d/%s", RegistryCertificateBaseName) + case common.Conatinerd: + dstDir = common.RegistryCertDir + case common.Crio: + // TODO: Add the steps of cri-o's installation. + case common.Isula: + // TODO: Add the steps of iSula's installation. + default: + logger.Log.Fatalf("Unsupported container runtime: %s", strings.TrimSpace(s.KubeConf.Cluster.Kubernetes.ContainerManager)) + } + + for _, fileName := range fileList { + var dstFileName string + switch fileName { + case "ca.pem": + dstFileName = "ca.crt" + case "ca-key.pem": + continue + default: + if strings.HasSuffix(fileName, "-key.pem") { + dstFileName = strings.Replace(fileName, "-key.pem", ".key", -1) + } else { + dstFileName = strings.Replace(fileName, ".pem", ".cert", -1) + } + } + + if err := runtime.GetRunner().SudoScp(filepath.Join(dir, fileName), filepath.Join(dstDir, dstFileName)); err != nil { + return errors.Wrap(errors.WithStack(err), "scp etcd certs file failed") + } + } + + return nil +} + +type InstallRegistryBinary struct { + common.KubeAction +} + +func (g *InstallRegistryBinary) Execute(runtime connector.Runtime) error { + if err := utils.ResetTmpDir(runtime); err != nil { + return err + } + + registryFile := fmt.Sprintf("registry-2-linux-%s", runtime.RemoteHost().GetArch()) + filesDir := filepath.Join(runtime.GetWorkDir(), "registry", runtime.RemoteHost().GetArch()) + if err := runtime.GetRunner().Scp(fmt.Sprintf("%s/%s.tar.gz", filesDir, registryFile), fmt.Sprintf("%s/%s.tar.gz", common.TmpDir, registryFile)); err != nil { + return errors.Wrap(errors.WithStack(err), "sync etcd tar.gz failed") + } + + installCmd := fmt.Sprintf("tar -zxf %s/%s.tar.gz && mv -f registry /usr/local/bin/ && chmod +x /usr/local/bin/registry", common.TmpDir, registryFile) + if _, err := runtime.GetRunner().SudoCmd(installCmd, false); err != nil { + return errors.Wrap(errors.WithStack(err), "install etcd binaries failed") + } + return nil +} + +type StartRegistryService struct { + common.KubeAction +} + +func (g *StartRegistryService) Execute(runtime connector.Runtime) error { + installCmd := "systemctl daemon-reload && systemctl enable registry && systemctl restart registry" + if _, err := runtime.GetRunner().SudoCmd(installCmd, false); err != nil { + return errors.Wrap(errors.WithStack(err), "start registry service failed") + } + + fmt.Println() + fmt.Println("Local image registry created successfully. Address: dockerhub.kubekey.local") + fmt.Println() + + return nil +} + +type InstallDockerCompose struct { + common.KubeAction +} + +func (g *InstallDockerCompose) Execute(runtime connector.Runtime) error { + if err := utils.ResetTmpDir(runtime); err != nil { + return err + } + + dockerComposeFile := "docker-compose-linux-x86_64" + filesDir := filepath.Join(runtime.GetWorkDir(), runtime.RemoteHost().GetArch()) + if err := runtime.GetRunner().Scp(fmt.Sprintf("%s/%s", filesDir, dockerComposeFile), fmt.Sprintf("%s/%s", common.TmpDir, dockerComposeFile)); err != nil { + return errors.Wrap(errors.WithStack(err), "sync docker-compose failed") + } + + installCmd := fmt.Sprintf("mv -f %s/%s /usr/local/bin/docker-compose && chmod +x /usr/local/bin/docker-compose", common.TmpDir, dockerComposeFile) + if _, err := runtime.GetRunner().SudoCmd(installCmd, false); err != nil { + return errors.Wrap(errors.WithStack(err), "install dokcer-compose failed") + } + + return nil +} + +type SyncHarborPackage struct { + common.KubeAction +} + +func (g *SyncHarborPackage) Execute(runtime connector.Runtime) error { + if err := utils.ResetTmpDir(runtime); err != nil { + return err + } + + harborPackageFile := fmt.Sprintf("harbor-offline-installer-%s.tgz", kubekeyapiv1alpha2.DefaultHarborVersion) + filesDir := filepath.Join(runtime.GetWorkDir(), "registry", runtime.RemoteHost().GetArch()) + if err := runtime.GetRunner().Scp(fmt.Sprintf("%s/%s", filesDir, harborPackageFile), fmt.Sprintf("%s/%s", common.TmpDir, harborPackageFile)); err != nil { + return errors.Wrap(errors.WithStack(err), "sync harbor package failed") + } + + installCmd := fmt.Sprintf("tar -zxvf %s/%s -C /opt", common.TmpDir, harborPackageFile) + if _, err := runtime.GetRunner().SudoCmd(installCmd, false); err != nil { + return errors.Wrap(errors.WithStack(err), "unzip harbor package failed") + } + + return nil +} + +type StartHarbor struct { + common.KubeAction +} + +func (g *StartHarbor) Execute(runtime connector.Runtime) error { + startCmd := "cd /opt/harbor && chmod +x install.sh && export PATH=$PATH:/usr/local/bin; ./install.sh --with-notary --with-trivy --with-chartmuseum" + if _, err := runtime.GetRunner().SudoCmd(startCmd, false); err != nil { + return errors.Wrap(errors.WithStack(err), "start harbor failed") + } + + fmt.Println() + fmt.Println("Local image registry created successfully. Address: dockerhub.kubekey.local") + fmt.Println() + + return nil +} diff --git a/pkg/bootstrap/registry/templates/harbor.go b/pkg/bootstrap/registry/templates/harbor.go new file mode 100644 index 00000000..8aca4546 --- /dev/null +++ b/pkg/bootstrap/registry/templates/harbor.go @@ -0,0 +1,119 @@ +/* +Copyright 2022 The KubeSphere Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package templates + +import ( + "github.com/lithammer/dedent" + "text/template" +) + +var ( + // HarborConfigTempl defines the template of registry's configuration file. + HarborConfigTempl = template.Must(template.New("harborConfig").Parse( + dedent.Dedent(`# Configuration file of Harbor + +# The IP address or hostname to access admin UI and registry service. +# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients. +hostname: {{ .Domain }} + +# http related config +http: + # port for http, default is 80. If https enabled, this port will redirect to https port + port: 80 + +# https related config +https: + # https port for harbor, default is 443 + port: 443 + # The path of cert and key files for nginx + certificate: /etc/ssl/registry/ssl/{{ .Certificate }} + private_key: /etc/ssl/registry/ssl/{{ .Key }} + +# The initial password of Harbor admin +# It only works in first time to install harbor +# Remember Change the admin password from UI after launching Harbor. +harbor_admin_password: Harbor12345 + +# Harbor DB configuration +database: + # The password for the root user of Harbor DB. Change this before any production use. + password: root123 + # The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained. + max_idle_conns: 100 + # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections. + # Note: the default number of connections is 1024 for postgres of harbor. + max_open_conns: 900 + +# The default data volume +data_volume: /mnt/registry + +# Trivy configuration +# +# Trivy DB contains vulnerability information from NVD, Red Hat, and many other upstream vulnerability databases. +# It is downloaded by Trivy from the GitHub release page https://github.com/aquasecurity/trivy-db/releases and cached +# in the local file system. In addition, the database contains the update timestamp so Trivy can detect whether it +# should download a newer version from the Internet or use the cached one. Currently, the database is updated every +# 12 hours and published as a new release to GitHub. +trivy: + # ignoreUnfixed The flag to display only fixed vulnerabilities + ignore_unfixed: false + # skipUpdate The flag to enable or disable Trivy DB downloads from GitHub + # + skip_update: false + # + # insecure The flag to skip verifying registry certificate + insecure: false + +jobservice: + # Maximum number of job workers in job service + max_job_workers: 10 + +notification: + # Maximum retry count for webhook job + webhook_job_max_retry: 10 + +chart: + # Change the value of absolute_url to enabled can enable absolute url in chart + absolute_url: disabled + +# Log configurations +log: + # options are debug, info, warning, error, fatal + level: info + # configs for logs in local storage + local: + # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated. + rotate_count: 50 + # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. + # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G + # are all valid. + rotate_size: 200M + # The directory on your host that store log + location: /var/log/harbor + +#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY! +_version: 2.4.0 + +# Global proxy +proxy: + http_proxy: + https_proxy: + no_proxy: + components: + - core + - jobservice + - trivy + + `))) +) diff --git a/pkg/bootstrap/registry/templates/registry.go b/pkg/bootstrap/registry/templates/registry.go new file mode 100644 index 00000000..527935f1 --- /dev/null +++ b/pkg/bootstrap/registry/templates/registry.go @@ -0,0 +1,52 @@ +/* +Copyright 2022 The KubeSphere Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package templates + +import ( + "github.com/lithammer/dedent" + "text/template" +) + +var ( + // RegistryServiceTempl defines the template of registry service for systemd. + RegistryServiceTempl = template.Must(template.New("registryService").Parse( + dedent.Dedent(`[Unit] +Description=v2 Registry server for Container +After=network.target +[Service] +Type=simple +ExecStart=/usr/local/bin/registry serve /etc/kubekey/registry/config.yaml +Restart=on-failure +[Install] +WantedBy=multi-user.target + `))) + + // RegistryConfigTempl defines the template of registry's configuration file. + RegistryConfigTempl = template.Must(template.New("registryConfig").Parse( + dedent.Dedent(`version: 0.1 +log: + fields: + service: registry +storage: + cache: + layerinfo: inmemory + filesystem: + rootdirectory: /mnt/registry +http: + addr: :443 + tls: + certificate: /etc/ssl/registry/ssl/{{ .Certificate }} + key: /etc/ssl/registry/ssl/{{ .Key }} + `))) +) diff --git a/pkg/common/common.go b/pkg/common/common.go index 62090f5d..a30e3a2b 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -26,11 +26,13 @@ const ( File = "file" Operator = "operator" - Master = "master" - Worker = "worker" - ETCD = "etcd" - K8s = "k8s" - KubeKey = "kubekey" + Master = "master" + Worker = "worker" + ETCD = "etcd" + K8s = "k8s" + Registry = "registry" + KubeKey = "kubekey" + Harbor = "harbor" KubeBinaries = "KubeBinaries" @@ -43,7 +45,8 @@ const ( KubeScriptDir = "/usr/local/bin/kube-scripts" KubeletFlexvolumesPluginsDir = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec" - ETCDCertDir = "/etc/ssl/etcd/ssl" + ETCDCertDir = "/etc/ssl/etcd/ssl" + RegistryCertDir = "/etc/ssl/registry/ssl" HaproxyDir = "/etc/kubekey/haproxy" diff --git a/pkg/common/kube_runtime.go b/pkg/common/kube_runtime.go index b7c5eb80..107e0a9f 100644 --- a/pkg/common/kube_runtime.go +++ b/pkg/common/kube_runtime.go @@ -85,6 +85,9 @@ func NewKubeRuntime(flag string, arg Argument) (*KubeRuntime, error) { if v.IsMaster || v.IsWorker { host.SetRole(K8s) } + if v.IsRegistry { + host.SetRole(Registry) + } base.AppendHost(host) base.AppendRoleMap(host) } diff --git a/pkg/common/loader.go b/pkg/common/loader.go index cfd40d3e..712a0e56 100644 --- a/pkg/common/loader.go +++ b/pkg/common/loader.go @@ -101,9 +101,10 @@ func (d *DefaultLoader) Load() (*kubekeyapiv1alpha2.Cluster, error) { }) allInOne.Spec.RoleGroups = kubekeyapiv1alpha2.RoleGroups{ - Etcd: []string{hostname}, - Master: []string{hostname}, - Worker: []string{hostname}, + Etcd: []string{hostname}, + Master: []string{hostname}, + Worker: []string{hostname}, + Registry: []string{hostname}, } if d.KubernetesVersion != "" { s := strings.Split(d.KubernetesVersion, "-") diff --git a/pkg/config/templates/cluster.go b/pkg/config/templates/cluster.go index 6fdb4edf..f9b6e14c 100644 --- a/pkg/config/templates/cluster.go +++ b/pkg/config/templates/cluster.go @@ -59,6 +59,7 @@ spec: # multus support. https://github.com/k8snetworkplumbingwg/multus-cni enableMultusCNI: false registry: + privateRegistry: "" registryMirrors: [] insecureRegistries: [] addons: [] diff --git a/pkg/container/templates/containerd_config.go b/pkg/container/templates/containerd_config.go index 691a214f..d26a9196 100644 --- a/pkg/container/templates/containerd_config.go +++ b/pkg/container/templates/containerd_config.go @@ -84,5 +84,4 @@ state = "/run/containerd" password = "{{$entry.Password}}" {{- end}} {{- end}} - `))) diff --git a/pkg/etcd/certs.go b/pkg/etcd/certs.go index 6a689ebd..8b08ae1b 100644 --- a/pkg/etcd/certs.go +++ b/pkg/etcd/certs.go @@ -82,7 +82,7 @@ func KubekeyCertEtcdMember(hostname string, altNames *certutil.AltNames) *certs. } } -// KubekeyCertEtcdMember is the definition of the cert for etcd client. +// KubekeyCertEtcdClient is the definition of the cert for etcd client. func KubekeyCertEtcdClient(hostname string, altNames *certutil.AltNames) *certs.KubekeyCert { l := strings.Split(hostname, ".") return &certs.KubekeyCert{ diff --git a/pkg/files/file.go b/pkg/files/file.go index cd9e4af3..71a08ba5 100644 --- a/pkg/files/file.go +++ b/pkg/files/file.go @@ -23,17 +23,20 @@ import ( ) const ( - kubeadm = "kubeadm" - kubelet = "kubelet" - kubectl = "kubectl" - kubecni = "kubecni" - etcd = "etcd" - helm = "helm" - amd64 = "amd64" - arm64 = "arm64" - k3s = "k3s" - docker = "docker" - crictl = "crictl" + kubeadm = "kubeadm" + kubelet = "kubelet" + kubectl = "kubectl" + kubecni = "kubecni" + etcd = "etcd" + helm = "helm" + amd64 = "amd64" + arm64 = "arm64" + k3s = "k3s" + docker = "docker" + crictl = "crictl" + registry = "registry" + harbor = "harbor" + compose = "compose" ) type KubeBinary struct { @@ -362,6 +365,21 @@ var ( "v1.22.0": "a713c37fade0d96a989bc15ebe906e08ef5c8fe5e107c2161b0665e9963b770e", }, }, + registry: { + amd64: { + "2": "7706e46674fa2cf20f734dfb7e4dd7f1390710e9c0a2c520563e3c55f3e4b5c5", + }, + }, + compose: { + amd64: { + "v2.2.2": "92551cd3d22b41536ce8345fe06795ad0d08cb3c17b693ecbfe41176e501bfd4", + }, + }, + harbor: { + amd64: { + "v2.4.1": "cfd799c150b59353aefb34835f3a2e859763cb2e91966cd3ffeb1b6ceaa19841", + }, + }, } ) diff --git a/pkg/pipelines/add_nodes.go b/pkg/pipelines/add_nodes.go index 12f2f6c2..a7543c2e 100644 --- a/pkg/pipelines/add_nodes.go +++ b/pkg/pipelines/add_nodes.go @@ -23,6 +23,7 @@ import ( "github.com/kubesphere/kubekey/pkg/bootstrap/confirm" "github.com/kubesphere/kubekey/pkg/bootstrap/os" "github.com/kubesphere/kubekey/pkg/bootstrap/precheck" + "github.com/kubesphere/kubekey/pkg/bootstrap/registry" "github.com/kubesphere/kubekey/pkg/certs" "github.com/kubesphere/kubekey/pkg/common" "github.com/kubesphere/kubekey/pkg/container" @@ -43,6 +44,7 @@ func NewAddNodesPipeline(runtime *common.KubeRuntime) error { &confirm.InstallConfirmModule{Skip: runtime.Arg.SkipConfirmCheck}, &binaries.NodeBinariesModule{}, &os.ConfigureOSModule{}, + ®istry.RegistryCertsModule{Skip: len(runtime.GetHostsByRole(common.Registry)) == 0}, &kubernetes.StatusModule{}, &container.InstallContainerModule{}, &images.PullModule{Skip: runtime.Arg.SkipPullImages}, diff --git a/pkg/pipelines/init_registry.go b/pkg/pipelines/init_registry.go new file mode 100644 index 00000000..f34e22f2 --- /dev/null +++ b/pkg/pipelines/init_registry.go @@ -0,0 +1,72 @@ +/* + Copyright 2022 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package pipelines + +import ( + "fmt" + "github.com/kubesphere/kubekey/pkg/binaries" + "github.com/kubesphere/kubekey/pkg/bootstrap/os" + "github.com/kubesphere/kubekey/pkg/bootstrap/registry" + "github.com/kubesphere/kubekey/pkg/common" + "github.com/kubesphere/kubekey/pkg/core/module" + "github.com/kubesphere/kubekey/pkg/core/pipeline" +) + +func NewInitRegistryPipeline(runtime *common.KubeRuntime) error { + m := []module.Module{ + &binaries.RegistryPackageModule{}, + &os.ConfigureOSModule{}, + ®istry.RegistryCertsModule{}, + ®istry.InstallRegistryModule{}, + } + + p := pipeline.Pipeline{ + Name: "InitRegistryPipeline", + Modules: m, + Runtime: runtime, + } + if err := p.Start(); err != nil { + return err + } + return nil +} + +func InitRegistry(args common.Argument, downloadCmd string) error { + args.DownloadCommand = func(path, url string) string { + // this is an extension point for downloading tools, for example users can set the timeout, proxy or retry under + // some poor network environment. Or users even can choose another cli, it might be wget. + // perhaps we should have a build-in download function instead of totally rely on the external one + return fmt.Sprintf(downloadCmd, path, url) + } + + var loaderType string + if args.FilePath != "" { + loaderType = common.File + } else { + loaderType = common.AllInOne + } + + runtime, err := common.NewKubeRuntime(loaderType, args) + if err != nil { + return err + } + + if err := NewInitRegistryPipeline(runtime); err != nil { + return err + } + return nil +}