From dbe5027cf01dca4a7e0303d813cea5cdb2bb828a Mon Sep 17 00:00:00 2001 From: pixiake Date: Sun, 20 Mar 2022 21:53:54 +0800 Subject: [PATCH] Support the use of kubeadm to manage etcd and use of existing etcd Signed-off-by: pixiake --- apis/kubekey/v1alpha2/cluster_types.go | 11 +- apis/kubekey/v1alpha2/default.go | 33 ++++-- apis/kubekey/v1alpha2/etcd_types.go | 48 ++++++++ apis/kubekey/v1alpha2/kubernetes_types.go | 4 - .../kubekey/v1alpha2/zz_generated.deepcopy.go | 17 +++ cmd/ctl/create/cluster.go | 3 - .../bases/kubekey.kubesphere.io_clusters.yaml | 67 ++++++++--- docs/config-example.md | 10 ++ pkg/bootstrap/os/module.go | 11 +- pkg/bootstrap/os/prepares.go | 13 +++ pkg/bootstrap/os/tasks.go | 11 +- pkg/bootstrap/registry/certs.go | 6 +- pkg/common/kube_runtime.go | 1 - pkg/core/connector/runtime.go | 6 +- pkg/core/module/task_module.go | 1 - pkg/core/task/remote_task.go | 11 -- pkg/etcd/certs.go | 107 +++++++++++++----- pkg/etcd/module.go | 61 +++++++++- pkg/etcd/tasks.go | 14 +-- pkg/etcd/templates/backup_script.go | 6 +- pkg/images/tasks.go | 1 + pkg/k3s/tasks.go | 41 ++++--- pkg/k3s/templates/k3sServiceEnv.go | 6 + pkg/kubernetes/tasks.go | 45 +++++--- .../templates/v1beta2/kubeadm_config.go | 18 ++- pkg/kubesphere/tasks.go | 99 +++++++++++++--- pkg/pipelines/add_nodes.go | 17 +-- pkg/pipelines/create_cluster.go | 17 +-- pkg/version/kubesphere/templates/cc_v320.go | 1 + pkg/version/kubesphere/templates/cc_v321.go | 3 +- 30 files changed, 517 insertions(+), 172 deletions(-) create mode 100644 apis/kubekey/v1alpha2/etcd_types.go diff --git a/apis/kubekey/v1alpha2/cluster_types.go b/apis/kubekey/v1alpha2/cluster_types.go index 9f83cbee..aef35a46 100644 --- a/apis/kubekey/v1alpha2/cluster_types.go +++ b/apis/kubekey/v1alpha2/cluster_types.go @@ -44,6 +44,7 @@ type ClusterSpec struct { RoleGroups map[string][]string `yaml:"roleGroups" json:"roleGroups,omitempty"` ControlPlaneEndpoint ControlPlaneEndpoint `yaml:"controlPlaneEndpoint" json:"controlPlaneEndpoint,omitempty"` System System `yaml:"system" json:"system,omitempty"` + Etcd EtcdCluster `yaml:"etcd" json:"etcd,omitempty"` Kubernetes Kubernetes `yaml:"kubernetes" json:"kubernetes,omitempty"` Network NetworkConfig `yaml:"network" json:"network,omitempty"` Registry RegistryConfig `yaml:"registry" json:"registry,omitempty"` @@ -186,14 +187,6 @@ type KubeSphere struct { Configurations string `json:"configurations,omitempty"` } -// ExternalEtcd defines configuration information of external etcd. -type ExternalEtcd struct { - Endpoints []string - CaFile string - CertFile string - KeyFile string -} - // GenerateCertSANs is used to generate cert sans for cluster. func (cfg *ClusterSpec) GenerateCertSANs() []string { clusterSvc := fmt.Sprintf("kubernetes.default.svc.%s", cfg.Kubernetes.DNSDomain) @@ -239,7 +232,7 @@ func (cfg *ClusterSpec) GroupHosts() map[string][]*KubeHost { if len(roleGroups[Master]) == 0 && len(roleGroups[ControlPlane]) == 0 { logger.Log.Fatal(errors.New("The number of master/control-plane cannot be 0")) } - if len(roleGroups[Etcd]) == 0 { + if len(roleGroups[Etcd]) == 0 && cfg.Etcd.Type == KubeKey { logger.Log.Fatal(errors.New("The number of etcd cannot be 0")) } if len(roleGroups[Registry]) > 1 { diff --git a/apis/kubekey/v1alpha2/default.go b/apis/kubekey/v1alpha2/default.go index d4ca8821..47732739 100644 --- a/apis/kubekey/v1alpha2/default.go +++ b/apis/kubekey/v1alpha2/default.go @@ -92,6 +92,7 @@ func (cfg *ClusterSpec) SetDefaultClusterSpec(incluster bool) (*ClusterSpec, map clusterCfg.Hosts = SetDefaultHostsCfg(cfg) clusterCfg.RoleGroups = cfg.RoleGroups + clusterCfg.Etcd = SetDefaultEtcdCfg(cfg) roleGroups := clusterCfg.GroupHosts() clusterCfg.ControlPlaneEndpoint = SetDefaultLBCfg(cfg, roleGroups[Master], incluster) clusterCfg.Network = SetDefaultNetworkCfg(cfg) @@ -254,18 +255,6 @@ func SetDefaultClusterCfg(cfg *ClusterSpec) Kubernetes { if cfg.Kubernetes.DNSDomain == "" { cfg.Kubernetes.DNSDomain = DefaultDNSDomain } - if cfg.Kubernetes.EtcdBackupDir == "" { - cfg.Kubernetes.EtcdBackupDir = DefaultEtcdBackupDir - } - if cfg.Kubernetes.EtcdBackupPeriod == 0 { - cfg.Kubernetes.EtcdBackupPeriod = DefaultEtcdBackupPeriod - } - if cfg.Kubernetes.KeepBackupNumber == 0 { - cfg.Kubernetes.KeepBackupNumber = DefaultKeepBackNumber - } - if cfg.Kubernetes.EtcdBackupScriptDir == "" { - cfg.Kubernetes.EtcdBackupScriptDir = DefaultEtcdBackupScriptDir - } if cfg.Kubernetes.ContainerManager == "" { cfg.Kubernetes.ContainerManager = Docker } @@ -287,3 +276,23 @@ func SetDefaultClusterCfg(cfg *ClusterSpec) Kubernetes { return defaultClusterCfg } + +func SetDefaultEtcdCfg(cfg *ClusterSpec) EtcdCluster { + if cfg.Etcd.Type == "" || ((cfg.Kubernetes.Type == "k3s" || (len(strings.Split(cfg.Kubernetes.Version, "-")) > 1) && strings.Split(cfg.Kubernetes.Version, "-")[1] == "k3s") && cfg.Etcd.Type == Kubeadm) { + cfg.Etcd.Type = KubeKey + } + if cfg.Etcd.BackupDir == "" { + cfg.Etcd.BackupDir = DefaultEtcdBackupDir + } + if cfg.Etcd.BackupPeriod == 0 { + cfg.Etcd.BackupPeriod = DefaultEtcdBackupPeriod + } + if cfg.Etcd.KeepBackupNumber == 0 { + cfg.Etcd.KeepBackupNumber = DefaultKeepBackNumber + } + if cfg.Etcd.BackupScriptDir == "" { + cfg.Etcd.BackupScriptDir = DefaultEtcdBackupScriptDir + } + + return cfg.Etcd +} diff --git a/apis/kubekey/v1alpha2/etcd_types.go b/apis/kubekey/v1alpha2/etcd_types.go new file mode 100644 index 00000000..0d2375d9 --- /dev/null +++ b/apis/kubekey/v1alpha2/etcd_types.go @@ -0,0 +1,48 @@ +/* + Copyright 2021 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package v1alpha2 + +const ( + KubeKey = "kubekey" + Kubeadm = "kubeadm" + External = "external" +) + +type EtcdCluster struct { + // Type of etcd cluster, can be set to 'kubekey' 'kubeadm' 'external' + Type string `yaml:"type" json:"type,omitempty"` + // ExternalEtcd describes how to connect to an external etcd cluster when type is set to external + External ExternalEtcd `yaml:"external" json:"external,omitempty"` + BackupDir string `yaml:"backupDir" json:"backupDir,omitempty"` + BackupPeriod int `yaml:"backupPeriod" json:"backupPeriod,omitempty"` + KeepBackupNumber int `yaml:"keepBackupNumber" json:"keepBackupNumber,omitempty"` + BackupScriptDir string `yaml:"backupScript" json:"backupScript,omitempty"` +} + +// ExternalEtcd describes how to connect to an external etcd cluster +// KubeKey, Kubeadm and External are mutually exclusive +type ExternalEtcd struct { + // Endpoints of etcd members. Useful for using external etcd. + // If not provided, kubeadm will run etcd in a static pod. + Endpoints []string `yaml:"endpoints" json:"endpoints,omitempty"` + // CAFile is an SSL Certificate Authority file used to secure etcd communication. + CAFile string `yaml:"caFile" json:"caFile,omitempty"` + // CertFile is an SSL certification file used to secure etcd communication. + CertFile string `yaml:"certFile" json:"certFile,omitempty"` + // KeyFile is an SSL key file used to secure etcd communication. + KeyFile string `yaml:"keyFile" json:"keyFile,omitempty"` +} diff --git a/apis/kubekey/v1alpha2/kubernetes_types.go b/apis/kubekey/v1alpha2/kubernetes_types.go index 86d133be..b69cefc9 100644 --- a/apis/kubekey/v1alpha2/kubernetes_types.go +++ b/apis/kubekey/v1alpha2/kubernetes_types.go @@ -31,10 +31,6 @@ type Kubernetes struct { ProxyMode string `yaml:"proxyMode" json:"proxyMode,omitempty"` // +optional Nodelocaldns *bool `yaml:"nodelocaldns" json:"nodelocaldns,omitempty"` - EtcdBackupDir string `yaml:"etcdBackupDir" json:"etcdBackupDir,omitempty"` - EtcdBackupPeriod int `yaml:"etcdBackupPeriod" json:"etcdBackupPeriod,omitempty"` - KeepBackupNumber int `yaml:"keepBackupNumber" json:"keepBackupNumber,omitempty"` - EtcdBackupScriptDir string `yaml:"etcdBackupScript" json:"etcdBackupScript,omitempty"` ContainerManager string `yaml:"containerManager" json:"containerManager,omitempty"` ContainerRuntimeEndpoint string `yaml:"containerRuntimeEndpoint" json:"containerRuntimeEndpoint,omitempty"` NodeFeatureDiscovery NodeFeatureDiscovery `yaml:"nodeFeatureDiscovery" json:"nodeFeatureDiscovery,omitempty"` diff --git a/apis/kubekey/v1alpha2/zz_generated.deepcopy.go b/apis/kubekey/v1alpha2/zz_generated.deepcopy.go index 0d65089b..2cfa25ff 100644 --- a/apis/kubekey/v1alpha2/zz_generated.deepcopy.go +++ b/apis/kubekey/v1alpha2/zz_generated.deepcopy.go @@ -177,6 +177,7 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { } out.ControlPlaneEndpoint = in.ControlPlaneEndpoint in.System.DeepCopyInto(&out.System) + in.Etcd.DeepCopyInto(&out.Etcd) in.Kubernetes.DeepCopyInto(&out.Kubernetes) in.Network.DeepCopyInto(&out.Network) in.Registry.DeepCopyInto(&out.Registry) @@ -402,6 +403,22 @@ func (in *ETCD) DeepCopy() *ETCD { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdCluster) DeepCopyInto(out *EtcdCluster) { + *out = *in + in.External.DeepCopyInto(&out.External) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdCluster. +func (in *EtcdCluster) DeepCopy() *EtcdCluster { + if in == nil { + return nil + } + out := new(EtcdCluster) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Event) DeepCopyInto(out *Event) { *out = *in diff --git a/cmd/ctl/create/cluster.go b/cmd/ctl/create/cluster.go index 64b4f149..83d1161b 100644 --- a/cmd/ctl/create/cluster.go +++ b/cmd/ctl/create/cluster.go @@ -42,7 +42,6 @@ type CreateClusterOptions struct { DownloadCmd string Artifact string InstallPackages bool - CertificatesDir string localStorageChanged bool } @@ -118,7 +117,6 @@ func (o *CreateClusterOptions) Run() error { ContainerManager: o.ContainerManager, Artifact: o.Artifact, InstallPackages: o.InstallPackages, - CertificatesDir: o.CertificatesDir, } if o.localStorageChanged { @@ -137,7 +135,6 @@ func (o *CreateClusterOptions) AddFlags(cmd *cobra.Command) { cmd.Flags().BoolVarP(&o.SkipPullImages, "skip-pull-images", "", false, "Skip pre pull images") cmd.Flags().BoolVarP(&o.SkipPushImages, "skip-push-images", "", false, "Skip pre push images") cmd.Flags().StringVarP(&o.ContainerManager, "container-manager", "", "docker", "Container runtime: docker, crio, containerd and isula.") - cmd.Flags().StringVarP(&o.CertificatesDir, "certificates-dir", "", "", "Specifies where to store or look for all required certificates.") cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s", `The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`) cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a KubeKey artifact") diff --git a/config/crd/bases/kubekey.kubesphere.io_clusters.yaml b/config/crd/bases/kubekey.kubesphere.io_clusters.yaml index 53e10291..65ef0eef 100644 --- a/config/crd/bases/kubekey.kubesphere.io_clusters.yaml +++ b/config/crd/bases/kubekey.kubesphere.io_clusters.yaml @@ -430,6 +430,45 @@ spec: port: type: integer type: object + etcd: + properties: + backupDir: + type: string + backupPeriod: + type: integer + backupScript: + type: string + external: + description: ExternalEtcd describes how to connect to an external + etcd cluster when type is set to external + properties: + caFile: + description: CAFile is an SSL Certificate Authority file used + to secure etcd communication. + type: string + certFile: + description: CertFile is an SSL certification file used to + secure etcd communication. + type: string + endpoints: + description: Endpoints of etcd members. Useful for using external + etcd. If not provided, kubeadm will run etcd in a static + pod. + items: + type: string + type: array + keyFile: + description: KeyFile is an SSL key file used to secure etcd + communication. + type: string + type: object + keepBackupNumber: + type: integer + type: + description: Type of etcd cluster, can be set to 'kubekey' 'kubeadm' + 'external' + type: string + type: object hosts: description: Foo is an example field of Cluster. Edit Cluster_types.go to remove/update @@ -440,13 +479,12 @@ spec: type: string arch: type: string - id: - type: string internalAddress: type: string labels: additionalProperties: type: string + description: Labels defines the kubernetes labels for the node. type: object name: type: string @@ -485,25 +523,16 @@ spec: type: array dnsDomain: type: string - etcdBackupDir: - type: string - etcdBackupPeriod: - type: integer - etcdBackupScript: - type: string featureGates: additionalProperties: type: boolean type: object kata: - description: NodeFeatureDiscovery contains the configuration for - the node-feature-discovery in cluster + description: Kata contains the configuration for the kata in cluster properties: enabled: type: boolean type: object - keepBackupNumber: - type: integer kubeProxyArgs: items: type: string @@ -523,7 +552,8 @@ spec: nodeCidrMaskSize: type: integer nodeFeatureDiscovery: - description: Kata contains the configuration for the kata in cluster + description: NodeFeatureDiscovery contains the configuration for + the node-feature-discovery in cluster properties: enabled: type: boolean @@ -563,8 +593,6 @@ spec: vxlanMode: type: string type: object - enableMultusCNI: - type: boolean flannel: properties: backendMode: @@ -605,6 +633,11 @@ spec: vlanInterfaceName: type: string type: object + multusCNI: + properties: + enabled: + type: boolean + type: object plugin: type: string type: object @@ -612,12 +645,14 @@ spec: description: RegistryConfig defines the configuration information of the image's repository. properties: - Auths: + auths: type: object insecureRegistries: items: type: string type: array + namespaceOverride: + type: string plainHTTP: type: boolean privateRegistry: diff --git a/docs/config-example.md b/docs/config-example.md index c8282dcd..19d0a13a 100644 --- a/docs/config-example.md +++ b/docs/config-example.md @@ -40,6 +40,16 @@ spec: ExpandCSIVolumes: true RotateKubeletServerCertificate: true TTLAfterFinished: true + etcd: + type: kubekey # Specify the type of etcd used by the cluster. When the cluster type is k3s, setting this parameter to kubeadm is invalid. [kubekey | kubeadm | external] [Default: kubekey] + ## The following parameters need to be added only when the type is set to external. + ## caFile, certFile and keyFile need not be set, if TLS authentication is not enabled for the existing etcd. + # external: + # endpoints: + # - https://192.168.6.6:2379 + # caFile: /pki/etcd/ca.crt + # certFile: /pki/etcd/etcd.crt + # keyFile: /pki/etcd/etcd.key network: plugin: calico calico: diff --git a/pkg/bootstrap/os/module.go b/pkg/bootstrap/os/module.go index 886b2c46..0012bc11 100644 --- a/pkg/bootstrap/os/module.go +++ b/pkg/bootstrap/os/module.go @@ -95,11 +95,12 @@ func (c *ClearOSEnvironmentModule) Init() { Parallel: true, } - stopETCD := &task.RemoteTask{ - Name: "StopETCDService", - Desc: "Stop etcd service", + uninstallETCD := &task.RemoteTask{ + Name: "UninstallETCD", + Desc: "Uninstall etcd", Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Action: new(StopETCDService), + Prepare: new(EtcdTypeIsKubeKey), + Action: new(UninstallETCD), Parallel: true, } @@ -121,7 +122,7 @@ func (c *ClearOSEnvironmentModule) Init() { c.Tasks = []task.Interface{ resetNetworkConfig, - stopETCD, + uninstallETCD, removeFiles, daemonReload, } diff --git a/pkg/bootstrap/os/prepares.go b/pkg/bootstrap/os/prepares.go index 19491205..f1837196 100644 --- a/pkg/bootstrap/os/prepares.go +++ b/pkg/bootstrap/os/prepares.go @@ -17,6 +17,7 @@ package os import ( + kubekeyv1alpha2 "github.com/kubesphere/kubekey/apis/kubekey/v1alpha2" "github.com/kubesphere/kubekey/pkg/common" "github.com/kubesphere/kubekey/pkg/core/connector" ) @@ -33,3 +34,15 @@ func (n *NodeConfigureNtpCheck) PreCheck(_ connector.Runtime) (bool, error) { return true, nil } + +type EtcdTypeIsKubeKey struct { + common.KubePrepare +} + +func (e *EtcdTypeIsKubeKey) PreCheck(_ connector.Runtime) (bool, error) { + if len(e.KubeConf.Cluster.Etcd.Type) == 0 || e.KubeConf.Cluster.Etcd.Type == kubekeyv1alpha2.KubeKey { + return true, nil + } + + return false, nil +} diff --git a/pkg/bootstrap/os/tasks.go b/pkg/bootstrap/os/tasks.go index f9afd3a7..ed3f2acf 100644 --- a/pkg/bootstrap/os/tasks.go +++ b/pkg/bootstrap/os/tasks.go @@ -131,11 +131,13 @@ func (n *NodeExecScript) Execute(runtime connector.Runtime) error { } var ( - clusterFiles = []string{ + etcdFiles = []string{ "/usr/local/bin/etcd", "/etc/ssl/etcd", "/var/lib/etcd", "/etc/etcd.env", + } + clusterFiles = []string{ "/etc/kubernetes", "/etc/systemd/system/etcd.service", "/var/log/calico", @@ -177,12 +179,15 @@ func (r *ResetNetworkConfig) Execute(runtime connector.Runtime) error { return nil } -type StopETCDService struct { +type UninstallETCD struct { common.KubeAction } -func (s *StopETCDService) Execute(runtime connector.Runtime) error { +func (s *UninstallETCD) Execute(runtime connector.Runtime) error { _, _ = runtime.GetRunner().SudoCmd("systemctl stop etcd && exit 0", false) + for _, file := range etcdFiles { + _, _ = runtime.GetRunner().SudoCmd(fmt.Sprintf("rm -rf %s", file), true) + } return nil } diff --git a/pkg/bootstrap/registry/certs.go b/pkg/bootstrap/registry/certs.go index c57caadd..4784ce90 100644 --- a/pkg/bootstrap/registry/certs.go +++ b/pkg/bootstrap/registry/certs.go @@ -98,10 +98,8 @@ type GenerateCerts struct { } func (g *GenerateCerts) Execute(runtime connector.Runtime) error { - var pkiPath string - if g.KubeConf.Arg.CertificatesDir == "" { - pkiPath = fmt.Sprintf("%s/pki/registry", runtime.GetWorkDir()) - } + + pkiPath := fmt.Sprintf("%s/pki/registry", runtime.GetWorkDir()) var altName cert.AltNames diff --git a/pkg/common/kube_runtime.go b/pkg/common/kube_runtime.go index 827a0101..89a513a6 100644 --- a/pkg/common/kube_runtime.go +++ b/pkg/common/kube_runtime.go @@ -52,7 +52,6 @@ type Argument struct { KubeConfig string Artifact string InstallPackages bool - CertificatesDir string ImagesDir string } diff --git a/pkg/core/connector/runtime.go b/pkg/core/connector/runtime.go index fa93314c..d4110d8b 100644 --- a/pkg/core/connector/runtime.go +++ b/pkg/core/connector/runtime.go @@ -130,7 +130,11 @@ func (b *BaseRuntime) SetAllHosts(hosts []Host) { } func (b *BaseRuntime) GetHostsByRole(role string) []Host { - return b.roleHosts[role] + if _, ok := b.roleHosts[role]; ok { + return b.roleHosts[role] + } else { + return []Host{} + } } func (b *BaseRuntime) RemoteHost() Host { diff --git a/pkg/core/module/task_module.go b/pkg/core/module/task_module.go index ee5b6393..abe3148b 100644 --- a/pkg/core/module/task_module.go +++ b/pkg/core/module/task_module.go @@ -45,7 +45,6 @@ func (b *BaseTaskModule) Run(result *ending.ModuleResult) { t.Init(b.Runtime.(connector.Runtime), b.ModuleCache, b.PipelineCache) logger.Log.Infof("[%s] %s", b.Name, t.GetDesc()) - res := t.Execute() for j := range res.ActionResults { ac := res.ActionResults[j] diff --git a/pkg/core/task/remote_task.go b/pkg/core/task/remote_task.go index 4b7b34a7..7abd566d 100644 --- a/pkg/core/task/remote_task.go +++ b/pkg/core/task/remote_task.go @@ -312,21 +312,10 @@ func (t *RemoteTask) Default() { t.Name = DefaultTaskName } - if len(t.Hosts) < 1 { - t.Hosts = []connector.Host{} - t.TaskResult.AppendErr(nil, errors.New("the length of task hosts is 0")) - return - } - if t.Prepare == nil { t.Prepare = new(prepare.BasePrepare) } - if t.Action == nil { - t.TaskResult.AppendErr(nil, errors.New("the action is nil")) - return - } - if t.Retry <= 0 { t.Retry = 3 } diff --git a/pkg/etcd/certs.go b/pkg/etcd/certs.go index 8b08ae1b..d08e96d6 100644 --- a/pkg/etcd/certs.go +++ b/pkg/etcd/certs.go @@ -22,12 +22,15 @@ import ( kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/apis/kubekey/v1alpha2" "github.com/kubesphere/kubekey/pkg/common" "github.com/kubesphere/kubekey/pkg/core/connector" + "github.com/kubesphere/kubekey/pkg/core/util" "github.com/kubesphere/kubekey/pkg/utils/certs" "github.com/pkg/errors" + "io/ioutil" "k8s.io/client-go/util/cert" certutil "k8s.io/client-go/util/cert" netutils "k8s.io/utils/net" "net" + "os" "path/filepath" "strings" ) @@ -139,32 +142,10 @@ type GenerateCerts struct { } func (g *GenerateCerts) Execute(runtime connector.Runtime) error { - var pkiPath string - if g.KubeConf.Arg.CertificatesDir == "" { - pkiPath = fmt.Sprintf("%s/pki/etcd", runtime.GetWorkDir()) - } - var altName cert.AltNames + pkiPath := fmt.Sprintf("%s/pki/etcd", runtime.GetWorkDir()) - dnsList := []string{"localhost", "etcd.kube-system.svc.cluster.local", "etcd.kube-system.svc", "etcd.kube-system", "etcd"} - ipList := []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback} - - if g.KubeConf.Cluster.ControlPlaneEndpoint.Domain == "" { - dnsList = append(dnsList, kubekeyapiv1alpha2.DefaultLBDomain) - } else { - dnsList = append(dnsList, g.KubeConf.Cluster.ControlPlaneEndpoint.Domain) - } - - for _, host := range g.KubeConf.Cluster.Hosts { - dnsList = append(dnsList, host.Name) - internalAddress := netutils.ParseIPSloppy(host.InternalAddress) - if internalAddress != nil { - ipList = append(ipList, internalAddress) - } - } - - altName.DNSNames = dnsList - altName.IPs = ipList + altName := GenerateAltName(g.KubeConf, &runtime) files := []string{"ca.pem", "ca-key.pem"} @@ -174,13 +155,13 @@ func (g *GenerateCerts) Execute(runtime connector.Runtime) error { // Certs for _, host := range runtime.GetAllHosts() { if host.IsRole(common.ETCD) { - certsList = append(certsList, KubekeyCertEtcdAdmin(host.GetName(), &altName)) + certsList = append(certsList, KubekeyCertEtcdAdmin(host.GetName(), altName)) files = append(files, []string{fmt.Sprintf("admin-%s.pem", host.GetName()), fmt.Sprintf("admin-%s-key.pem", host.GetName())}...) - certsList = append(certsList, KubekeyCertEtcdMember(host.GetName(), &altName)) + certsList = append(certsList, KubekeyCertEtcdMember(host.GetName(), altName)) files = append(files, []string{fmt.Sprintf("member-%s.pem", host.GetName()), fmt.Sprintf("member-%s-key.pem", host.GetName())}...) } if host.IsRole(common.Master) { - certsList = append(certsList, KubekeyCertEtcdClient(host.GetName(), &altName)) + certsList = append(certsList, KubekeyCertEtcdClient(host.GetName(), altName)) files = append(files, []string{fmt.Sprintf("node-%s.pem", host.GetName()), fmt.Sprintf("node-%s-key.pem", host.GetName())}...) } } @@ -206,3 +187,75 @@ func (g *GenerateCerts) Execute(runtime connector.Runtime) error { return nil } + +func GenerateAltName(k *common.KubeConf, runtime *connector.Runtime) *cert.AltNames { + var altName cert.AltNames + + dnsList := []string{"localhost", "etcd.kube-system.svc.cluster.local", "etcd.kube-system.svc", "etcd.kube-system", "etcd"} + ipList := []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback} + + if k.Cluster.ControlPlaneEndpoint.Domain == "" { + dnsList = append(dnsList, kubekeyapiv1alpha2.DefaultLBDomain) + } else { + dnsList = append(dnsList, k.Cluster.ControlPlaneEndpoint.Domain) + } + + for _, host := range k.Cluster.Hosts { + dnsList = append(dnsList, host.Name) + internalAddress := netutils.ParseIPSloppy(host.InternalAddress) + if internalAddress != nil { + ipList = append(ipList, internalAddress) + } + } + + altName.DNSNames = dnsList + altName.IPs = ipList + + return &altName +} + +type FetchCertsForExternalEtcd struct { + common.KubeAction +} + +func (f *FetchCertsForExternalEtcd) Execute(runtime connector.Runtime) error { + + pkiPath := fmt.Sprintf("%s/pki/etcd", runtime.GetWorkDir()) + + if err := util.CreateDir(pkiPath); err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to create dir %s", pkiPath)) + } + + srcCertsFiles := []string{f.KubeConf.Cluster.Etcd.External.CAFile, f.KubeConf.Cluster.Etcd.External.CertFile, f.KubeConf.Cluster.Etcd.External.KeyFile} + dstCertsFiles := []string{} + for _, certFile := range srcCertsFiles { + if len(certFile) != 0 { + certPath, err := filepath.Abs(certFile) + if err != nil { + return errors.Wrap(err, "bad certificate file path") + } + _, err = os.Stat(certPath) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("%s does not exist", certPath)) + } + + dstCertFileName := filepath.Base(certPath) + dstCert := fmt.Sprintf("%s/%s", pkiPath, dstCertFileName) + dstCertsFiles = append(dstCertsFiles, dstCertFileName) + + data, err := ioutil.ReadFile(certPath) + if err != nil { + return errors.Wrap(err, "failed to copy certificate content") + } + + if err := ioutil.WriteFile(dstCert, data, 0600); err != nil { + return errors.Wrap(err, "failed to copy certificate content") + } + } + } + + f.ModuleCache.Set(LocalCertsDir, pkiPath) + f.ModuleCache.Set(CertsFileList, dstCertsFiles) + + return nil +} diff --git a/pkg/etcd/module.go b/pkg/etcd/module.go index 9ed9e2e9..bbed1c41 100644 --- a/pkg/etcd/module.go +++ b/pkg/etcd/module.go @@ -17,6 +17,7 @@ package etcd import ( + kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/apis/kubekey/v1alpha2" "github.com/kubesphere/kubekey/pkg/common" "github.com/kubesphere/kubekey/pkg/core/action" "github.com/kubesphere/kubekey/pkg/core/task" @@ -25,6 +26,11 @@ import ( type PreCheckModule struct { common.KubeModule + Skip bool +} + +func (p *PreCheckModule) IsSkip() bool { + return p.Skip } func (p *PreCheckModule) Init() { @@ -46,16 +52,30 @@ func (p *PreCheckModule) Init() { type CertsModule struct { common.KubeModule + Skip bool +} + +func (p *CertsModule) IsSkip() bool { + return p.Skip } func (c *CertsModule) Init() { c.Name = "CertsModule" c.Desc = "Sign ETCD cluster certs" + switch c.KubeConf.Cluster.Etcd.Type { + case kubekeyapiv1alpha2.KubeKey: + c.Tasks = CertsModuleForKubeKey(c) + case kubekeyapiv1alpha2.External: + c.Tasks = CertsModuleForExternal(c) + } +} + +func CertsModuleForKubeKey(c *CertsModule) []task.Interface { // If the etcd cluster already exists, obtain the certificate in use from the etcd node. fetchCerts := &task.RemoteTask{ Name: "FetchETCDCerts", - Desc: "Fetcd etcd certs", + Desc: "Fetch etcd certs", Hosts: c.Runtime.GetHostsByRole(common.ETCD), Prepare: new(FirstETCDNode), Action: new(FetchCerts), @@ -87,7 +107,7 @@ func (c *CertsModule) Init() { Retry: 1, } - c.Tasks = []task.Interface{ + return []task.Interface{ fetchCerts, generateCerts, syncCertsFile, @@ -95,8 +115,35 @@ func (c *CertsModule) Init() { } } +func CertsModuleForExternal(c *CertsModule) []task.Interface { + fetchCerts := &task.LocalTask{ + Name: "FetchETCDCerts", + Desc: "Fetch etcd certs", + Action: new(FetchCertsForExternalEtcd), + } + + syncCertsToMaster := &task.RemoteTask{ + Name: "SyncCertsFileToMaster", + Desc: "Synchronize certs file to master", + Hosts: c.Runtime.GetHostsByRole(common.Master), + Action: new(SyncCertsFile), + Parallel: true, + Retry: 1, + } + + return []task.Interface{ + fetchCerts, + syncCertsToMaster, + } +} + type InstallETCDBinaryModule struct { common.KubeModule + Skip bool +} + +func (p *InstallETCDBinaryModule) IsSkip() bool { + return p.Skip } func (i *InstallETCDBinaryModule) Init() { @@ -143,6 +190,11 @@ func (i *InstallETCDBinaryModule) Init() { type ConfigureModule struct { common.KubeModule + Skip bool +} + +func (p *ConfigureModule) IsSkip() bool { + return p.Skip } func (e *ConfigureModule) Init() { @@ -316,6 +368,11 @@ func handleExistCluster(c *ConfigureModule) []task.Interface { type BackupModule struct { common.KubeModule + Skip bool +} + +func (p *BackupModule) IsSkip() bool { + return p.Skip } func (b *BackupModule) Init() { diff --git a/pkg/etcd/tasks.go b/pkg/etcd/tasks.go index 10e15bff..a52b3943 100644 --- a/pkg/etcd/tasks.go +++ b/pkg/etcd/tasks.go @@ -384,14 +384,14 @@ type BackupETCD struct { func (b *BackupETCD) Execute(runtime connector.Runtime) error { templateAction := action.Template{ Template: templates.EtcdBackupScript, - Dst: filepath.Join(b.KubeConf.Cluster.Kubernetes.EtcdBackupScriptDir, "etcd-backup.sh"), + Dst: filepath.Join(b.KubeConf.Cluster.Etcd.BackupScriptDir, "etcd-backup.sh"), Data: util.Data{ "Hostname": runtime.RemoteHost().GetName(), "Etcdendpoint": fmt.Sprintf("https://%s:2379", runtime.RemoteHost().GetInternalAddress()), - "Backupdir": b.KubeConf.Cluster.Kubernetes.EtcdBackupDir, - "KeepbackupNumber": b.KubeConf.Cluster.Kubernetes.KeepBackupNumber, - "EtcdBackupPeriod": b.KubeConf.Cluster.Kubernetes.EtcdBackupPeriod, - "EtcdBackupScriptDir": b.KubeConf.Cluster.Kubernetes.EtcdBackupScriptDir, + "Backupdir": b.KubeConf.Cluster.Etcd.BackupDir, + "KeepbackupNumber": b.KubeConf.Cluster.Etcd.KeepBackupNumber, + "EtcdBackupPeriod": b.KubeConf.Cluster.Etcd.BackupPeriod, + "EtcdBackupScriptDir": b.KubeConf.Cluster.Etcd.BackupScriptDir, "EtcdBackupHour": templates.BackupTimeInterval(runtime, b.KubeConf), }, } @@ -401,11 +401,11 @@ func (b *BackupETCD) Execute(runtime connector.Runtime) error { return err } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chmod +x %s/etcd-backup.sh", b.KubeConf.Cluster.Kubernetes.EtcdBackupScriptDir), false); err != nil { + if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chmod +x %s/etcd-backup.sh", b.KubeConf.Cluster.Etcd.BackupScriptDir), false); err != nil { return errors.Wrap(errors.WithStack(err), "chmod etcd backup script failed") } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("sh %s/etcd-backup.sh", b.KubeConf.Cluster.Kubernetes.EtcdBackupScriptDir), false); err != nil { + if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("sh %s/etcd-backup.sh", b.KubeConf.Cluster.Etcd.BackupScriptDir), false); err != nil { return errors.Wrap(errors.WithStack(err), "Failed to run the etcd-backup.sh") } return nil diff --git a/pkg/etcd/templates/backup_script.go b/pkg/etcd/templates/backup_script.go index 0bb4f261..2a80eed6 100644 --- a/pkg/etcd/templates/backup_script.go +++ b/pkg/etcd/templates/backup_script.go @@ -78,10 +78,10 @@ rm -rf /tmp/file func BackupTimeInterval(runtime connector.Runtime, kubeConf *common.KubeConf) string { var etcdBackupHour string - if kubeConf.Cluster.Kubernetes.EtcdBackupPeriod != 0 { - period := kubeConf.Cluster.Kubernetes.EtcdBackupPeriod + if kubeConf.Cluster.Etcd.BackupPeriod != 0 { + period := kubeConf.Cluster.Etcd.BackupPeriod if period > 60 && period < 1440 { - kubeConf.Cluster.Kubernetes.EtcdBackupPeriod = period % 60 + kubeConf.Cluster.Etcd.BackupPeriod = period % 60 etcdBackupHour = strconv.Itoa(period / 60) } if period > 1440 { diff --git a/pkg/images/tasks.go b/pkg/images/tasks.go index c02886d6..99a2161a 100644 --- a/pkg/images/tasks.go +++ b/pkg/images/tasks.go @@ -98,6 +98,7 @@ func GetImage(runtime connector.ModuleRuntime, kubeConf *common.KubeConf, name s ImageList := map[string]Image{ "pause": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "pause", Tag: pauseTag, Group: kubekeyv1alpha2.K8s, Enable: true}, + "etcd": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "etcd", Tag: kubekeyv1alpha2.DefaultEtcdVersion, Group: kubekeyv1alpha2.Master, Enable: strings.EqualFold(kubeConf.Cluster.Etcd.Type, kubekeyv1alpha2.Kubeadm)}, "kube-apiserver": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "kube-apiserver", Tag: kubeConf.Cluster.Kubernetes.Version, Group: kubekeyv1alpha2.Master, Enable: true}, "kube-controller-manager": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "kube-controller-manager", Tag: kubeConf.Cluster.Kubernetes.Version, Group: kubekeyv1alpha2.Master, Enable: true}, "kube-scheduler": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "kube-scheduler", Tag: kubeConf.Cluster.Kubernetes.Version, Group: kubekeyv1alpha2.Master, Enable: true}, diff --git a/pkg/k3s/tasks.go b/pkg/k3s/tasks.go index de10a9a3..05174a94 100644 --- a/pkg/k3s/tasks.go +++ b/pkg/k3s/tasks.go @@ -239,23 +239,30 @@ func (g *GenerateK3sServiceEnv) Execute(runtime connector.Runtime) error { var externalEtcd kubekeyapiv1alpha2.ExternalEtcd var endpointsList []string - var caFile, certFile, keyFile string - var token string + var externalEtcdEndpoints, token string - for _, node := range runtime.GetHostsByRole(common.ETCD) { - endpoint := fmt.Sprintf("https://%s:%s", node.GetInternalAddress(), kubekeyapiv1alpha2.DefaultEtcdPort) - endpointsList = append(endpointsList, endpoint) + switch g.KubeConf.Cluster.Etcd.Type { + case kubekeyapiv1alpha2.External: + externalEtcd.Endpoints = g.KubeConf.Cluster.Etcd.External.Endpoints + + if len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 && len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 && len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 { + externalEtcd.CAFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.CAFile)) + externalEtcd.CertFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.CertFile)) + externalEtcd.KeyFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.KeyFile)) + } + default: + for _, node := range runtime.GetHostsByRole(common.ETCD) { + endpoint := fmt.Sprintf("https://%s:%s", node.GetInternalAddress(), kubekeyapiv1alpha2.DefaultEtcdPort) + endpointsList = append(endpointsList, endpoint) + } + externalEtcd.Endpoints = endpointsList + + externalEtcd.CAFile = "/etc/ssl/etcd/ssl/ca.pem" + externalEtcd.CertFile = fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s.pem", runtime.GetHostsByRole(common.Master)[0].GetName()) + externalEtcd.KeyFile = fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s-key.pem", runtime.GetHostsByRole(common.Master)[0].GetName()) } - externalEtcd.Endpoints = endpointsList - externalEtcdEndpoints := strings.Join(endpointsList, ",") - caFile = "/etc/ssl/etcd/ssl/ca.pem" - certFile = fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s.pem", runtime.GetHostsByRole(common.Master)[0].GetName()) - keyFile = fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s-key.pem", runtime.GetHostsByRole(common.Master)[0].GetName()) - - externalEtcd.CaFile = caFile - externalEtcd.CertFile = certFile - externalEtcd.KeyFile = keyFile + externalEtcdEndpoints = strings.Join(externalEtcd.Endpoints, ",") if !host.IsRole(common.Master) { token = cluster.NodeToken @@ -266,9 +273,9 @@ func (g *GenerateK3sServiceEnv) Execute(runtime connector.Runtime) error { Dst: filepath.Join("/etc/systemd/system/", templates.K3sServiceEnv.Name()), Data: util.Data{ "DataStoreEndPoint": externalEtcdEndpoints, - "DataStoreCaFile": caFile, - "DataStoreCertFile": certFile, - "DataStoreKeyFile": keyFile, + "DataStoreCaFile": externalEtcd.CAFile, + "DataStoreCertFile": externalEtcd.CertFile, + "DataStoreKeyFile": externalEtcd.KeyFile, "IsMaster": host.IsRole(common.Master), "Token": token, }, diff --git a/pkg/k3s/templates/k3sServiceEnv.go b/pkg/k3s/templates/k3sServiceEnv.go index 0e5254fd..49e92614 100644 --- a/pkg/k3s/templates/k3sServiceEnv.go +++ b/pkg/k3s/templates/k3sServiceEnv.go @@ -26,9 +26,15 @@ var K3sServiceEnv = template.Must(template.New("k3s.service.env").Parse( dedent.Dedent(`# Note: This dropin only works with k3s {{ if .IsMaster }} K3S_DATASTORE_ENDPOINT={{ .DataStoreEndPoint }} +{{- if .DataStoreCaFile }} K3S_DATASTORE_CAFILE={{ .DataStoreCaFile }} +{{- end }} +{{- if .DataStoreCertFile }} K3S_DATASTORE_CERTFILE={{ .DataStoreCertFile }} +{{- end }} +{{- if .DataStoreKeyFile }} K3S_DATASTORE_KEYFILE={{ .DataStoreKeyFile }} +{{- end }} K3S_KUBECONFIG_MODE=644 {{ end }} {{ if .Token }} diff --git a/pkg/kubernetes/tasks.go b/pkg/kubernetes/tasks.go index 7d1946ca..49b5d13a 100644 --- a/pkg/kubernetes/tasks.go +++ b/pkg/kubernetes/tasks.go @@ -20,6 +20,7 @@ import ( "context" "encoding/base64" "fmt" + "github.com/kubesphere/kubekey/pkg/etcd" "os" "path/filepath" "sort" @@ -216,22 +217,34 @@ func (g *GenerateKubeadmConfig) Execute(runtime connector.Runtime) error { } else { // generate etcd configuration var externalEtcd kubekeyv1alpha2.ExternalEtcd - var endpointsList []string - var caFile, certFile, keyFile string + var endpointsList, etcdCertSANs []string - for _, host := range runtime.GetHostsByRole(common.ETCD) { - endpoint := fmt.Sprintf("https://%s:%s", host.GetInternalAddress(), kubekeyv1alpha2.DefaultEtcdPort) - endpointsList = append(endpointsList, endpoint) + switch g.KubeConf.Cluster.Etcd.Type { + case kubekeyv1alpha2.KubeKey: + for _, host := range runtime.GetHostsByRole(common.ETCD) { + endpoint := fmt.Sprintf("https://%s:%s", host.GetInternalAddress(), kubekeyv1alpha2.DefaultEtcdPort) + endpointsList = append(endpointsList, endpoint) + } + externalEtcd.Endpoints = endpointsList + + externalEtcd.CAFile = "/etc/ssl/etcd/ssl/ca.pem" + externalEtcd.CertFile = fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s.pem", host.GetName()) + externalEtcd.KeyFile = fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s-key.pem", host.GetName()) + case kubekeyv1alpha2.External: + externalEtcd.Endpoints = g.KubeConf.Cluster.Etcd.External.Endpoints + + if len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 && len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 && len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 { + externalEtcd.CAFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.CAFile)) + externalEtcd.CertFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.CertFile)) + externalEtcd.KeyFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.KeyFile)) + } + case kubekeyv1alpha2.Kubeadm: + altNames := etcd.GenerateAltName(g.KubeConf, &runtime) + etcdCertSANs = append(etcdCertSANs, altNames.DNSNames...) + for _, ip := range altNames.IPs { + etcdCertSANs = append(etcdCertSANs, string(ip)) + } } - externalEtcd.Endpoints = endpointsList - - caFile = "/etc/ssl/etcd/ssl/ca.pem" - certFile = fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s.pem", host.GetName()) - keyFile = fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s-key.pem", host.GetName()) - - externalEtcd.CaFile = caFile - externalEtcd.CertFile = certFile - externalEtcd.KeyFile = keyFile _, ApiServerArgs := util.GetArgs(v1beta2.ApiServerArgs, g.KubeConf.Cluster.Kubernetes.ApiServerArgs) _, ControllerManagerArgs := util.GetArgs(v1beta2.ControllermanagerArgs, g.KubeConf.Cluster.Kubernetes.ControllerManagerArgs) @@ -262,6 +275,10 @@ func (g *GenerateKubeadmConfig) Execute(runtime connector.Runtime) error { Data: util.Data{ "IsInitCluster": g.IsInitConfiguration, "ImageRepo": strings.TrimSuffix(images.GetImage(runtime, g.KubeConf, "kube-apiserver").ImageRepo(), "/kube-apiserver"), + "EtcdTypeIsKubeadm": g.KubeConf.Cluster.Etcd.Type == kubekeyv1alpha2.Kubeadm, + "EtcdCertSANs": etcdCertSANs, + "EtcdRepo": strings.TrimSuffix(images.GetImage(runtime, g.KubeConf, "etcd").ImageRepo(), "/etcd"), + "EtcdTag": images.GetImage(runtime, g.KubeConf, "etcd").Tag, "CorednsRepo": strings.TrimSuffix(images.GetImage(runtime, g.KubeConf, "coredns").ImageRepo(), "/coredns"), "CorednsTag": images.GetImage(runtime, g.KubeConf, "coredns").Tag, "Version": g.KubeConf.Cluster.Kubernetes.Version, diff --git a/pkg/kubernetes/templates/v1beta2/kubeadm_config.go b/pkg/kubernetes/templates/v1beta2/kubeadm_config.go index 7d13867a..71671143 100644 --- a/pkg/kubernetes/templates/v1beta2/kubeadm_config.go +++ b/pkg/kubernetes/templates/v1beta2/kubeadm_config.go @@ -41,14 +41,30 @@ var ( apiVersion: kubeadm.k8s.io/v1beta2 kind: ClusterConfiguration etcd: +{{- if .EtcdTypeIsKubeadm }} + local: + imageRepository: {{ .EtcdRepo }} + imageTag: {{ .EtcdTag }} + serverCertSANs: + {{- range .ExternalEtcd.Endpoints }} + - {{ . }} + {{- end }} +{{- else }} external: endpoints: {{- range .ExternalEtcd.Endpoints }} - {{ . }} {{- end }} - caFile: {{ .ExternalEtcd.CaFile }} +{{- if .ExternalEtcd.CAFile }} + caFile: {{ .ExternalEtcd.CAFile }} +{{- end }} +{{- if .ExternalEtcd.CertFile }} certFile: {{ .ExternalEtcd.CertFile }} +{{- end }} +{{- if .ExternalEtcd.KeyFile }} keyFile: {{ .ExternalEtcd.KeyFile }} +{{- end }} +{{- end }} dns: type: CoreDNS imageRepository: {{ .CorednsRepo }} diff --git a/pkg/kubesphere/tasks.go b/pkg/kubesphere/tasks.go index a23eda41..1c91c6b6 100644 --- a/pkg/kubesphere/tasks.go +++ b/pkg/kubesphere/tasks.go @@ -20,6 +20,7 @@ import ( "context" "encoding/base64" "fmt" + kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/apis/kubekey/v1alpha2" "github.com/kubesphere/kubekey/pkg/common" "github.com/kubesphere/kubekey/pkg/core/connector" "github.com/kubesphere/kubekey/pkg/core/logger" @@ -80,9 +81,75 @@ func (s *Setup) Execute(runtime connector.Runtime) error { filePath := filepath.Join(common.KubeAddonsDir, templates.KsInstaller.Name()) var addrList []string - for _, host := range runtime.GetHostsByRole(common.ETCD) { - addrList = append(addrList, host.GetInternalAddress()) + var tlsDisable bool + var port string + switch s.KubeConf.Cluster.Etcd.Type { + case kubekeyapiv1alpha2.KubeKey: + for _, host := range runtime.GetHostsByRole(common.ETCD) { + addrList = append(addrList, host.GetInternalAddress()) + } + + caFile := "/etc/ssl/etcd/ssl/ca.pem" + certFile := fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s.pem", runtime.GetHostsByRole(common.ETCD)[0].GetName()) + keyFile := fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s-key.pem", runtime.GetHostsByRole(common.ETCD)[0].GetName()) + if output, err := runtime.GetRunner().SudoCmd( + fmt.Sprintf("/usr/local/bin/kubectl -n kubesphere-monitoring-system create secret generic kube-etcd-client-certs "+ + "--from-file=etcd-client-ca.crt=%s "+ + "--from-file=etcd-client.crt=%s "+ + "--from-file=etcd-client.key=%s", caFile, certFile, keyFile), true); err != nil { + if !strings.Contains(output, "exists") { + return err + } + } + case kubekeyapiv1alpha2.Kubeadm: + for _, host := range runtime.GetHostsByRole(common.Master) { + addrList = append(addrList, host.GetInternalAddress()) + } + + caFile := "/etc/kubernetes/pki/etcd/ca.crt" + certFile := "/etc/kubernetes/pki/etcd/healthcheck-client.crt" + keyFile := "/etc/kubernetes/pki/etcd/healthcheck-client.key" + if output, err := runtime.GetRunner().SudoCmd( + fmt.Sprintf("/usr/local/bin/kubectl -n kubesphere-monitoring-system create secret generic kube-etcd-client-certs "+ + "--from-file=etcd-client-ca.crt=%s "+ + "--from-file=etcd-client.crt=%s "+ + "--from-file=etcd-client.key=%s", caFile, certFile, keyFile), true); err != nil { + if !strings.Contains(output, "exists") { + return err + } + } + case kubekeyapiv1alpha2.External: + for _, endpoint := range s.KubeConf.Cluster.Etcd.External.Endpoints { + e := strings.Split(strings.TrimSpace(endpoint), "://") + s := strings.Split(e[1], ":") + port = s[1] + addrList = append(addrList, s[0]) + if e[0] == "http" { + tlsDisable = true + } + } + if tlsDisable { + if output, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl -n kubesphere-monitoring-system create secret generic kube-etcd-client-certs", true); err != nil { + if !strings.Contains(output, "exists") { + return err + } + } + } else { + caFile := fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(s.KubeConf.Cluster.Etcd.External.CAFile)) + certFile := fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(s.KubeConf.Cluster.Etcd.External.CertFile)) + keyFile := fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(s.KubeConf.Cluster.Etcd.External.KeyFile)) + if output, err := runtime.GetRunner().SudoCmd( + fmt.Sprintf("/usr/local/bin/kubectl -n kubesphere-monitoring-system create secret generic kube-etcd-client-certs "+ + "--from-file=etcd-client-ca.crt=%s "+ + "--from-file=etcd-client.crt=%s "+ + "--from-file=etcd-client.key=%s", caFile, certFile, keyFile), true); err != nil { + if !strings.Contains(output, "exists") { + return err + } + } + } } + etcdEndPoint := strings.Join(addrList, ",") if _, err := runtime.GetRunner().SudoCmd( fmt.Sprintf("sed -i '/endpointIps/s/\\:.*/\\: %s/g' %s", etcdEndPoint, filePath), @@ -90,6 +157,22 @@ func (s *Setup) Execute(runtime connector.Runtime) error { return errors.Wrap(errors.WithStack(err), fmt.Sprintf("update etcd endpoint failed")) } + if tlsDisable { + if _, err := runtime.GetRunner().SudoCmd( + fmt.Sprintf("sed -i '/tlsEnable/s/\\:.*/\\: false/g' %s", filePath), + false); err != nil { + return errors.Wrap(errors.WithStack(err), fmt.Sprintf("update etcd tls failed")) + } + } + + if len(port) != 0 { + if _, err := runtime.GetRunner().SudoCmd( + fmt.Sprintf("sed -i 's/2379/%s/g' %s", port, filePath), + false); err != nil { + return errors.Wrap(errors.WithStack(err), fmt.Sprintf("update etcd tls failed")) + } + } + if s.KubeConf.Cluster.Registry.PrivateRegistry != "" { PrivateRegistry := strings.Replace(s.KubeConf.Cluster.Registry.PrivateRegistry, "/", "\\/", -1) if _, err := runtime.GetRunner().SudoCmd( @@ -142,18 +225,6 @@ func (s *Setup) Execute(runtime connector.Runtime) error { s.KubeConf.Cluster.Kubernetes.ContainerManager, s.KubeConf.Cluster.Kubernetes.ContainerManager)) } - caFile := "/etc/ssl/etcd/ssl/ca.pem" - certFile := fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s.pem", runtime.GetHostsByRole(common.ETCD)[0].GetName()) - keyFile := fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s-key.pem", runtime.GetHostsByRole(common.ETCD)[0].GetName()) - if output, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("/usr/local/bin/kubectl -n kubesphere-monitoring-system create secret generic kube-etcd-client-certs "+ - "--from-file=etcd-client-ca.crt=%s "+ - "--from-file=etcd-client.crt=%s "+ - "--from-file=etcd-client.key=%s", caFile, certFile, keyFile), true); err != nil { - if !strings.Contains(output, "exists") { - return err - } - } return nil } diff --git a/pkg/pipelines/add_nodes.go b/pkg/pipelines/add_nodes.go index 6b72dcd6..044499f6 100644 --- a/pkg/pipelines/add_nodes.go +++ b/pkg/pipelines/add_nodes.go @@ -18,6 +18,7 @@ package pipelines import ( "fmt" + kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/apis/kubekey/v1alpha2" kubekeycontroller "github.com/kubesphere/kubekey/controllers/kubekey" "github.com/kubesphere/kubekey/pkg/artifact" "github.com/kubesphere/kubekey/pkg/binaries" @@ -53,11 +54,11 @@ func NewAddNodesPipeline(runtime *common.KubeRuntime) error { &kubernetes.StatusModule{}, &container.InstallContainerModule{}, &images.PullModule{Skip: runtime.Arg.SkipPullImages}, - &etcd.PreCheckModule{}, + &etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, &etcd.CertsModule{}, - &etcd.InstallETCDBinaryModule{}, - &etcd.ConfigureModule{}, - &etcd.BackupModule{}, + &etcd.InstallETCDBinaryModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, + &etcd.ConfigureModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, + &etcd.BackupModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, &kubernetes.InstallKubeBinariesModule{}, &kubernetes.JoinNodesModule{}, &loadbalancer.HaproxyModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabled()}, @@ -105,11 +106,11 @@ func NewK3sAddNodesPipeline(runtime *common.KubeRuntime) error { &binaries.K3sNodeBinariesModule{}, &os.ConfigureOSModule{}, &k3s.StatusModule{}, - &etcd.PreCheckModule{}, + &etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, &etcd.CertsModule{}, - &etcd.InstallETCDBinaryModule{}, - &etcd.ConfigureModule{}, - &etcd.BackupModule{}, + &etcd.InstallETCDBinaryModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, + &etcd.ConfigureModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, + &etcd.BackupModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, &k3s.InstallKubeBinariesModule{}, &k3s.JoinNodesModule{}, &loadbalancer.K3sHaproxyModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabled()}, diff --git a/pkg/pipelines/create_cluster.go b/pkg/pipelines/create_cluster.go index 5b2936e5..60f7120d 100644 --- a/pkg/pipelines/create_cluster.go +++ b/pkg/pipelines/create_cluster.go @@ -19,6 +19,7 @@ package pipelines import ( "encoding/base64" "fmt" + kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/apis/kubekey/v1alpha2" "github.com/kubesphere/kubekey/pkg/artifact" "github.com/kubesphere/kubekey/pkg/bootstrap/confirm" "github.com/kubesphere/kubekey/pkg/bootstrap/precheck" @@ -69,11 +70,11 @@ func NewCreateClusterPipeline(runtime *common.KubeRuntime) error { &container.InstallContainerModule{}, &images.PushModule{Skip: skipPushImages}, &images.PullModule{Skip: runtime.Arg.SkipPullImages}, - &etcd.PreCheckModule{}, + &etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, &etcd.CertsModule{}, - &etcd.InstallETCDBinaryModule{}, - &etcd.ConfigureModule{}, - &etcd.BackupModule{}, + &etcd.InstallETCDBinaryModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, + &etcd.ConfigureModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, + &etcd.BackupModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, &kubernetes.InstallKubeBinariesModule{}, &kubernetes.InitKubernetesModule{}, &dns.ClusterDNSModule{}, @@ -159,11 +160,11 @@ func NewK3sCreateClusterPipeline(runtime *common.KubeRuntime) error { &binaries.K3sNodeBinariesModule{}, &os.ConfigureOSModule{}, &k3s.StatusModule{}, - &etcd.PreCheckModule{}, + &etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, &etcd.CertsModule{}, - &etcd.InstallETCDBinaryModule{}, - &etcd.ConfigureModule{}, - &etcd.BackupModule{}, + &etcd.InstallETCDBinaryModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, + &etcd.ConfigureModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, + &etcd.BackupModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, &k3s.InstallKubeBinariesModule{}, &k3s.InitClusterModule{}, &k3s.StatusModule{}, diff --git a/pkg/version/kubesphere/templates/cc_v320.go b/pkg/version/kubesphere/templates/cc_v320.go index e73f0707..e3720966 100644 --- a/pkg/version/kubesphere/templates/cc_v320.go +++ b/pkg/version/kubesphere/templates/cc_v320.go @@ -36,6 +36,7 @@ spec: storageClass: "" authentication: jwtSecret: "" + zone: "" local_registry: "" # dev_tag: "" etcd: diff --git a/pkg/version/kubesphere/templates/cc_v321.go b/pkg/version/kubesphere/templates/cc_v321.go index 2c116094..5fc4f35d 100644 --- a/pkg/version/kubesphere/templates/cc_v321.go +++ b/pkg/version/kubesphere/templates/cc_v321.go @@ -36,11 +36,12 @@ spec: storageClass: "" authentication: jwtSecret: "" + zone: "" local_registry: "" namespace_override: "" # dev_tag: "" etcd: - monitoring: false + monitoring: true endpointIps: localhost port: 2379 tlsEnable: true