From 8b74ead3079c954a0854dff1f3e5dbea2214c2a2 Mon Sep 17 00:00:00 2001 From: pixiake Date: Mon, 15 Mar 2021 10:13:43 +0800 Subject: [PATCH] enhancement of support for k3s Signed-off-by: pixiake --- cmd/init_os.go | 2 +- cmd/init_registry.go | 37 +++++ pkg/bootstrap/dependencies/init.go | 2 +- pkg/bootstrap/dependencies/os.go | 8 +- pkg/bootstrap/registry/config.go | 93 +++++++++++++ pkg/bootstrap/registry/registry.go | 157 ++++++++++++++++++++++ pkg/cluster/add/add.go | 12 +- pkg/cluster/delete/delete.go | 13 +- pkg/cluster/install/install.go | 2 +- pkg/config/parse.go | 12 +- pkg/files/file.go | 1 + pkg/k3s/config/service.go | 15 ++- pkg/k3s/master.go | 19 ++- pkg/k3s/nodes.go | 17 ++- pkg/k3s/preinstall/preinstall.go | 7 +- pkg/plugins/network/calico/tmpl-v1.16+.go | 3 + 16 files changed, 367 insertions(+), 33 deletions(-) create mode 100644 cmd/init_registry.go create mode 100644 pkg/bootstrap/registry/config.go create mode 100644 pkg/bootstrap/registry/registry.go diff --git a/cmd/init_os.go b/cmd/init_os.go index 729379bb..1c3b31c5 100644 --- a/cmd/init_os.go +++ b/cmd/init_os.go @@ -27,7 +27,7 @@ var osCmd = &cobra.Command{ Short: "Init operating system", RunE: func(cmd *cobra.Command, args []string) error { logger := util.InitLogger(opt.Verbose) - return dependencies.Init(opt.ClusterCfgFile, opt.SourcesDir, opt.AddImagesRepo, logger) + return dependencies.InitDependencies(opt.ClusterCfgFile, opt.SourcesDir, opt.AddImagesRepo, logger) }, } diff --git a/cmd/init_registry.go b/cmd/init_registry.go new file mode 100644 index 00000000..3ba73df3 --- /dev/null +++ b/cmd/init_registry.go @@ -0,0 +1,37 @@ +/* +Copyright 2020 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cmd + +import ( + "github.com/kubesphere/kubekey/pkg/bootstrap/registry" + "github.com/kubesphere/kubekey/pkg/util" + "github.com/spf13/cobra" +) + +// osCmd represents the os command +var registryCmd = &cobra.Command{ + Use: "registry", + Short: "Init a local image registry", + RunE: func(cmd *cobra.Command, args []string) error { + logger := util.InitLogger(opt.Verbose) + return registry.InitRegistry(opt.ClusterCfgFile, logger) + }, +} + +func init() { + initCmd.AddCommand(registryCmd) + registryCmd.Flags().StringVarP(&opt.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") +} diff --git a/pkg/bootstrap/dependencies/init.go b/pkg/bootstrap/dependencies/init.go index 31a4371e..7648df7c 100644 --- a/pkg/bootstrap/dependencies/init.go +++ b/pkg/bootstrap/dependencies/init.go @@ -28,7 +28,7 @@ import ( "path/filepath" ) -func Init(clusterCfgFile, sourcesDir string, addImagesRepo bool, logger *log.Logger) error { +func InitDependencies(clusterCfgFile, sourcesDir string, addImagesRepo bool, logger *log.Logger) error { currentDir, err := filepath.Abs(filepath.Dir(os.Args[0])) if err != nil { return errors.Wrap(err, "Failed to get current dir") diff --git a/pkg/bootstrap/dependencies/os.go b/pkg/bootstrap/dependencies/os.go index f4900fcc..54e8b54b 100644 --- a/pkg/bootstrap/dependencies/os.go +++ b/pkg/bootstrap/dependencies/os.go @@ -149,10 +149,10 @@ func initOS(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) error { return errors.New(fmt.Sprintf("Unsupported operating system: %s", osrData.ID)) } - output, err1 := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"systemctl start docker && systemctl enable docker && echo %s | base64 -d > /etc/docker/daemon.json && systemctl reload docker && systemctl restart docker\"", dockerConfigBase64), 0, false) - if err1 != nil { - return errors.Wrap(errors.WithStack(err1), fmt.Sprintf("Failed to install docker:\n%s", output)) - } + _, _ = mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"systemctl start docker && systemctl enable docker && echo %s | base64 -d > /etc/docker/daemon.json && systemctl reload docker && systemctl restart docker\"", dockerConfigBase64), 0, false) + //if err1 != nil { + // return errors.Wrap(errors.WithStack(err1), fmt.Sprintf("Failed to install docker:\n%s", output)) + //} } mgr.Logger.Info(fmt.Sprintf("Complete initialization %s [%s]\n", node.Name, node.InternalAddress)) } diff --git a/pkg/bootstrap/registry/config.go b/pkg/bootstrap/registry/config.go new file mode 100644 index 00000000..3972a17d --- /dev/null +++ b/pkg/bootstrap/registry/config.go @@ -0,0 +1,93 @@ +/* +Copyright 2020 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "text/template" + + "github.com/kubesphere/kubekey/pkg/util" + "github.com/lithammer/dedent" +) + +var ( + // RegistryServiceTempl defines the template of registry service for systemd. + RegistryServiceTempl = template.Must(template.New("registryService").Parse( + dedent.Dedent(`[Unit] +Description=v2 Registry server for Container +After=network.target + +[Service] +Type=simple +ExecStart=/usr/local/bin/registry serve /etc/kubekey/registry/config.yaml +Restart=on-failure + +[Install] +WantedBy=multi-user.target + + `))) + + // RegistryConfigTempl defines the template of registry's configuration file. + RegistryConfigTempl = template.Must(template.New("registryConfig").Parse( + dedent.Dedent(`version: 0.1 +log: + fields: + service: registry +storage: + cache: + layerinfo: inmemory + filesystem: + rootdirectory: /mnt/registry +http: + addr: :5000 + tls: + certificate: /etc/kubekey/registry/certs/domain.crt + key: /etc/kubekey/registry/certs/domain.key + + `))) + + // k3sRegistryConfigTempl defines the template of k3s' registry. + K3sRegistryConfigTempl = template.Must(template.New("k3sRegistryConfig").Parse( + dedent.Dedent(`mirrors: + "dockerhub.kubekey.local:5000": + endpoint: + - "https://dockerhub.kubekey.local:5000" + "docker.io": + endpoint: + - "https://dockerhub.kubekey.local:5000" +configs: + "dockerhub.kubekey.local:5000": + tls: + ca_file: "/etc/kubekey/registry/certs/ca.crt" + insecure_skip_verify: true + + `))) +) + +// GenerateRegistryService is used to generate registry's service content for systemd. +func GenerateRegistryService() (string, error) { + return util.Render(RegistryServiceTempl, util.Data{}) +} + +// GenerateRegistryConfig is used to generate the configuration file for registry. +func GenerateRegistryConfig() (string, error) { + return util.Render(RegistryConfigTempl, util.Data{}) +} + +// GenerateK3sRegistryConfig is used to generate the configuration file for registry. +func GenerateK3sRegistryConfig() (string, error) { + return util.Render(K3sRegistryConfigTempl, util.Data{}) +} diff --git a/pkg/bootstrap/registry/registry.go b/pkg/bootstrap/registry/registry.go new file mode 100644 index 00000000..6c07c267 --- /dev/null +++ b/pkg/bootstrap/registry/registry.go @@ -0,0 +1,157 @@ +/* +Copyright 2020 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "encoding/base64" + "fmt" + kubekeyapiv1alpha1 "github.com/kubesphere/kubekey/apis/kubekey/v1alpha1" + "github.com/kubesphere/kubekey/pkg/config" + "github.com/kubesphere/kubekey/pkg/util" + "github.com/kubesphere/kubekey/pkg/util/executor" + "github.com/kubesphere/kubekey/pkg/util/manager" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "os" + "os/exec" + "os/user" + "path/filepath" + "strings" +) + +var registryCrt string + +func InitRegistry(clusterCfgFile string, logger *log.Logger) error { + currentDir, err := filepath.Abs(filepath.Dir(os.Args[0])) + if err != nil { + return errors.Wrap(err, "Failed to get current dir") + } + if err := util.CreateDir(fmt.Sprintf("%s/kubekey", currentDir)); err != nil { + return errors.Wrap(err, "Failed to create work dir") + } + + cfg, objName, err := config.ParseClusterCfg(clusterCfgFile, "", "", false, logger) + if err != nil { + return errors.Wrap(err, "Failed to download cluster config") + } + + return Execute(executor.NewExecutor(&cfg.Spec, objName, logger, "", true, true, true, false, false, nil)) +} + +func Execute(executor *executor.Executor) error { + mgr, err := executor.CreateManager() + if err != nil { + return err + } + return ExecTasks(mgr) +} + +func ExecTasks(mgr *manager.Manager) error { + createTasks := []manager.Task{ + {Task: CreateRegistry, ErrMsg: "Failed to init operating system"}, + } + + for _, step := range createTasks { + if err := step.Run(mgr); err != nil { + return errors.Wrap(err, step.ErrMsg) + } + } + + mgr.Logger.Infoln("Init local registry successful.") + + return nil +} + +func CreateRegistry(mgr *manager.Manager) error { + user, _ := user.Current() + if user.Username != "root" { + return errors.New(fmt.Sprintf("Current user is %s. Please use root!", user.Username)) + } + mgr.Logger.Infoln("Init local registry") + + if output, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("cp %s/registry /usr/local/bin/registry", mgr.WorkDir)).CombinedOutput(); err != nil { + return errors.Wrapf(err, string(output)) + } + + if output, err := exec.Command("/bin/bash", "-c", "if [[ ! -f /etc/kubekey/registry/certs/domain.crt ]]; then "+ + "mkdir -p /etc/kubekey/registry/certs && "+ + "openssl req -newkey rsa:4096 -nodes -sha256 -keyout /etc/kubekey/registry/certs/domain.key -x509 -days 36500 -out /etc/kubekey/registry/certs/domain.crt -subj '/CN=dockerhub.kubekey.local';"+ + "fi").CombinedOutput(); err != nil { + return errors.Wrapf(err, string(output)) + } + + registryCrtBase64Cmd := "cat /etc/kubekey/registry/certs/domain.crt | base64 --wrap=0" + if output, err := exec.Command("/bin/sh", "-c", registryCrtBase64Cmd).CombinedOutput(); err != nil { + return err + } else { + registryCrt = strings.TrimSpace(string(output)) + } + + registryConfig, err := GenerateRegistryConfig() + if err != nil { + return err + } + + if output, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("echo %s | base64 -d > /etc/kubekey/registry/config.yaml", base64.StdEncoding.EncodeToString([]byte(registryConfig)))).CombinedOutput(); err != nil { + return errors.Wrapf(err, string(output)) + } + + registryService, err := GenerateRegistryService() + if err != nil { + return err + } + + if output, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("echo %s | base64 -d > /etc/systemd/system/registry.service", base64.StdEncoding.EncodeToString([]byte(registryService)))).CombinedOutput(); err != nil { + return errors.Wrapf(err, string(output)) + } + + if output, err := exec.Command("/bin/sh", "-c", "systemctl daemon-reload && systemctl enable --now registry").CombinedOutput(); err != nil { + return errors.Wrapf(err, string(output)) + } + + if err := mgr.RunTaskOnAllNodes(syncRegistryConfig, true); err != nil { + return err + } + + fmt.Print("\nLocal image registry created successfully. Address: dockerhub.kubekey.local:5000\n") + return nil +} + +func syncRegistryConfig(mgr *manager.Manager, _ *kubekeyapiv1alpha1.HostCfg) error { + if _, err := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"echo '%s dockerhub.kubekey.local' >> /etc/hosts\"", util.LocalIP())+" && "+ + "sudo awk ' !x[$0]++{print > \"/etc/hosts\"}' /etc/hosts", 2, false); err != nil { + return err + } + + crtPaths := []string{"/etc/docker/certs.d/dockerhub.kubekey.local:5000", "/etc/kubekey/registry/certs"} + for _, crtPath := range crtPaths { + syncCrtCmd := fmt.Sprintf("sudo -E /bin/sh -c \"mkdir -p %s && echo %s | base64 -d > %s/ca.crt\"", crtPath, registryCrt, crtPath) + if _, err := mgr.Runner.ExecuteCmd(syncCrtCmd, 1, false); err != nil { + return errors.Wrap(errors.WithStack(err), "Failed to sync registry crt") + } + } + + k3sRegistryConfig, err := GenerateK3sRegistryConfig() + if err != nil { + return err + } + if _, err := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"mkdir -p /etc/rancher/k3s && echo %s | base64 -d > /etc/rancher/k3s/registries.yaml\"", base64.StdEncoding.EncodeToString([]byte(k3sRegistryConfig))), 1, false); err != nil { + return errors.Wrap(errors.WithStack(err), "Failed to generate k3s registries config") + } + + return nil +} diff --git a/pkg/cluster/add/add.go b/pkg/cluster/add/add.go index be7397a0..7b3d2368 100644 --- a/pkg/cluster/add/add.go +++ b/pkg/cluster/add/add.go @@ -90,13 +90,15 @@ func ExecTasks(mgr *manager.Manager) error { } for _, step := range addNodeTasks { - if err := step.Run(mgr); err != nil { - if mgr.InCluster { - if err := kubekeycontroller.PatchNodeImportStatus(mgr, kubekeycontroller.Failed); err != nil { - return err + if !step.Skip { + if err := step.Run(mgr); err != nil { + if mgr.InCluster { + if err := kubekeycontroller.PatchNodeImportStatus(mgr, kubekeycontroller.Failed); err != nil { + return err + } } + return errors.Wrap(err, step.ErrMsg) } - return errors.Wrap(err, step.ErrMsg) } } diff --git a/pkg/cluster/delete/delete.go b/pkg/cluster/delete/delete.go index 52d4faa3..3ebe12b2 100644 --- a/pkg/cluster/delete/delete.go +++ b/pkg/cluster/delete/delete.go @@ -19,6 +19,7 @@ package delete import ( "bufio" "fmt" + "github.com/kubesphere/kubekey/pkg/util" "os" "os/exec" "path/filepath" @@ -277,6 +278,7 @@ var ( "/usr/local/bin/kubeadm", "/usr/local/bin/kubectl", "/usr/bin/kubelet", + "/var/lib/rook", } kubeovnFiles = []string{ @@ -310,11 +312,14 @@ func resetKubeCluster(mgr *manager.Manager, _ *kubekeyapiv1alpha1.HostCfg) error case "k3s": _, _ = mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"systemctl daemon-reload && /usr/local/bin/k3s-uninstall.sh\"", 0, true) default: - _, _ = mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"/usr/local/bin/kubeadm reset -f\"", 0, true) - _, _ = mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", strings.Join(cmdsList, " && ")), 0, true, "printCmd") - _ = deleteFiles(mgr) + if util.IsExist("/usr/local/bin/k3s-uninstall.sh") { + _, _ = mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"systemctl daemon-reload && /usr/local/bin/k3s-uninstall.sh\"", 0, true) + } else { + _, _ = mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"/usr/local/bin/kubeadm reset -f\"", 0, true) + _, _ = mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", strings.Join(cmdsList, " && ")), 0, true, "printCmd") + } } - + _ = deleteFiles(mgr) return nil } diff --git a/pkg/cluster/install/install.go b/pkg/cluster/install/install.go index 1960ce8c..d66980ce 100644 --- a/pkg/cluster/install/install.go +++ b/pkg/cluster/install/install.go @@ -97,7 +97,7 @@ func ExecTasks(mgr *manager.Manager) error { {Task: InstallKubeBinaries, ErrMsg: "Failed to install kube binaries"}, {Task: InitKubernetesCluster, ErrMsg: "Failed to init kubernetes cluster"}, {Task: JoinNodesToCluster, ErrMsg: "Failed to join node"}, - {Task: network.DeployNetworkPlugin, ErrMsg: "Failed to deploy network plugin", Skip: skipCondition2}, + {Task: network.DeployNetworkPlugin, ErrMsg: "Failed to deploy network plugin"}, {Task: addons.InstallAddons, ErrMsg: "Failed to deploy addons", Skip: skipCondition1 && !skipCondition2}, {Task: kubesphere.DeployLocalVolume, ErrMsg: "Failed to deploy localVolume", Skip: skipCondition1 || skipCondition2}, {Task: kubesphere.DeployKubeSphere, ErrMsg: "Failed to deploy kubesphere", Skip: skipCondition1}, diff --git a/pkg/config/parse.go b/pkg/config/parse.go index d07ee8d4..4b1d3d17 100644 --- a/pkg/config/parse.go +++ b/pkg/config/parse.go @@ -173,8 +173,16 @@ func AllinoneCfg(user *user.User, k8sVersion, ksVersion string, ksEnabled bool, Worker: []string{hostname}, } if k8sVersion != "" { - allinoneCfg.Spec.Kubernetes = kubekeyapiv1alpha1.Kubernetes{ - Version: k8sVersion, + s := strings.Split(k8sVersion, "-") + if len(s) > 1 { + allinoneCfg.Spec.Kubernetes = kubekeyapiv1alpha1.Kubernetes{ + Version: s[0], + Type: s[1], + } + } else { + allinoneCfg.Spec.Kubernetes = kubekeyapiv1alpha1.Kubernetes{ + Version: k8sVersion, + } } } else { allinoneCfg.Spec.Kubernetes = kubekeyapiv1alpha1.Kubernetes{ diff --git a/pkg/files/file.go b/pkg/files/file.go index 08a5a07d..75238863 100644 --- a/pkg/files/file.go +++ b/pkg/files/file.go @@ -172,6 +172,7 @@ var ( k3s: { amd64: { "v1.20.2": "ce3055783cf115ee68fc00bb8d25421d068579ece2fafa4ee1d09f3415aaeabf", + "v1.20.4": "1c7b68b0b7d54f21a9c1727545a7db181668115f161a3986bc137261dd817e98", }, }, } diff --git a/pkg/k3s/config/service.go b/pkg/k3s/config/service.go index 9c296d19..aa4bd14f 100644 --- a/pkg/k3s/config/service.go +++ b/pkg/k3s/config/service.go @@ -18,6 +18,7 @@ package config import ( "fmt" + "github.com/kubesphere/kubekey/pkg/kubernetes/preinstall" "strings" "text/template" @@ -63,9 +64,9 @@ ExecStart=/usr/local/bin/k3s server dedent.Dedent(`# Note: This dropin only works with k3s [Service] {{ if .IsMaster }} -Environment="K3S_ARGS=--datastore-endpoint={{ .DataStoreEndPoint }} --datastore-cafile={{ .DataStoreCaFile }} --datastore-certfile={{ .DataStoreCertFile }} --datastore-keyfile={{ .DataStoreKeyFile }}" +Environment="K3S_ARGS=--datastore-endpoint={{ .DataStoreEndPoint }} --datastore-cafile={{ .DataStoreCaFile }} --datastore-certfile={{ .DataStoreCertFile }} --datastore-keyfile={{ .DataStoreKeyFile }} {{ range .CertSANs }} --tls-san={{ . }}{{- end }} --cluster-cidr={{ .PodSubnet }} --service-cidr={{ .ServiceSubnet }} --cluster-dns={{ .ClusterDns }} --flannel-backend=none --disable-network-policy --disable-cloud-controller --disable=servicelb,traefik,local-storage,metrics-server" {{ end }} -Environment="K3S_EXTRA_ARGS=--node-name={{ .HostName }} --node-ip={{ .NodeIP }} {{ if .Server }}--server={{ .Server }}{{ end }} {{ if .Token }}--token={{ .Token }}{{ end }}" +Environment="K3S_EXTRA_ARGS=--node-name={{ .HostName }} --node-ip={{ .NodeIP }} {{ if .Server }}--server={{ .Server }}{{ end }} {{ if .Token }}--token={{ .Token }}{{ end }} --pause-image={{ .PauseImage }} --kubelet-arg=cni-conf-dir=/etc/cni/net.d --kubelet-arg=cni-bin-dir=/opt/cni/bin --kube-proxy-arg=proxy-mode=ipvs --kube-proxy-arg=masquerade-all=true" Environment="K3S_ROLE={{ if .IsMaster }}server{{ else }}agent{{ end }}" ExecStart= ExecStart=/usr/local/bin/k3s $K3S_ROLE $K3S_ARGS $K3S_EXTRA_ARGS @@ -79,8 +80,6 @@ func GenerateK3sService() (string, error) { // GenerateK3sEnv is used to generate the env content of kubelet's service for systemd. func GenerateK3sEnv(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg, token string) (string, error) { - // var containerRuntime string - // generate etcd configuration var externalEtcd kubekeyapiv1alpha1.ExternalEtcd var endpointsList []string @@ -103,7 +102,7 @@ func GenerateK3sEnv(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg, toke var server string if token != "" { - server = fmt.Sprintf("https://%s:6443", mgr.MasterNodes[0].InternalAddress) + server = fmt.Sprintf("https://%s:%d", mgr.Cluster.ControlPlaneEndpoint.Domain, mgr.Cluster.ControlPlaneEndpoint.Port) } else { server = "" } @@ -118,6 +117,10 @@ func GenerateK3sEnv(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg, toke "HostName": node.Name, "Token": token, "Server": server, - // "ContainerRuntime": containerRuntime, + "PodSubnet": mgr.Cluster.Network.KubePodsCIDR, + "ServiceSubnet": mgr.Cluster.Network.KubeServiceCIDR, + "ClusterDns": mgr.Cluster.ClusterIP(), + "CertSANs": mgr.Cluster.GenerateCertSANs(), + "PauseImage": preinstall.GetImage(mgr, "pause").ImageName(), }) } diff --git a/pkg/k3s/master.go b/pkg/k3s/master.go index 71092544..c6e71878 100644 --- a/pkg/k3s/master.go +++ b/pkg/k3s/master.go @@ -91,7 +91,7 @@ func InitKubernetesCluster(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCf return errors.Wrap(errors.WithStack(err), "Failed to generate kubelet env") } - _, err1 := mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"systemctl daemon-reload && systemctl restart k3s\"", 1, false) + _, err1 := mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"systemctl daemon-reload && systemctl enable --now k3s\"", 1, false) if err1 != nil { return errors.Wrap(errors.WithStack(err1), "Failed to start k3s") } @@ -100,6 +100,11 @@ func InitKubernetesCluster(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCf return err } + if !node.IsWorker { + addTaintForMasterCmd := fmt.Sprintf("sudo -E /bin/sh -c \"/usr/local/bin/kubectl taint nodes %s node-role.kubernetes.io/master=effect:NoSchedule --overwrite\"", node.Name) + _, _ = mgr.Runner.ExecuteCmd(addTaintForMasterCmd, 5, true) + } + if err := addWorkerLabel(mgr, node); err != nil { return err } @@ -217,6 +222,10 @@ func JoinNodesToCluster(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) if err2 != nil { return err2 } + if !node.IsWorker { + addTaintForMasterCmd := fmt.Sprintf("sudo -E /bin/sh -c \"/usr/local/bin/kubectl taint nodes %s node-role.kubernetes.io/master=effect:NoSchedule --overwrite\"", node.Name) + _, _ = mgr.Runner.ExecuteCmd(addTaintForMasterCmd, 5, true) + } } if node.IsWorker && !node.IsMaster { err := addWorker(mgr, node) @@ -233,7 +242,7 @@ func JoinNodesToCluster(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) } func addMaster(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) error { - kubeletEnv, err3 := config.GenerateK3sEnv(mgr, node, clusterStatus["nodeToken"]) + kubeletEnv, err3 := config.GenerateK3sEnv(mgr, node, "") if err3 != nil { return err3 } @@ -242,7 +251,7 @@ func addMaster(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) error { return errors.Wrap(errors.WithStack(err), "Failed to generate kubelet env") } - if _, err := mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"systemctl daemon-reload && systemctl restart k3s\"", 2, false); err != nil { + if _, err := mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"systemctl daemon-reload && systemctl enable --now k3s\"", 2, false); err != nil { return errors.Wrap(errors.WithStack(err), "Failed to up k3s") } @@ -259,7 +268,7 @@ func addWorker(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) error { return errors.Wrap(errors.WithStack(err), "Failed to generate kubelet env") } - if _, err := mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"systemctl daemon-reload && systemctl restart k3s\"", 2, false); err != nil { + if _, err := mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"systemctl daemon-reload && systemctl enable --now k3s\"", 2, false); err != nil { return errors.Wrap(errors.WithStack(err), "Failed to up k3s") } @@ -272,7 +281,7 @@ func addWorker(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) error { } oldServer := "server: https://127.0.0.1:6443" - newServer := fmt.Sprintf("server: https://%s:%d", mgr.Cluster.ControlPlaneEndpoint.Address, mgr.Cluster.ControlPlaneEndpoint.Port) + newServer := fmt.Sprintf("server: https://%s:%d", mgr.Cluster.ControlPlaneEndpoint.Domain, mgr.Cluster.ControlPlaneEndpoint.Port) newKubeconfigStr := strings.Replace(string(kubeconfigStr), oldServer, newServer, -1) if _, err := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", createConfigDirCmd), 1, false); err != nil { diff --git a/pkg/k3s/nodes.go b/pkg/k3s/nodes.go index 07ef94ac..f4326397 100644 --- a/pkg/k3s/nodes.go +++ b/pkg/k3s/nodes.go @@ -36,7 +36,7 @@ func InstallKubeBinaries(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) return err } - if err := SetKubelet(mgr, node); err != nil { + if err := SetKubelet(mgr); err != nil { return err } } @@ -72,19 +72,30 @@ func SyncKubeBinaries(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) er k3s := "k3s" helm := "helm" - binaryList := []string{k3s, helm} + kubecni := fmt.Sprintf("cni-plugins-linux-%s-%s.tgz", node.Arch, kubekeyapiv1alpha1.DefaultCniVersion) + binaryList := []string{k3s, helm, kubecni} + var cmdlist []string for _, binary := range binaryList { if err := mgr.Runner.ScpFile(fmt.Sprintf("%s/%s", filesDir, binary), fmt.Sprintf("%s/%s", "/tmp/kubekey", binary)); err != nil { return errors.Wrap(errors.WithStack(err), fmt.Sprintf("Failed to sync binaries")) } + + if strings.Contains(binary, "cni-plugins-linux") { + cmdlist = append(cmdlist, fmt.Sprintf("mkdir -p /opt/cni/bin && tar -zxf %s/%s -C /opt/cni/bin", "/tmp/kubekey", binary)) + } + } + + cmd := strings.Join(cmdlist, " && ") + if _, err := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", cmd), 2, false); err != nil { + return errors.Wrap(errors.WithStack(err), fmt.Sprintf("Failed to install kube cni")) } return nil } // SetKubelet is used to configure the kubelet's startup parameters. -func SetKubelet(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) error { +func SetKubelet(mgr *manager.Manager) error { if _, err := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", "cp -f /tmp/kubekey/k3s /usr/local/bin/k3s && chmod +x /usr/local/bin/k3s"), 2, false); err != nil { return errors.Wrap(errors.WithStack(err), fmt.Sprintf("Failed to create kubelet link")) diff --git a/pkg/k3s/preinstall/preinstall.go b/pkg/k3s/preinstall/preinstall.go index 1bb07fc6..65a2c9c0 100644 --- a/pkg/k3s/preinstall/preinstall.go +++ b/pkg/k3s/preinstall/preinstall.go @@ -34,29 +34,34 @@ func FilesDownloadHTTP(mgr *manager.Manager, filepath, version, arch string) err kkzone := os.Getenv("KKZONE") etcd := files.KubeBinary{Name: "etcd", Arch: arch, Version: kubekeyapiv1alpha1.DefaultEtcdVersion} k3s := files.KubeBinary{Name: "k3s", Arch: arch, Version: version} + kubecni := files.KubeBinary{Name: "kubecni", Arch: arch, Version: kubekeyapiv1alpha1.DefaultCniVersion} helm := files.KubeBinary{Name: "helm", Arch: arch, Version: kubekeyapiv1alpha1.DefaultHelmVersion} etcd.Path = fmt.Sprintf("%s/etcd-%s-linux-%s.tar.gz", filepath, kubekeyapiv1alpha1.DefaultEtcdVersion, arch) k3s.Path = fmt.Sprintf("%s/k3s", filepath) + kubecni.Path = fmt.Sprintf("%s/cni-plugins-linux-%s-%s.tgz", filepath, arch, kubekeyapiv1alpha1.DefaultCniVersion) helm.Path = fmt.Sprintf("%s/helm", filepath) if kkzone == "cn" { etcd.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/etcd/release/download/%s/etcd-%s-linux-%s.tar.gz", etcd.Version, etcd.Version, etcd.Arch) k3s.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/k3s/releases/download/%s+k3s1/linux/%s/k3s", k3s.Version, k3s.Arch) + kubecni.Url = fmt.Sprintf("https://containernetworking.pek3b.qingstor.com/plugins/releases/download/%s/cni-plugins-linux-%s-%s.tgz", kubecni.Version, kubecni.Arch, kubecni.Version) helm.Url = fmt.Sprintf("https://kubernetes-helm.pek3b.qingstor.com/linux-%s/%s/helm", helm.Arch, helm.Version) helm.GetCmd = mgr.DownloadCommand(helm.Path, helm.Url) } else { etcd.Url = fmt.Sprintf("https://github.com/coreos/etcd/releases/download/%s/etcd-%s-linux-%s.tar.gz", etcd.Version, etcd.Version, etcd.Arch) k3s.Url = fmt.Sprintf("https://github.com/rancher/k3s/releases/download/%s+k3s1/k3s", k3s.Version) + kubecni.Url = fmt.Sprintf("https://github.com/containernetworking/plugins/releases/download/%s/cni-plugins-linux-%s-%s.tgz", kubecni.Version, kubecni.Arch, kubecni.Version) helm.Url = fmt.Sprintf("https://get.helm.sh/helm-%s-linux-%s.tar.gz", helm.Version, helm.Arch) getCmd := mgr.DownloadCommand(fmt.Sprintf("%s/helm-%s-linux-%s.tar.gz", filepath, helm.Version, helm.Arch), helm.Url) helm.GetCmd = fmt.Sprintf("%s && cd %s && tar -zxf helm-%s-linux-%s.tar.gz && mv linux-%s/helm . && rm -rf *linux-%s*", getCmd, filepath, helm.Version, helm.Arch, helm.Arch, helm.Arch) } k3s.GetCmd = mgr.DownloadCommand(k3s.Path, k3s.Url) + kubecni.GetCmd = mgr.DownloadCommand(kubecni.Path, kubecni.Url) etcd.GetCmd = mgr.DownloadCommand(etcd.Path, etcd.Url) - binaries := []files.KubeBinary{k3s, helm, etcd} + binaries := []files.KubeBinary{k3s, helm, kubecni, etcd} for _, binary := range binaries { if binary.Name == "etcd" && mgr.EtcdContainer { diff --git a/pkg/plugins/network/calico/tmpl-v1.16+.go b/pkg/plugins/network/calico/tmpl-v1.16+.go index 75816db4..85930d0d 100644 --- a/pkg/plugins/network/calico/tmpl-v1.16+.go +++ b/pkg/plugins/network/calico/tmpl-v1.16+.go @@ -66,6 +66,9 @@ data: "policy": { "type": "k8s" }, + "container_settings": { + "allow_ip_forwarding": true + }, "kubernetes": { "kubeconfig": "__KUBECONFIG_FILEPATH__" }