diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 7f79cfc6..b2b52fc2 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -2,7 +2,7 @@ // +build !ignore_autogenerated /* -Copyright 2022 The KubeSphere Authors. +Copyright 2020 The KubeSphere Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go b/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go index e0565fba..ad0ad151 100644 --- a/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go +++ b/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go @@ -85,12 +85,21 @@ type KubeVip struct { Mode string `yaml:"mode" json:"mode,omitempty"` } +// CustomScripts defines the custom shell scripts for each node to exec before and finished kubernetes install. +type CustomScripts struct { + Name string `yaml:"name" json:"name,omitempty"` + Bash string `yaml:"bash" json:"shell,omitempty"` + Materials []string `yaml:"materials" json:"materials,omitempty"` +} + // System defines the system config for each node in cluster. type System struct { - NtpServers []string `yaml:"ntpServers" json:"ntpServers,omitempty"` - Timezone string `yaml:"timezone" json:"timezone,omitempty"` - Rpms []string `yaml:"rpms" json:"rpms,omitempty"` - Debs []string `yaml:"debs" json:"debs,omitempty"` + NtpServers []string `yaml:"ntpServers" json:"ntpServers,omitempty"` + Timezone string `yaml:"timezone" json:"timezone,omitempty"` + Rpms []string `yaml:"rpms" json:"rpms,omitempty"` + Debs []string `yaml:"debs" json:"debs,omitempty"` + PreInstall []CustomScripts `yaml:"preInstall" json:"preInstall,omitempty"` + PostInstall []CustomScripts `yaml:"postInstall" json:"postInstall,omitempty"` } // RegistryConfig defines the configuration information of the image's repository. diff --git a/cmd/kk/pkg/bootstrap/customscripts/module.go b/cmd/kk/pkg/bootstrap/customscripts/module.go new file mode 100644 index 00000000..a2b8a72f --- /dev/null +++ b/cmd/kk/pkg/bootstrap/customscripts/module.go @@ -0,0 +1,52 @@ +/* + Copyright 2021 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package customscripts + +import ( + "fmt" + + kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/cmd/kk/apis/kubekey/v1alpha2" + "github.com/kubesphere/kubekey/cmd/kk/pkg/core/module" + "github.com/kubesphere/kubekey/cmd/kk/pkg/core/task" +) + +type CustomScriptsModule struct { + module.BaseTaskModule + Phase string + Scripts []kubekeyapiv1alpha2.CustomScripts +} + +func (m *CustomScriptsModule) Init() { + m.Name = fmt.Sprintf("CustomScriptsModule Phase:%s", m.Phase) + m.Desc = "Exec custom shell scripts for each nodes." + + for idx, script := range m.Scripts { + + taskName := fmt.Sprintf("Phase:%s(%d/%d) script:%s", m.Phase, idx, len(m.Scripts), script.Name) + taskDir := fmt.Sprintf("%s-%d-script", m.Phase, idx) + task := &task.RemoteTask{ + Name: taskName, + Desc: taskName, + Hosts: m.Runtime.GetAllHosts(), + Action: &CustomScriptTask{taskDir: taskDir, script: script}, + Parallel: true, + Retry: 1, + } + + m.Tasks = append(m.Tasks, task) + } +} diff --git a/cmd/kk/pkg/bootstrap/customscripts/tasks.go b/cmd/kk/pkg/bootstrap/customscripts/tasks.go new file mode 100644 index 00000000..9e9aa1aa --- /dev/null +++ b/cmd/kk/pkg/bootstrap/customscripts/tasks.go @@ -0,0 +1,120 @@ +/* + Copyright 2021 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package customscripts + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" + + "github.com/pkg/errors" + + kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/cmd/kk/apis/kubekey/v1alpha2" + "github.com/kubesphere/kubekey/cmd/kk/pkg/common" + "github.com/kubesphere/kubekey/cmd/kk/pkg/core/action" + "github.com/kubesphere/kubekey/cmd/kk/pkg/core/connector" + "github.com/kubesphere/kubekey/cmd/kk/pkg/core/util" +) + +type CustomScriptTask struct { + action.BaseAction + taskDir string + script kubekeyapiv1alpha2.CustomScripts +} + + + +func (t *CustomScriptTask) Execute(runtime connector.Runtime) error { + + if len(t.script.Bash) <= 0 { + return errors.Errorf("custom script %s Bash is empty", t.script.Name) + } + + remoteTaskHome := common.TmpDir + t.taskDir + + if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("mkdir -p %s", remoteTaskHome), false); err != nil { + return errors.Wrapf(err, "create remoteTaskHome: %s err:%s", remoteTaskHome, err) + } + + // dilver the dependency materials to the remotehost + for idx, localPath := range t.script.Materials { + + if !util.IsExist(localPath) { + return errors.Errorf("Not found Path: %s", localPath) + } + + targetPath := filepath.Join(remoteTaskHome, filepath.Base(localPath)) + + // first clean the target to makesure target path always is the lastest. + cleanCmd := fmt.Sprintf("rm -fr %s", targetPath) + if _, err := runtime.GetRunner().SudoCmd(cleanCmd, false); err != nil { + return errors.Wrapf(err, "Can not remove target found Path: %s", targetPath) + } + + start := time.Now() + err := runtime.GetRunner().SudoScp(localPath, targetPath) + if err != nil { + return errors.Wrapf(err, "Can not Scp -fr %s root@%s:%s", localPath, runtime.RemoteHost().GetAddress(), targetPath) + } + + fmt.Printf("Copy %d/%d materials: Scp -fr %s root@%s:%s done, take %s\n", + idx, len(t.script.Materials), localPath, runtime.RemoteHost().GetAddress(), targetPath, time.Since(start)) + } + + // wrap use bash file if shell has many lines. + RunBash := t.script.Bash + if strings.Index(RunBash, "\n") > 0 { + tmpFile, err := ioutil.TempFile(os.TempDir(), t.taskDir) + if err != nil { + return errors.Wrapf(err, "create tmp Bash: %s/%s in local node, err:%s", os.TempDir(), t.taskDir, err) + } + defer os.Remove(tmpFile.Name()) + + if _, err := tmpFile.WriteString(RunBash); err != nil { + return errors.Wrapf(err, "write to tmp:%s in local node, err:%s", tmpFile.Name(), err) + } + + targetPath := filepath.Join(remoteTaskHome, "task.sh") + if err := runtime.GetRunner().SudoScp(tmpFile.Name(), targetPath); err != nil { + return errors.Wrapf(err, "Can not Scp -fr %s root@%s:%s", tmpFile.Name(), runtime.RemoteHost().GetAddress(), targetPath) + } + + RunBash = "/bin/bash " + targetPath + } + + start := time.Now() + out, err := runtime.GetRunner().SudoCmd(RunBash, false) + if err != nil { + return errors.Errorf("Exec Bash: %s err:%s", RunBash, err) + } + + if !runtime.GetRunner().Debug { + fmt.Printf("Exec Bash:%s done, take %s", RunBash, time.Since(start)) + cleanCmd := fmt.Sprintf("rm -fr %s", remoteTaskHome) + if _, err := runtime.GetRunner().SudoCmd(cleanCmd, false); err != nil { + return errors.Wrapf(err, "Exec cmd:%s err:%s", cleanCmd, err) + } + }else { + // keep the Materials for debug + fmt.Printf("Exec Bash:%s done, take %s, output:\n%s", RunBash, time.Since(start), out) + } + + return nil +} diff --git a/cmd/kk/pkg/pipelines/add_nodes.go b/cmd/kk/pkg/pipelines/add_nodes.go index 8309fc87..4c46e4cd 100644 --- a/cmd/kk/pkg/pipelines/add_nodes.go +++ b/cmd/kk/pkg/pipelines/add_nodes.go @@ -23,6 +23,7 @@ import ( "github.com/kubesphere/kubekey/cmd/kk/pkg/artifact" "github.com/kubesphere/kubekey/cmd/kk/pkg/binaries" "github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/confirm" + "github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/customscripts" "github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/os" "github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/precheck" "github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/registry" @@ -51,6 +52,7 @@ func NewAddNodesPipeline(runtime *common.KubeRuntime) error { &os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages}, &binaries.NodeBinariesModule{}, &os.ConfigureOSModule{}, + &customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall}, ®istry.RegistryCertsModule{Skip: len(runtime.GetHostsByRole(common.Registry)) == 0}, &kubernetes.StatusModule{}, &container.InstallContainerModule{}, @@ -66,6 +68,7 @@ func NewAddNodesPipeline(runtime *common.KubeRuntime) error { &kubernetes.ConfigureKubernetesModule{}, &filesystem.ChownModule{}, &certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()}, + &customscripts.CustomScriptsModule{Phase: "PostInstall", Scripts: runtime.Cluster.System.PostInstall}, } p := pipeline.Pipeline{ @@ -89,6 +92,7 @@ func NewK3sAddNodesPipeline(runtime *common.KubeRuntime) error { &os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages}, &binaries.K3sNodeBinariesModule{}, &os.ConfigureOSModule{}, + &customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall}, &k3s.StatusModule{}, &etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, &etcd.CertsModule{}, @@ -101,6 +105,7 @@ func NewK3sAddNodesPipeline(runtime *common.KubeRuntime) error { &kubernetes.ConfigureKubernetesModule{}, &filesystem.ChownModule{}, &certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()}, + &customscripts.CustomScriptsModule{Phase: "PostInstall", Scripts: runtime.Cluster.System.PostInstall}, } p := pipeline.Pipeline{ @@ -124,6 +129,7 @@ func NewK8eAddNodesPipeline(runtime *common.KubeRuntime) error { &os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages}, &binaries.K8eNodeBinariesModule{}, &os.ConfigureOSModule{}, + &k8e.StatusModule{}, &etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, &etcd.CertsModule{}, diff --git a/cmd/kk/pkg/pipelines/create_cluster.go b/cmd/kk/pkg/pipelines/create_cluster.go index 64fd45de..009600fa 100644 --- a/cmd/kk/pkg/pipelines/create_cluster.go +++ b/cmd/kk/pkg/pipelines/create_cluster.go @@ -24,6 +24,7 @@ import ( "github.com/kubesphere/kubekey/cmd/kk/pkg/artifact" "github.com/kubesphere/kubekey/cmd/kk/pkg/binaries" "github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/confirm" + "github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/customscripts" "github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/os" "github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/precheck" "github.com/kubesphere/kubekey/cmd/kk/pkg/certs" @@ -63,6 +64,7 @@ func NewCreateClusterPipeline(runtime *common.KubeRuntime) error { &os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages}, &binaries.NodeBinariesModule{}, &os.ConfigureOSModule{}, + &customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall}, &kubernetes.StatusModule{}, &container.InstallContainerModule{}, &images.CopyImagesToRegistryModule{Skip: skipPushImages}, @@ -91,6 +93,7 @@ func NewCreateClusterPipeline(runtime *common.KubeRuntime) error { &storage.DeployLocalVolumeModule{Skip: skipLocalStorage}, &kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, &kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, + &customscripts.CustomScriptsModule{Phase: "PostInstall", Scripts: runtime.Cluster.System.PostInstall}, } p := pipeline.Pipeline{ @@ -141,6 +144,7 @@ func NewK3sCreateClusterPipeline(runtime *common.KubeRuntime) error { &os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages}, &binaries.K3sNodeBinariesModule{}, &os.ConfigureOSModule{}, + &customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall}, &k3s.StatusModule{}, &etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, &etcd.CertsModule{}, @@ -163,6 +167,7 @@ func NewK3sCreateClusterPipeline(runtime *common.KubeRuntime) error { &storage.DeployLocalVolumeModule{Skip: skipLocalStorage}, &kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, &kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, + &customscripts.CustomScriptsModule{Phase: "PostInstall", Scripts: runtime.Cluster.System.PostInstall}, } p := pipeline.Pipeline{ @@ -213,6 +218,7 @@ func NewK8eCreateClusterPipeline(runtime *common.KubeRuntime) error { &os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages}, &binaries.K8eNodeBinariesModule{}, &os.ConfigureOSModule{}, + &customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall}, &k8e.StatusModule{}, &etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, &etcd.CertsModule{}, @@ -235,6 +241,7 @@ func NewK8eCreateClusterPipeline(runtime *common.KubeRuntime) error { &storage.DeployLocalVolumeModule{Skip: skipLocalStorage}, &kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, &kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, + &customscripts.CustomScriptsModule{Phase: "PostInstall", Scripts: runtime.Cluster.System.PostInstall}, } p := pipeline.Pipeline{ diff --git a/cmd/kk/pkg/pipelines/init_dependencies.go b/cmd/kk/pkg/pipelines/init_dependencies.go index 394dd0e8..bdc9a2aa 100644 --- a/cmd/kk/pkg/pipelines/init_dependencies.go +++ b/cmd/kk/pkg/pipelines/init_dependencies.go @@ -18,6 +18,7 @@ package pipelines import ( "github.com/kubesphere/kubekey/cmd/kk/pkg/artifact" + "github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/customscripts" "github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/os" "github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/precheck" "github.com/kubesphere/kubekey/cmd/kk/pkg/common" @@ -35,6 +36,7 @@ func NewInitDependenciesPipeline(runtime *common.KubeRuntime) error { &os.RepositoryModule{Skip: noArtifact}, &os.RepositoryOnlineModule{Skip: !noArtifact}, &filesystem.ChownWorkDirModule{}, + &customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall}, } p := pipeline.Pipeline{ diff --git a/docs/config-example.md b/docs/config-example.md index bdf724d1..8d84292e 100644 --- a/docs/config-example.md +++ b/docs/config-example.md @@ -32,6 +32,17 @@ spec: - nfs-utils debs: # Specify additional packages to be installed. The ISO file which is contained in the artifact is required. - nfs-common + #preInstall: # Specify custom init shell scripts for each nodes, and execute according to the list order. + # - name: format and mount disk + # bash: /bin/bash -x setup-disk.sh + # materials: # scripts can has some dependency materials. those will copy to the node + # - ./setup-disk.sh # the script which shell execute need + # - xxx # other tools materials need by this script + #postInstall: # Specify custom finish clean up shell scripts for each nodes after the kubernetes install. + # - name: clean tmps files + # bash: | + # rm -fr /tmp/kubekey/* + kubernetes: version: v1.21.5 imageRepo: kubesphere