Merge pull request #1545 from tanguofu/master

fea(#1541): Support Execute custom setup scripts when init os and cre…
2025-12-26 01:22:51 +00:00 · 2022-10-14 12:54:32 +08:00 · 2022-10-14 12:54:32 +08:00 · 1b21e966a0
parent 4337d491df af8ec18d50
commit 1b21e966a0
8 changed files with 212 additions and 5 deletions
--- a/api/v1beta1/zz_generated.deepcopy.go
+++ b/api/v1beta1/zz_generated.deepcopy.go
@ -2,7 +2,7 @@
 // +build !ignore_autogenerated

 /*
-Copyright 2022 The KubeSphere Authors.
+Copyright 2020 The KubeSphere Authors.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
--- a/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go
+++ b/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go
@ -85,12 +85,21 @@ type KubeVip struct {
 	Mode string `yaml:"mode" json:"mode,omitempty"`
 }

+// CustomScripts defines the custom shell scripts for each node to exec before and finished kubernetes install.
+type CustomScripts struct {
+	Name      string   `yaml:"name" json:"name,omitempty"`
+	Bash      string   `yaml:"bash" json:"shell,omitempty"`
+	Materials []string `yaml:"materials" json:"materials,omitempty"`
+}
+
 // System defines the system config for each node in cluster.
 type System struct {
-	NtpServers []string `yaml:"ntpServers" json:"ntpServers,omitempty"`
-	Timezone   string   `yaml:"timezone" json:"timezone,omitempty"`
-	Rpms       []string `yaml:"rpms" json:"rpms,omitempty"`
-	Debs       []string `yaml:"debs" json:"debs,omitempty"`
+	NtpServers  []string        `yaml:"ntpServers" json:"ntpServers,omitempty"`
+	Timezone    string          `yaml:"timezone" json:"timezone,omitempty"`
+	Rpms        []string        `yaml:"rpms" json:"rpms,omitempty"`
+	Debs        []string        `yaml:"debs" json:"debs,omitempty"`
+	PreInstall  []CustomScripts `yaml:"preInstall" json:"preInstall,omitempty"`
+	PostInstall []CustomScripts `yaml:"postInstall" json:"postInstall,omitempty"`
 }

 // RegistryConfig defines the configuration information of the image's repository.
--- a/cmd/kk/pkg/bootstrap/customscripts/module.go
+++ b/cmd/kk/pkg/bootstrap/customscripts/module.go
@ -0,0 +1,52 @@
+/*
+ Copyright 2021 The KubeSphere Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package customscripts
+
+import (
+	"fmt"
+
+	kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/cmd/kk/apis/kubekey/v1alpha2"
+	"github.com/kubesphere/kubekey/cmd/kk/pkg/core/module"
+	"github.com/kubesphere/kubekey/cmd/kk/pkg/core/task"
+)
+
+type CustomScriptsModule struct {
+	module.BaseTaskModule
+	Phase   string
+	Scripts []kubekeyapiv1alpha2.CustomScripts
+}
+
+func (m *CustomScriptsModule) Init() {
+	m.Name = fmt.Sprintf("CustomScriptsModule Phase:%s", m.Phase)
+	m.Desc = "Exec custom shell scripts for each nodes."
+
+	for idx, script := range m.Scripts {
+
+		taskName := fmt.Sprintf("Phase:%s(%d/%d) script:%s", m.Phase, idx, len(m.Scripts), script.Name)
+		taskDir := fmt.Sprintf("%s-%d-script", m.Phase, idx)
+		task := &task.RemoteTask{
+			Name:     taskName,
+			Desc:     taskName,
+			Hosts:    m.Runtime.GetAllHosts(),
+			Action:   &CustomScriptTask{taskDir: taskDir, script: script},
+			Parallel: true,
+			Retry: 1,
+		}
+
+		m.Tasks = append(m.Tasks, task)
+	}
+}
--- a/cmd/kk/pkg/bootstrap/customscripts/tasks.go
+++ b/cmd/kk/pkg/bootstrap/customscripts/tasks.go
@ -0,0 +1,120 @@
+/*
+ Copyright 2021 The KubeSphere Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package customscripts
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+
+	kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/cmd/kk/apis/kubekey/v1alpha2"
+	"github.com/kubesphere/kubekey/cmd/kk/pkg/common"
+	"github.com/kubesphere/kubekey/cmd/kk/pkg/core/action"
+	"github.com/kubesphere/kubekey/cmd/kk/pkg/core/connector"
+	"github.com/kubesphere/kubekey/cmd/kk/pkg/core/util"
+)
+
+type CustomScriptTask struct {
+	action.BaseAction
+	taskDir string
+	script  kubekeyapiv1alpha2.CustomScripts
+}
+
+
+
+func (t *CustomScriptTask) Execute(runtime connector.Runtime) error {
+
+	if len(t.script.Bash) <= 0 {
+		return errors.Errorf("custom script %s Bash is empty", t.script.Name)
+	}
+
+	remoteTaskHome := common.TmpDir + t.taskDir
+
+	if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("mkdir -p %s", remoteTaskHome), false); err != nil {
+		return errors.Wrapf(err, "create remoteTaskHome: %s  err:%s", remoteTaskHome, err)
+	}
+
+	// dilver the dependency materials to the remotehost
+	for idx, localPath := range t.script.Materials {
+
+		if !util.IsExist(localPath) {
+			return errors.Errorf("Not found Path: %s", localPath)
+		}
+
+		targetPath := filepath.Join(remoteTaskHome, filepath.Base(localPath))
+
+		// first clean the target to makesure target path always is the lastest.
+		cleanCmd := fmt.Sprintf("rm -fr %s", targetPath)
+		if _, err := runtime.GetRunner().SudoCmd(cleanCmd, false); err != nil {
+			return errors.Wrapf(err, "Can not remove target found Path: %s", targetPath)
+		}
+
+		start := time.Now()
+		err := runtime.GetRunner().SudoScp(localPath, targetPath)
+		if err != nil {
+			return errors.Wrapf(err, "Can not Scp -fr %s root@%s:%s", localPath, runtime.RemoteHost().GetAddress(), targetPath)
+		}
+
+		fmt.Printf("Copy %d/%d materials: Scp -fr %s root@%s:%s done, take %s\n",
+			idx, len(t.script.Materials), localPath, runtime.RemoteHost().GetAddress(), targetPath, time.Since(start))
+	}
+
+	// wrap use bash file if shell has many lines.
+	RunBash := t.script.Bash
+	if strings.Index(RunBash, "\n") > 0 {
+		tmpFile, err := ioutil.TempFile(os.TempDir(), t.taskDir)
+		if err != nil {
+			return errors.Wrapf(err, "create tmp Bash: %s/%s in local node, err:%s", os.TempDir(), t.taskDir, err)
+		}
+		defer os.Remove(tmpFile.Name())
+
+		if _, err := tmpFile.WriteString(RunBash); err != nil {
+			return errors.Wrapf(err, "write to tmp:%s in local node, err:%s", tmpFile.Name(), err)
+		}
+
+		targetPath := filepath.Join(remoteTaskHome, "task.sh")
+		if err := runtime.GetRunner().SudoScp(tmpFile.Name(), targetPath); err != nil {
+			return errors.Wrapf(err, "Can not Scp -fr %s root@%s:%s", tmpFile.Name(), runtime.RemoteHost().GetAddress(), targetPath)
+		}
+
+		RunBash = "/bin/bash " + targetPath
+	}
+
+	start := time.Now()
+	out, err := runtime.GetRunner().SudoCmd(RunBash, false)
+	if err != nil {
+		return errors.Errorf("Exec Bash: %s err:%s", RunBash, err)
+	}
+
+	if !runtime.GetRunner().Debug {
+		fmt.Printf("Exec Bash:%s done, take %s", RunBash, time.Since(start))
+		cleanCmd := fmt.Sprintf("rm -fr %s", remoteTaskHome)
+		if _, err := runtime.GetRunner().SudoCmd(cleanCmd, false); err != nil {
+			return errors.Wrapf(err, "Exec cmd:%s err:%s", cleanCmd, err)
+		}
+	}else {
+		// keep the Materials for debug
+		fmt.Printf("Exec Bash:%s done, take %s, output:\n%s", RunBash, time.Since(start), out)
+	}
+
+	return nil
+}
--- a/cmd/kk/pkg/pipelines/add_nodes.go
+++ b/cmd/kk/pkg/pipelines/add_nodes.go
@ -23,6 +23,7 @@ import (
 	"github.com/kubesphere/kubekey/cmd/kk/pkg/artifact"
 	"github.com/kubesphere/kubekey/cmd/kk/pkg/binaries"
 	"github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/confirm"
+	"github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/customscripts"
 	"github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/os"
 	"github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/precheck"
 	"github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/registry"
@ -51,6 +52,7 @@ func NewAddNodesPipeline(runtime *common.KubeRuntime) error {
 		&os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages},
 		&binaries.NodeBinariesModule{},
 		&os.ConfigureOSModule{},
+		&customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall},
 		&registry.RegistryCertsModule{Skip: len(runtime.GetHostsByRole(common.Registry)) == 0},
 		&kubernetes.StatusModule{},
 		&container.InstallContainerModule{},
@ -66,6 +68,7 @@ func NewAddNodesPipeline(runtime *common.KubeRuntime) error {
 		&kubernetes.ConfigureKubernetesModule{},
 		&filesystem.ChownModule{},
 		&certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()},
+		&customscripts.CustomScriptsModule{Phase: "PostInstall", Scripts: runtime.Cluster.System.PostInstall},
 	}

 	p := pipeline.Pipeline{
@ -89,6 +92,7 @@ func NewK3sAddNodesPipeline(runtime *common.KubeRuntime) error {
 		&os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages},
 		&binaries.K3sNodeBinariesModule{},
 		&os.ConfigureOSModule{},
+		&customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall},
 		&k3s.StatusModule{},
 		&etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey},
 		&etcd.CertsModule{},
@ -101,6 +105,7 @@ func NewK3sAddNodesPipeline(runtime *common.KubeRuntime) error {
 		&kubernetes.ConfigureKubernetesModule{},
 		&filesystem.ChownModule{},
 		&certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()},
+		&customscripts.CustomScriptsModule{Phase: "PostInstall", Scripts: runtime.Cluster.System.PostInstall},
 	}

 	p := pipeline.Pipeline{
@ -124,6 +129,7 @@ func NewK8eAddNodesPipeline(runtime *common.KubeRuntime) error {
 		&os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages},
 		&binaries.K8eNodeBinariesModule{},
 		&os.ConfigureOSModule{},
+
 		&k8e.StatusModule{},
 		&etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey},
 		&etcd.CertsModule{},
--- a/cmd/kk/pkg/pipelines/create_cluster.go
+++ b/cmd/kk/pkg/pipelines/create_cluster.go
@ -24,6 +24,7 @@ import (
 	"github.com/kubesphere/kubekey/cmd/kk/pkg/artifact"
 	"github.com/kubesphere/kubekey/cmd/kk/pkg/binaries"
 	"github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/confirm"
+	"github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/customscripts"
 	"github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/os"
 	"github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/precheck"
 	"github.com/kubesphere/kubekey/cmd/kk/pkg/certs"
@ -63,6 +64,7 @@ func NewCreateClusterPipeline(runtime *common.KubeRuntime) error {
 		&os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages},
 		&binaries.NodeBinariesModule{},
 		&os.ConfigureOSModule{},
+		&customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall},
 		&kubernetes.StatusModule{},
 		&container.InstallContainerModule{},
 		&images.CopyImagesToRegistryModule{Skip: skipPushImages},
@ -91,6 +93,7 @@ func NewCreateClusterPipeline(runtime *common.KubeRuntime) error {
 		&storage.DeployLocalVolumeModule{Skip: skipLocalStorage},
 		&kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled},
 		&kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled},
+		&customscripts.CustomScriptsModule{Phase: "PostInstall", Scripts: runtime.Cluster.System.PostInstall},
 	}

 	p := pipeline.Pipeline{
@ -141,6 +144,7 @@ func NewK3sCreateClusterPipeline(runtime *common.KubeRuntime) error {
 		&os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages},
 		&binaries.K3sNodeBinariesModule{},
 		&os.ConfigureOSModule{},
+		&customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall},
 		&k3s.StatusModule{},
 		&etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey},
 		&etcd.CertsModule{},
@ -163,6 +167,7 @@ func NewK3sCreateClusterPipeline(runtime *common.KubeRuntime) error {
 		&storage.DeployLocalVolumeModule{Skip: skipLocalStorage},
 		&kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled},
 		&kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled},
+		&customscripts.CustomScriptsModule{Phase: "PostInstall", Scripts: runtime.Cluster.System.PostInstall},
 	}

 	p := pipeline.Pipeline{
@ -213,6 +218,7 @@ func NewK8eCreateClusterPipeline(runtime *common.KubeRuntime) error {
 		&os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages},
 		&binaries.K8eNodeBinariesModule{},
 		&os.ConfigureOSModule{},
+		&customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall},
 		&k8e.StatusModule{},
 		&etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey},
 		&etcd.CertsModule{},
@ -235,6 +241,7 @@ func NewK8eCreateClusterPipeline(runtime *common.KubeRuntime) error {
 		&storage.DeployLocalVolumeModule{Skip: skipLocalStorage},
 		&kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled},
 		&kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled},
+		&customscripts.CustomScriptsModule{Phase: "PostInstall", Scripts: runtime.Cluster.System.PostInstall},
 	}

 	p := pipeline.Pipeline{
--- a/cmd/kk/pkg/pipelines/init_dependencies.go
+++ b/cmd/kk/pkg/pipelines/init_dependencies.go
@ -18,6 +18,7 @@ package pipelines

 import (
 	"github.com/kubesphere/kubekey/cmd/kk/pkg/artifact"
+	"github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/customscripts"
 	"github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/os"
 	"github.com/kubesphere/kubekey/cmd/kk/pkg/bootstrap/precheck"
 	"github.com/kubesphere/kubekey/cmd/kk/pkg/common"
@ -35,6 +36,7 @@ func NewInitDependenciesPipeline(runtime *common.KubeRuntime) error {
 		&os.RepositoryModule{Skip: noArtifact},
 		&os.RepositoryOnlineModule{Skip: !noArtifact},
 		&filesystem.ChownWorkDirModule{},
+		&customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall},
 	}

 	p := pipeline.Pipeline{
--- a/docs/config-example.md
+++ b/docs/config-example.md
@ -32,6 +32,17 @@ spec:
      - nfs-utils
    debs: # Specify additional packages to be installed. The ISO file which is contained in the artifact is required.
      - nfs-common
+    #preInstall:  # Specify custom init shell scripts for each nodes, and execute according to the list order.
+    #  - name: format and mount disk  
+    #    bash: /bin/bash -x setup-disk.sh
+    #    materials: # scripts can has some dependency materials. those will copy to the node        
+    #      - ./setup-disk.sh # the script which shell execute need
+    #      -  xxx            # other tools materials need by this script
+    #postInstall: # Specify custom finish clean up shell scripts for each nodes after the kubernetes install.
+    #  - name: clean tmps files
+    #    bash: |
+    #       rm -fr /tmp/kubekey/*
+    
  kubernetes:
    version: v1.21.5
    imageRepo: kubesphere