mirror of
https://github.com/kubesphere/kubekey.git
synced 2025-12-27 02:42:48 +00:00
419 lines
16 KiB
Go
419 lines
16 KiB
Go
/*
|
|
Copyright 2020 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package clusterctl
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"path"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
. "github.com/onsi/ginkgo"
|
|
. "github.com/onsi/gomega"
|
|
clusterctlclient "sigs.k8s.io/cluster-api/cmd/clusterctl/client"
|
|
clusterctllog "sigs.k8s.io/cluster-api/cmd/clusterctl/log"
|
|
|
|
"github.com/kubesphere/kubekey/test/e2e/framework/clusterctl/logger"
|
|
"github.com/kubesphere/kubekey/test/e2e/framework/internal/log"
|
|
)
|
|
|
|
// Provide E2E friendly wrappers for the clusterctl client library.
|
|
|
|
const (
|
|
// DefaultFlavor for ConfigClusterInput; use it for getting the cluster-template.yaml file.
|
|
DefaultFlavor = ""
|
|
)
|
|
|
|
const (
|
|
// DefaultInfrastructureProvider for ConfigClusterInput; use it for using the only infrastructure provider installed in a cluster.
|
|
DefaultInfrastructureProvider = ""
|
|
)
|
|
|
|
// InitInput is the input for Init.
|
|
type InitInput struct {
|
|
LogFolder string
|
|
ClusterctlConfigPath string
|
|
KubeconfigPath string
|
|
CoreProvider string
|
|
BootstrapProviders []string
|
|
ControlPlaneProviders []string
|
|
InfrastructureProviders []string
|
|
}
|
|
|
|
// Init calls clusterctl init with the list of providers defined in the local repository.
|
|
func Init(_ context.Context, input InitInput) {
|
|
log.Logf("clusterctl init --core %s --bootstrap %s --control-plane %s --infrastructure %s --config %s --kubeconfig %s",
|
|
input.CoreProvider,
|
|
strings.Join(input.BootstrapProviders, ","),
|
|
strings.Join(input.ControlPlaneProviders, ","),
|
|
strings.Join(input.InfrastructureProviders, ","),
|
|
input.ClusterctlConfigPath,
|
|
input.KubeconfigPath,
|
|
)
|
|
|
|
initOpt := clusterctlclient.InitOptions{
|
|
Kubeconfig: clusterctlclient.Kubeconfig{
|
|
Path: input.KubeconfigPath,
|
|
Context: "",
|
|
},
|
|
CoreProvider: input.CoreProvider,
|
|
BootstrapProviders: input.BootstrapProviders,
|
|
ControlPlaneProviders: input.ControlPlaneProviders,
|
|
InfrastructureProviders: input.InfrastructureProviders,
|
|
LogUsageInstructions: true,
|
|
WaitProviders: true,
|
|
}
|
|
|
|
clusterctlClient, log := getClusterctlClientWithLogger(input.ClusterctlConfigPath, "clusterctl-init.log", input.LogFolder)
|
|
defer log.Close()
|
|
|
|
_, err := clusterctlClient.Init(initOpt)
|
|
Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl init")
|
|
}
|
|
|
|
// InitWithBinary uses clusterctl binary to run init with the list of providers defined in the local repository.
|
|
func InitWithBinary(_ context.Context, binary string, input InitInput) {
|
|
log.Logf("clusterctl init --core %s --bootstrap %s --control-plane %s --infrastructure %s --config %s --kubeconfig %s",
|
|
input.CoreProvider,
|
|
strings.Join(input.BootstrapProviders, ","),
|
|
strings.Join(input.ControlPlaneProviders, ","),
|
|
strings.Join(input.InfrastructureProviders, ","),
|
|
input.ClusterctlConfigPath,
|
|
input.KubeconfigPath,
|
|
)
|
|
|
|
args := []string{"init", "--config", input.ClusterctlConfigPath, "--kubeconfig", input.KubeconfigPath}
|
|
if input.CoreProvider != "" {
|
|
args = append(args, "--core", input.CoreProvider)
|
|
}
|
|
if len(input.BootstrapProviders) > 0 {
|
|
args = append(args, "--bootstrap", strings.Join(input.BootstrapProviders, ","))
|
|
}
|
|
if len(input.InfrastructureProviders) > 0 {
|
|
args = append(args, "--infrastructure", strings.Join(input.InfrastructureProviders, ","))
|
|
}
|
|
|
|
cmd := exec.Command(binary, args...) //nolint:gosec // We don't care about command injection here.
|
|
|
|
out, err := cmd.CombinedOutput()
|
|
_ = os.WriteFile(filepath.Join(input.LogFolder, "clusterctl-init.log"), out, 0644) //nolint:gosec // this is a log file to be shared via prow artifacts
|
|
var stdErr string
|
|
if err != nil {
|
|
if exitErr, ok := err.(*exec.ExitError); ok {
|
|
stdErr = string(exitErr.Stderr)
|
|
}
|
|
}
|
|
Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl init:\nstdout:\n%s\nstderr:\n%s", string(out), stdErr)
|
|
}
|
|
|
|
// UpgradeInput is the input for Upgrade.
|
|
type UpgradeInput struct {
|
|
LogFolder string
|
|
ClusterctlConfigPath string
|
|
ClusterctlVariables map[string]string
|
|
ClusterName string
|
|
KubeconfigPath string
|
|
Contract string
|
|
CoreProvider string
|
|
BootstrapProviders []string
|
|
ControlPlaneProviders []string
|
|
InfrastructureProviders []string
|
|
IPAMProviders []string
|
|
RuntimeExtensionProviders []string
|
|
}
|
|
|
|
// Upgrade calls clusterctl upgrade apply with the list of providers defined in the local repository.
|
|
func Upgrade(ctx context.Context, input UpgradeInput) {
|
|
if len(input.ClusterctlVariables) > 0 {
|
|
outputPath := filepath.Join(filepath.Dir(input.ClusterctlConfigPath), fmt.Sprintf("clusterctl-upgrade-config-%s.yaml", input.ClusterName))
|
|
copyAndAmendClusterctlConfig(ctx, copyAndAmendClusterctlConfigInput{
|
|
ClusterctlConfigPath: input.ClusterctlConfigPath,
|
|
OutputPath: outputPath,
|
|
Variables: input.ClusterctlVariables,
|
|
})
|
|
input.ClusterctlConfigPath = outputPath
|
|
}
|
|
|
|
// Check if the user want a custom upgrade
|
|
isCustomUpgrade := input.CoreProvider != "" ||
|
|
len(input.BootstrapProviders) > 0 ||
|
|
len(input.ControlPlaneProviders) > 0 ||
|
|
len(input.InfrastructureProviders) > 0 ||
|
|
len(input.IPAMProviders) > 0 ||
|
|
len(input.RuntimeExtensionProviders) > 0
|
|
|
|
Expect((input.Contract != "" && !isCustomUpgrade) || (input.Contract == "" && isCustomUpgrade)).To(BeTrue(), `Invalid arguments. Either the input.Contract parameter or at least one of the following providers has to be set:
|
|
input.CoreProvider, input.BootstrapProviders, input.ControlPlaneProviders, input.InfrastructureProviders, input.IPAMProviders, input.RuntimeExtensionProviders`)
|
|
|
|
if isCustomUpgrade {
|
|
log.Logf("clusterctl upgrade apply --core %s --bootstrap %s --control-plane %s --infrastructure %s --ipam %s --runtime-extension %s --config %s --kubeconfig %s",
|
|
input.CoreProvider,
|
|
strings.Join(input.BootstrapProviders, ","),
|
|
strings.Join(input.ControlPlaneProviders, ","),
|
|
strings.Join(input.InfrastructureProviders, ","),
|
|
strings.Join(input.IPAMProviders, ","),
|
|
strings.Join(input.RuntimeExtensionProviders, ","),
|
|
input.ClusterctlConfigPath,
|
|
input.KubeconfigPath,
|
|
)
|
|
} else {
|
|
log.Logf("clusterctl upgrade apply --contract %s --config %s --kubeconfig %s",
|
|
input.Contract,
|
|
input.ClusterctlConfigPath,
|
|
input.KubeconfigPath,
|
|
)
|
|
}
|
|
|
|
upgradeOpt := clusterctlclient.ApplyUpgradeOptions{
|
|
Kubeconfig: clusterctlclient.Kubeconfig{
|
|
Path: input.KubeconfigPath,
|
|
Context: "",
|
|
},
|
|
Contract: input.Contract,
|
|
CoreProvider: input.CoreProvider,
|
|
BootstrapProviders: input.BootstrapProviders,
|
|
ControlPlaneProviders: input.ControlPlaneProviders,
|
|
InfrastructureProviders: input.InfrastructureProviders,
|
|
WaitProviders: true,
|
|
}
|
|
|
|
clusterctlClient, log := getClusterctlClientWithLogger(input.ClusterctlConfigPath, "clusterctl-upgrade.log", input.LogFolder)
|
|
defer log.Close()
|
|
|
|
err := clusterctlClient.ApplyUpgrade(upgradeOpt)
|
|
Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl upgrade")
|
|
}
|
|
|
|
// DeleteInput is the input for Delete.
|
|
type DeleteInput struct {
|
|
LogFolder string
|
|
ClusterctlConfigPath string
|
|
KubeconfigPath string
|
|
}
|
|
|
|
// Delete calls clusterctl delete --all.
|
|
func Delete(_ context.Context, input DeleteInput) {
|
|
log.Logf("clusterctl delete --all")
|
|
|
|
deleteOpts := clusterctlclient.DeleteOptions{
|
|
Kubeconfig: clusterctlclient.Kubeconfig{
|
|
Path: input.KubeconfigPath,
|
|
Context: "",
|
|
},
|
|
DeleteAll: true,
|
|
}
|
|
|
|
clusterctlClient, log := getClusterctlClientWithLogger(input.ClusterctlConfigPath, "clusterctl-delete.log", input.LogFolder)
|
|
defer log.Close()
|
|
|
|
err := clusterctlClient.Delete(deleteOpts)
|
|
Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl upgrade")
|
|
}
|
|
|
|
// ConfigClusterInput is the input for ConfigCluster.
|
|
type ConfigClusterInput struct {
|
|
LogFolder string
|
|
ClusterctlConfigPath string
|
|
KubeconfigPath string
|
|
InfrastructureProvider string
|
|
Namespace string
|
|
ClusterName string
|
|
KubernetesVersion string
|
|
ControlPlaneMachineCount *int64
|
|
WorkerMachineCount *int64
|
|
Flavor string
|
|
ClusterctlVariables map[string]string
|
|
}
|
|
|
|
// ConfigCluster gets a workload cluster based on a template.
|
|
func ConfigCluster(ctx context.Context, input ConfigClusterInput) []byte {
|
|
log.Logf("clusterctl config cluster %s --infrastructure %s --kubernetes-version %s --control-plane-machine-count %d --worker-machine-count %d --flavor %s",
|
|
input.ClusterName,
|
|
valueOrDefault(input.InfrastructureProvider),
|
|
input.KubernetesVersion,
|
|
*input.ControlPlaneMachineCount,
|
|
*input.WorkerMachineCount,
|
|
valueOrDefault(input.Flavor),
|
|
)
|
|
|
|
templateOptions := clusterctlclient.GetClusterTemplateOptions{
|
|
Kubeconfig: clusterctlclient.Kubeconfig{
|
|
Path: input.KubeconfigPath,
|
|
Context: "",
|
|
},
|
|
ProviderRepositorySource: &clusterctlclient.ProviderRepositorySourceOptions{
|
|
InfrastructureProvider: input.InfrastructureProvider,
|
|
Flavor: input.Flavor,
|
|
},
|
|
ClusterName: input.ClusterName,
|
|
KubernetesVersion: input.KubernetesVersion,
|
|
ControlPlaneMachineCount: input.ControlPlaneMachineCount,
|
|
WorkerMachineCount: input.WorkerMachineCount,
|
|
TargetNamespace: input.Namespace,
|
|
}
|
|
|
|
if len(input.ClusterctlVariables) > 0 {
|
|
outputPath := filepath.Join(filepath.Dir(input.ClusterctlConfigPath), fmt.Sprintf("clusterctl-upgrade-config-%s.yaml", input.ClusterName))
|
|
copyAndAmendClusterctlConfig(ctx, copyAndAmendClusterctlConfigInput{
|
|
ClusterctlConfigPath: input.ClusterctlConfigPath,
|
|
OutputPath: outputPath,
|
|
Variables: input.ClusterctlVariables,
|
|
})
|
|
input.ClusterctlConfigPath = outputPath
|
|
}
|
|
|
|
clusterctlClient, log := getClusterctlClientWithLogger(input.ClusterctlConfigPath, fmt.Sprintf("%s-cluster-template.yaml", input.ClusterName), input.LogFolder)
|
|
defer log.Close()
|
|
|
|
template, err := clusterctlClient.GetClusterTemplate(templateOptions)
|
|
Expect(err).ToNot(HaveOccurred(), "Failed to run clusterctl config cluster")
|
|
|
|
yaml, err := template.Yaml()
|
|
Expect(err).ToNot(HaveOccurred(), "Failed to generate yaml for the workload cluster template")
|
|
|
|
_, _ = log.WriteString(string(yaml))
|
|
return yaml
|
|
}
|
|
|
|
// ConfigClusterWithBinary uses clusterctl binary to run config cluster or generate cluster.
|
|
// NOTE: This func detects the clusterctl version and uses config cluster or generate cluster
|
|
// accordingly. We can drop the detection when we don't have to support clusterctl v0.3.x anymore.
|
|
func ConfigClusterWithBinary(_ context.Context, clusterctlBinaryPath string, input ConfigClusterInput) []byte {
|
|
log.Logf("Detect clusterctl version via: clusterctl version")
|
|
|
|
out, err := exec.Command(clusterctlBinaryPath, "version").Output()
|
|
Expect(err).ToNot(HaveOccurred(), "error running clusterctl version")
|
|
var clusterctlSupportsGenerateCluster bool
|
|
if strings.Contains(string(out), "Major:\"1\"") {
|
|
log.Logf("Detected clusterctl v1.x")
|
|
clusterctlSupportsGenerateCluster = true
|
|
}
|
|
|
|
var cmd *exec.Cmd
|
|
if clusterctlSupportsGenerateCluster {
|
|
log.Logf("clusterctl generate cluster %s --infrastructure %s --kubernetes-version %s --control-plane-machine-count %d --worker-machine-count %d --flavor %s",
|
|
input.ClusterName,
|
|
valueOrDefault(input.InfrastructureProvider),
|
|
input.KubernetesVersion,
|
|
*input.ControlPlaneMachineCount,
|
|
*input.WorkerMachineCount,
|
|
valueOrDefault(input.Flavor),
|
|
)
|
|
cmd = exec.Command(clusterctlBinaryPath, "generate", "cluster", //nolint:gosec // We don't care about command injection here.
|
|
input.ClusterName,
|
|
"--infrastructure", input.InfrastructureProvider,
|
|
"--kubernetes-version", input.KubernetesVersion,
|
|
"--control-plane-machine-count", fmt.Sprint(*input.ControlPlaneMachineCount),
|
|
"--worker-machine-count", fmt.Sprint(*input.WorkerMachineCount),
|
|
"--flavor", input.Flavor,
|
|
"--target-namespace", input.Namespace,
|
|
"--config", input.ClusterctlConfigPath,
|
|
"--kubeconfig", input.KubeconfigPath,
|
|
)
|
|
} else {
|
|
log.Logf("clusterctl config cluster %s --infrastructure %s --kubernetes-version %s --control-plane-machine-count %d --worker-machine-count %d --flavor %s",
|
|
input.ClusterName,
|
|
valueOrDefault(input.InfrastructureProvider),
|
|
input.KubernetesVersion,
|
|
*input.ControlPlaneMachineCount,
|
|
*input.WorkerMachineCount,
|
|
valueOrDefault(input.Flavor),
|
|
)
|
|
cmd = exec.Command(clusterctlBinaryPath, "config", "cluster", //nolint:gosec // We don't care about command injection here.
|
|
input.ClusterName,
|
|
"--infrastructure", input.InfrastructureProvider,
|
|
"--kubernetes-version", input.KubernetesVersion,
|
|
"--control-plane-machine-count", fmt.Sprint(*input.ControlPlaneMachineCount),
|
|
"--worker-machine-count", fmt.Sprint(*input.WorkerMachineCount),
|
|
"--flavor", input.Flavor,
|
|
"--target-namespace", input.Namespace,
|
|
"--config", input.ClusterctlConfigPath,
|
|
"--kubeconfig", input.KubeconfigPath,
|
|
)
|
|
}
|
|
|
|
out, err = cmd.Output()
|
|
_ = os.WriteFile(filepath.Join(input.LogFolder, fmt.Sprintf("%s-cluster-template.yaml", input.ClusterName)), out, 0644) //nolint:gosec // this is a log file to be shared via prow artifacts
|
|
var stdErr string
|
|
if err != nil {
|
|
if exitErr, ok := err.(*exec.ExitError); ok {
|
|
stdErr = string(exitErr.Stderr)
|
|
}
|
|
}
|
|
Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl config cluster:\nstdout:\n%s\nstderr:\n%s", string(out), stdErr)
|
|
|
|
return out
|
|
}
|
|
|
|
// MoveInput is the input for ClusterctlMove.
|
|
type MoveInput struct {
|
|
LogFolder string
|
|
ClusterctlConfigPath string
|
|
FromKubeconfigPath string
|
|
ToKubeconfigPath string
|
|
Namespace string
|
|
}
|
|
|
|
// Move moves workload clusters.
|
|
func Move(ctx context.Context, input MoveInput) {
|
|
Expect(ctx).NotTo(BeNil(), "ctx is required for Move")
|
|
Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling Move")
|
|
Expect(input.FromKubeconfigPath).To(BeAnExistingFile(), "Invalid argument. input.FromKubeconfigPath must be an existing file when calling Move")
|
|
Expect(input.ToKubeconfigPath).To(BeAnExistingFile(), "Invalid argument. input.ToKubeconfigPath must be an existing file when calling Move")
|
|
logDir := path.Join(input.LogFolder, "logs", input.Namespace)
|
|
Expect(os.MkdirAll(logDir, 0750)).To(Succeed(), "Invalid argument. input.LogFolder can't be created for Move")
|
|
|
|
By("Moving workload clusters")
|
|
log.Logf("clusterctl move --from-kubeconfig %s --to-kubeconfig %s --namespace %s",
|
|
input.FromKubeconfigPath,
|
|
input.ToKubeconfigPath,
|
|
input.Namespace,
|
|
)
|
|
|
|
clusterctlClient, log := getClusterctlClientWithLogger(input.ClusterctlConfigPath, "clusterctl-move.log", logDir)
|
|
defer log.Close()
|
|
options := clusterctlclient.MoveOptions{
|
|
FromKubeconfig: clusterctlclient.Kubeconfig{Path: input.FromKubeconfigPath, Context: ""},
|
|
ToKubeconfig: clusterctlclient.Kubeconfig{Path: input.ToKubeconfigPath, Context: ""},
|
|
Namespace: input.Namespace,
|
|
}
|
|
|
|
Expect(clusterctlClient.Move(options)).To(Succeed(), "Failed to run clusterctl move")
|
|
}
|
|
|
|
func getClusterctlClientWithLogger(configPath, logName, logFolder string) (clusterctlclient.Client, *logger.LogFile) {
|
|
log := logger.OpenLogFile(logger.OpenLogFileInput{
|
|
LogFolder: logFolder,
|
|
Name: logName,
|
|
})
|
|
clusterctllog.SetLogger(log.Logger())
|
|
|
|
c, err := clusterctlclient.New(configPath)
|
|
Expect(err).ToNot(HaveOccurred(), "Failed to create the clusterctl client library")
|
|
return c, log
|
|
}
|
|
|
|
func valueOrDefault(v string) string {
|
|
if v != "" {
|
|
return v
|
|
}
|
|
return "(default)"
|
|
}
|