15308555518/ kubekey
1
0
Fork 0
mirror of https://github.com/kubesphere/kubekey.git synced 2025-12-26 01:22:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

add security enhancement

This commit is contained in:
zhouqiu0103 2022-09-13 16:52:36 +08:00
parent bcf30d806e
commit c2600551d8
6 changed files with 328 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
cmd/ctl/create/cluster.go
View File

@ -18,6 +18,8 @@ package create
import (
"fmt"
"time"
"github.com/kubesphere/kubekey/cmd/ctl/options"
"github.com/kubesphere/kubekey/cmd/ctl/util"
"github.com/kubesphere/kubekey/pkg/common"
@ -25,23 +27,23 @@ import (
"github.com/kubesphere/kubekey/pkg/version/kubernetes"
"github.com/kubesphere/kubekey/pkg/version/kubesphere"
"github.com/spf13/cobra"
"time"
)
type CreateClusterOptions struct {
CommonOptions *options.CommonOptions
ClusterCfgFile string
Kubernetes string
EnableKubeSphere bool
KubeSphere string
LocalStorage bool
SkipPullImages bool
SkipPushImages bool
ContainerManager string
DownloadCmd string
Artifact string
InstallPackages bool
ClusterCfgFile string
Kubernetes string
EnableKubeSphere bool
KubeSphere string
LocalStorage bool
SkipPullImages bool
SkipPushImages bool
SecurityEnhancement bool
ContainerManager string
DownloadCmd string
Artifact string
InstallPackages bool
localStorageChanged bool
}
@ -104,20 +106,21 @@ func (o *CreateClusterOptions) Validate(_ *cobra.Command, _ []string) error {
func (o *CreateClusterOptions) Run() error {
arg := common.Argument{
FilePath: o.ClusterCfgFile,
KubernetesVersion: o.Kubernetes,
KsEnable: o.EnableKubeSphere,
KsVersion: o.KubeSphere,
SkipPullImages: o.SkipPullImages,
SKipPushImages: o.SkipPushImages,
InCluster: o.CommonOptions.InCluster,
Debug: o.CommonOptions.Verbose,
IgnoreErr: o.CommonOptions.IgnoreErr,
SkipConfirmCheck: o.CommonOptions.SkipConfirmCheck,
ContainerManager: o.ContainerManager,
Artifact: o.Artifact,
InstallPackages: o.InstallPackages,
Namespace: o.CommonOptions.Namespace,
FilePath: o.ClusterCfgFile,
KubernetesVersion: o.Kubernetes,
KsEnable: o.EnableKubeSphere,
KsVersion: o.KubeSphere,
SkipPullImages: o.SkipPullImages,
SKipPushImages: o.SkipPushImages,
SecurityEnhancement: o.SecurityEnhancement,
InCluster: o.CommonOptions.InCluster,
Debug: o.CommonOptions.Verbose,
IgnoreErr: o.CommonOptions.IgnoreErr,
SkipConfirmCheck: o.CommonOptions.SkipConfirmCheck,
ContainerManager: o.ContainerManager,
Artifact: o.Artifact,
InstallPackages: o.InstallPackages,
Namespace: o.CommonOptions.Namespace,
}
if o.localStorageChanged {
@ -135,6 +138,7 @@ func (o *CreateClusterOptions) AddFlags(cmd *cobra.Command) {
cmd.Flags().BoolVarP(&o.EnableKubeSphere, "with-kubesphere", "", false, fmt.Sprintf("Deploy a specific version of kubesphere (default %s)", kubesphere.Latest().Version))
cmd.Flags().BoolVarP(&o.SkipPullImages, "skip-pull-images", "", false, "Skip pre pull images")
cmd.Flags().BoolVarP(&o.SkipPushImages, "skip-push-images", "", false, "Skip pre push images")
cmd.Flags().BoolVarP(&o.SecurityEnhancement, "with-security-enhancement", "", false, "Security enhancement")
cmd.Flags().StringVarP(&o.ContainerManager, "container-manager", "", "docker", "Container runtime: docker, crio, containerd and isula.")
cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s",
`The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`)

43
pkg/common/kube_runtime.go
View File

@ -32,27 +32,28 @@ type KubeRuntime struct {
}
type Argument struct {
NodeName string
FilePath string
KubernetesVersion string
KsEnable bool
KsVersion string
Debug bool
IgnoreErr bool
SkipPullImages bool
SKipPushImages bool
DeployLocalStorage *bool
DownloadCommand func(path, url string) string
SkipConfirmCheck bool
InCluster bool
ContainerManager string
FromCluster bool
KubeConfig string
Artifact string
InstallPackages bool
ImagesDir string
Namespace string
DeleteCRI bool
NodeName string
FilePath string
KubernetesVersion string
KsEnable bool
KsVersion string
Debug bool
IgnoreErr bool
SkipPullImages bool
SKipPushImages bool
SecurityEnhancement bool
DeployLocalStorage *bool
DownloadCommand func(path, url string) string
SkipConfirmCheck bool
InCluster bool
ContainerManager string
FromCluster bool
KubeConfig string
Artifact string
InstallPackages bool
ImagesDir string
Namespace string
DeleteCRI bool
}
func NewKubeRuntime(flag string, arg Argument) (*KubeRuntime, error) {

54
pkg/kubernetes/module.go
View File

@ -143,7 +143,10 @@ func (i *InitKubernetesModule) Init() {
new(common.OnlyFirstMaster),
&ClusterIsExist{Not: true},
},
Action: &GenerateKubeadmConfig{IsInitConfiguration: true},
Action: &GenerateKubeadmConfig{
IsInitConfiguration: true,
WithSecurityEnhancement: i.KubeConf.Arg.SecurityEnhancement,
},
Parallel: true,
}
@ -226,7 +229,10 @@ func (j *JoinNodesModule) Init() {
Prepare: &prepare.PrepareCollection{
&NodeInCluster{Not: true},
},
Action: &GenerateKubeadmConfig{IsInitConfiguration: false},
Action: &GenerateKubeadmConfig{
IsInitConfiguration: false,
WithSecurityEnhancement: j.KubeConf.Arg.SecurityEnhancement,
},
Parallel: true,
}
@ -598,3 +604,47 @@ func (c *ConfigureKubernetesModule) Init() {
configure,
}
}
type SecurityEnhancementModule struct {
common.KubeModule
Skip bool
}
func (s *SecurityEnhancementModule) IsSkip() bool {
return s.Skip
}
func (s *SecurityEnhancementModule) Init() {
s.Name = "SecurityEnhancementModule"
s.Desc = "Security enhancement for the cluster"
etcdSecurityEnhancement := &task.RemoteTask{
Name: "EtcdSecurityEnhancementTask",
Desc: "Security enhancement for etcd",
Hosts: s.Runtime.GetHostsByRole(common.ETCD),
Action: new(EtcdSecurityEnhancemenAction),
Parallel: true,
}
masterSecurityEnhancement := &task.RemoteTask{
Name: "K8sSecurityEnhancementTask",
Desc: "Security enhancement for kubernetes",
Hosts: s.Runtime.GetHostsByRole(common.Master),
Action: new(MasterSecurityEnhancemenAction),
Parallel: true,
}
nodesSecurityEnhancement := &task.RemoteTask{
Name: "K8sSecurityEnhancementTask",
Desc: "Security enhancement for kubernetes",
Hosts: s.Runtime.GetHostsByRole(common.Worker),
Action: new(NodesSecurityEnhancemenAction),
Parallel: true,
}
s.Tasks = []task.Interface{
etcdSecurityEnhancement,
masterSecurityEnhancement,
nodesSecurityEnhancement,
}
}

157
pkg/kubernetes/tasks.go
View File

@ -204,7 +204,8 @@ func (g *GenerateKubeletEnv) Execute(runtime connector.Runtime) error {
type GenerateKubeadmConfig struct {
common.KubeAction
IsInitConfiguration bool
IsInitConfiguration bool
WithSecurityEnhancement bool
}
func (g *GenerateKubeadmConfig) Execute(runtime connector.Runtime) error {
@ -248,9 +249,9 @@ func (g *GenerateKubeadmConfig) Execute(runtime connector.Runtime) error {
}
}
_, ApiServerArgs := util.GetArgs(v1beta2.ApiServerArgs, g.KubeConf.Cluster.Kubernetes.ApiServerArgs)
_, ControllerManagerArgs := util.GetArgs(v1beta2.ControllermanagerArgs, g.KubeConf.Cluster.Kubernetes.ControllerManagerArgs)
_, SchedulerArgs := util.GetArgs(v1beta2.SchedulerArgs, g.KubeConf.Cluster.Kubernetes.SchedulerArgs)
_, ApiServerArgs := util.GetArgs(v1beta2.GetApiServerArgs(g.WithSecurityEnhancement), g.KubeConf.Cluster.Kubernetes.ApiServerArgs)
_, ControllerManagerArgs := util.GetArgs(v1beta2.GetControllermanagerArgs(g.WithSecurityEnhancement), g.KubeConf.Cluster.Kubernetes.ControllerManagerArgs)
_, SchedulerArgs := util.GetArgs(v1beta2.GetSchedulerArgs(g.WithSecurityEnhancement), g.KubeConf.Cluster.Kubernetes.SchedulerArgs)
checkCgroupDriver, err := v1beta2.GetKubeletCgroupDriver(runtime, g.KubeConf)
if err != nil {
@ -298,7 +299,7 @@ func (g *GenerateKubeadmConfig) Execute(runtime connector.Runtime) error {
"ApiServerArgs": v1beta2.UpdateFeatureGatesConfiguration(ApiServerArgs, g.KubeConf),
"ControllerManagerArgs": v1beta2.UpdateFeatureGatesConfiguration(ControllerManagerArgs, g.KubeConf),
"SchedulerArgs": v1beta2.UpdateFeatureGatesConfiguration(SchedulerArgs, g.KubeConf),
"KubeletConfiguration": v1beta2.GetKubeletConfiguration(runtime, g.KubeConf, g.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint),
"KubeletConfiguration": v1beta2.GetKubeletConfiguration(runtime, g.KubeConf, g.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint, g.WithSecurityEnhancement),
"KubeProxyConfiguration": v1beta2.GetKubeProxyConfiguration(g.KubeConf),
"IsControlPlane": host.IsRole(common.Master),
"CgroupDriver": checkCgroupDriver,
@ -1071,3 +1072,149 @@ func (c *ConfigureKubernetes) Execute(runtime connector.Runtime) error {
}
return nil
}
type EtcdSecurityEnhancemenAction struct {
common.KubeAction
ModuleName string
}
func (s *EtcdSecurityEnhancemenAction) Execute(runtime connector.Runtime) error {
chmodEtcdCertsDirCmd := "chmod 700 /etc/ssl/etcd/ssl"
chmodEtcdCertsCmd := "chmod 600 /etc/ssl/etcd/ssl/*"
chmodEtcdDataDirCmd := "chmod 700 /var/lib/etcd"
chmodEtcdCmd := "chmod 550 /usr/local/bin/etcd*"
chownEtcdCertsDirCmd := "chown root:root /etc/ssl/etcd/ssl"
chownEtcdCertsCmd := "chown root:root /etc/ssl/etcd/ssl/*"
chownEtcdDataDirCmd := "chown etcd:etcd /var/lib/etcd"
chownEtcdCmd := "chown root:root /usr/local/bin/etcd*"
ETCDcmds := []string{chmodEtcdCertsDirCmd, chmodEtcdCertsCmd, chmodEtcdDataDirCmd, chmodEtcdCmd, chownEtcdCertsDirCmd, chownEtcdCertsCmd, chownEtcdDataDirCmd, chownEtcdCmd}
if _, err := runtime.GetRunner().SudoCmd(strings.Join(ETCDcmds, " && "), true); err != nil {
return errors.Wrap(errors.WithStack(err), "Updating permissions failed.")
}
return nil
}
type MasterSecurityEnhancemenAction struct {
common.KubeAction
ModuleName string
}
func (k *MasterSecurityEnhancemenAction) Execute(runtime connector.Runtime) error {
// Control-plane Security Enhancemen
chmodKubernetesDirCmd := "chmod 644 /etc/kubernetes"
chownKubernetesDirCmd := "chown root:root /etc/kubernetes"
chmodKubernetesConfigCmd := "chmod 600 -R /etc/kubernetes"
chownKubernetesConfigCmd := "chown root:root -R /etc/kubernetes/*"
chmodKubenretesManifestsDirCmd := "chmod 644 /etc/kubernetes/manifests"
chownKubenretesManifestsDirCmd := "chown root:root /etc/kubernetes/manifests"
chmodKubenretesCertsDirCmd := "chmod 644 /etc/kubernetes/pki"
chownKubenretesCertsDirCmd := "chown root:root /etc/kubernetes/pki"
// node Security Enhancemen
chmodCniConfigDir := "chmod 600 -R /etc/cni/net.d"
chownCniConfigDir := "chown root:root -R /etc/cni/net.d"
chmodBinDir := "chmod 550 /usr/local/bin/"
chownBinDir := "chown root:root /usr/local/bin/"
chmodKubeCmd := "chmod 550 -R /usr/local/bin/kube*"
chownKubeCmd := "chown root:root -R /usr/local/bin/kube*"
chmodHelmCmd := "chmod 550 /usr/local/bin/helm"
chownHelmCmd := "chown root:root /usr/local/bin/helm"
chmodCniDir := "chmod 550 -R /opt/cni/bin"
chownCniDir := "chown root:root -R /opt/cni/bin"
chmodKubeletConfig := "chmod 640 /var/lib/kubelet/config.yaml && chmod 640 -R /etc/systemd/system/kubelet.service*"
chownKubeletConfig := "chown root:root /var/lib/kubelet/config.yaml && chown root:root -R /etc/systemd/system/kubelet.service*"
chmodCertsRenew := "chmod 640 /etc/systemd/system/k8s-certs-renew*"
chownCertsRenew := "chown root:root /etc/systemd/system/k8s-certs-renew*"
chmodMasterCmds := []string{chmodKubernetesConfigCmd, chmodKubernetesDirCmd, chmodKubenretesManifestsDirCmd, chmodKubenretesCertsDirCmd}
if _, err := runtime.GetRunner().SudoCmd(strings.Join(chmodMasterCmds, " && "), true); err != nil {
return errors.Wrap(errors.WithStack(err), "Updating permissions failed.")
}
chownMasterCmds := []string{chownKubernetesConfigCmd, chownKubernetesDirCmd, chownKubenretesManifestsDirCmd, chownKubenretesCertsDirCmd}
if _, err := runtime.GetRunner().SudoCmd(strings.Join(chownMasterCmds, " && "), true); err != nil {
return errors.Wrap(errors.WithStack(err), "Updating permissions failed.")
}
chmodNodesCmds := []string{chmodBinDir, chmodKubeCmd, chmodHelmCmd, chmodCniDir, chmodCniConfigDir, chmodKubeletConfig, chmodCertsRenew}
if _, err := runtime.GetRunner().SudoCmd(strings.Join(chmodNodesCmds, " && "), true); err != nil {
return errors.Wrap(errors.WithStack(err), "Updating permissions failed.")
}
chownNodesCmds := []string{chownBinDir, chownKubeCmd, chownHelmCmd, chownCniDir, chownCniConfigDir, chownKubeletConfig, chownCertsRenew}
if _, err := runtime.GetRunner().SudoCmd(strings.Join(chownNodesCmds, " && "), true); err != nil {
return errors.Wrap(errors.WithStack(err), "Updating permissions failed.")
}
return nil
}
type NodesSecurityEnhancemenAction struct {
common.KubeAction
ModuleName string
}
func (n *NodesSecurityEnhancemenAction) Execute(runtime connector.Runtime) error {
// Control-plane Security Enhancemen
chmodKubernetesDirCmd := "chmod 644 /etc/kubernetes"
chownKubernetesDirCmd := "chown root:root /etc/kubernetes"
chmodKubernetesConfigCmd := "chmod 600 -R /etc/kubernetes"
chownKubernetesConfigCmd := "chown root:root -R /etc/kubernetes"
chmodKubenretesManifestsDirCmd := "chmod 644 /etc/kubernetes/manifests"
chownKubenretesManifestsDirCmd := "chown root:root /etc/kubernetes/manifests"
chmodKubenretesCertsDirCmd := "chmod 644 /etc/kubernetes/pki"
chownKubenretesCertsDirCmd := "chown root:root /etc/kubernetes/pki"
// node Security Enhancemen
chmodCniConfigDir := "chmod 600 -R /etc/cni/net.d"
chownCniConfigDir := "chown root:root -R /etc/cni/net.d"
chmodBinDir := "chmod 550 /usr/local/bin/"
chownBinDir := "chown root:root /usr/local/bin/"
chmodKubeCmd := "chmod 550 -R /usr/local/bin/kube*"
chownKubeCmd := "chown root:root -R /usr/local/bin/kube*"
chmodHelmCmd := "chmod 550 /usr/local/bin/helm"
chownHelmCmd := "chown root:root /usr/local/bin/helm"
chmodCniDir := "chmod 550 -R /opt/cni/bin"
chownCniDir := "chown root:root -R /opt/cni/bin"
chmodKubeletConfig := "chmod 640 /var/lib/kubelet/config.yaml && chmod 640 -R /etc/systemd/system/kubelet.service*"
chownKubeletConfig := "chown root:root /var/lib/kubelet/config.yaml && chown root:root -R /etc/systemd/system/kubelet.service*"
chmodMasterCmds := []string{chmodKubernetesConfigCmd, chmodKubernetesDirCmd, chmodKubenretesManifestsDirCmd, chmodKubenretesCertsDirCmd}
if _, err := runtime.GetRunner().SudoCmd(strings.Join(chmodMasterCmds, " && "), true); err != nil {
return errors.Wrap(errors.WithStack(err), "Updating permissions failed.")
}
chownMasterCmds := []string{chownKubernetesConfigCmd, chownKubernetesDirCmd, chownKubenretesManifestsDirCmd, chownKubenretesCertsDirCmd}
if _, err := runtime.GetRunner().SudoCmd(strings.Join(chownMasterCmds, " && "), true); err != nil {
return errors.Wrap(errors.WithStack(err), "Updating permissions failed.")
}
chmodNodesCmds := []string{chmodBinDir, chmodKubeCmd, chmodHelmCmd, chmodCniDir, chmodCniConfigDir, chmodKubeletConfig}
if _, err := runtime.GetRunner().SudoCmd(strings.Join(chmodNodesCmds, " && "), true); err != nil {
return errors.Wrap(errors.WithStack(err), "Updating permissions failed.")
}
chownNodesCmds := []string{chownBinDir, chownKubeCmd, chownHelmCmd, chownCniDir, chownCniConfigDir, chownKubeletConfig}
if _, err := runtime.GetRunner().SudoCmd(strings.Join(chownNodesCmds, " && "), true); err != nil {
return errors.Wrap(errors.WithStack(err), "Updating permissions failed.")
}
return nil
}

73
pkg/kubernetes/templates/v1beta2/kubeadm_config.go
View File

@ -18,10 +18,11 @@ package v1beta2
import (
"fmt"
"github.com/kubesphere/kubekey/pkg/utils"
"strings"
"text/template"
"github.com/kubesphere/kubekey/pkg/utils"
versionutil "k8s.io/apimachinery/pkg/util/version"
"github.com/lithammer/dedent"
@ -157,6 +158,13 @@ var (
"ExpandCSIVolumes": true, //k8s 1.14+
"CSIStorageCapacity": true, //k8s 1.19+
}
FeatureGatesSecurityDefaultConfiguration = map[string]bool{
"RotateKubeletServerCertificate": true, //k8s 1.7+
"TTLAfterFinished": true, //k8s 1.12+
"ExpandCSIVolumes": true, //k8s 1.14+
"CSIStorageCapacity": true, //k8s 1.19+
"SeccompDefault": true, //kubelet
}
ApiServerArgs = map[string]string{
"bind-address": "0.0.0.0",
@ -164,15 +172,62 @@ var (
"audit-log-maxbackup": "10",
"audit-log-maxsize": "100",
}
ApiServerSecurityArgs = map[string]string{
"bind-address": "0.0.0.0",
"audit-log-maxage": "30",
"audit-log-maxbackup": "10",
"audit-log-maxsize": "100",
"authorization-mode": "Node,RBAC",
// --enable-admission-plugins=EventRateLimit must have a configuration file
"enable-admission-plugins": "AlwaysPullImages,ServiceAccount,NamespaceLifecycle,NodeRestriction,LimitRanger,ResourceQuota,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,PodNodeSelector,PodSecurity",
// "audit-log-path": "/var/log/apiserver/audit.log", // need audit policy
"profiling": "false",
"request-timeout": "120s",
"service-account-lookup": "true",
"tls-min-version": "VersionTLS12",
"tls-cipher-suites": "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
}
ControllermanagerArgs = map[string]string{
"bind-address": "0.0.0.0",
"experimental-cluster-signing-duration": "87600h",
}
ControllermanagerSecurityArgs = map[string]string{
"bind-address": "127.0.0.1",
"experimental-cluster-signing-duration": "87600h",
"profiling": "false",
"terminated-pod-gc-threshold": "50",
"use-service-account-credentials": "true",
}
SchedulerArgs = map[string]string{
"bind-address": "0.0.0.0",
}
SchedulerSecurityArgs = map[string]string{
"bind-address": "127.0.0.1",
"profiling": "false",
}
)
func GetApiServerArgs(securityEnhancement bool) map[string]string {
if securityEnhancement {
return ApiServerSecurityArgs
}
return ApiServerArgs
}
func GetControllermanagerArgs(securityEnhancement bool) map[string]string {
if securityEnhancement {
return ControllermanagerSecurityArgs
}
return ControllermanagerArgs
}
func GetSchedulerArgs(securityEnhancement bool) map[string]string {
if securityEnhancement {
return SchedulerSecurityArgs
}
return SchedulerArgs
}
func UpdateFeatureGatesConfiguration(args map[string]string, kubeConf *common.KubeConf) map[string]string {
// When kubernetes version is less than 1.21,`CSIStorageCapacity` should not be set.
cmp, _ := versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).Compare("v1.21.0")
@ -197,7 +252,7 @@ func UpdateFeatureGatesConfiguration(args map[string]string, kubeConf *common.Ku
return args
}
func GetKubeletConfiguration(runtime connector.Runtime, kubeConf *common.KubeConf, criSock string) map[string]interface{} {
func GetKubeletConfiguration(runtime connector.Runtime, kubeConf *common.KubeConf, criSock string, securityEnhancement bool) map[string]interface{} {
// When kubernetes version is less than 1.21,`CSIStorageCapacity` should not be set.
cmp, _ := versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).Compare("v1.21.0")
if cmp == -1 {
@ -232,6 +287,20 @@ func GetKubeletConfiguration(runtime connector.Runtime, kubeConf *common.KubeCon
"featureGates": FeatureGatesDefaultConfiguration,
}
if securityEnhancement {
defaultKubeletConfiguration["readOnlyPort"] = 0
defaultKubeletConfiguration["protectKernelDefaults"] = true
defaultKubeletConfiguration["eventRecordQPS"] = 1
defaultKubeletConfiguration["streamingConnectionIdleTimeout"] = "5m"
defaultKubeletConfiguration["makeIPTablesUtilChains"] = true
defaultKubeletConfiguration["tlsCipherSuites"] = []string{
"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
}
defaultKubeletConfiguration["featureGates"] = FeatureGatesSecurityDefaultConfiguration
}
cgroupDriver, err := GetKubeletCgroupDriver(runtime, kubeConf)
if err != nil {
logger.Log.Fatal(err)

1
pkg/pipelines/create_cluster.go
View File

@ -90,6 +90,7 @@ func NewCreateClusterPipeline(runtime *common.KubeRuntime) error {
&kubernetes.ConfigureKubernetesModule{},
&filesystem.ChownModule{},
&certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()},
&kubernetes.SecurityEnhancementModule{Skip: !runtime.Arg.SecurityEnhancement},
&kubernetes.SaveKubeConfigModule{},
&plugins.DeployPluginsModule{},
&addons.AddonsModule{},