diff --git a/cmd/create/config.go b/cmd/create/config.go index e39aa441..46e2a57c 100644 --- a/cmd/create/config.go +++ b/cmd/create/config.go @@ -51,10 +51,12 @@ spec: plugins: {{- if .Options.LocalVolumeEnable }} localVolume: + enabled: true isDefaultClass: {{ .Options.LocalVolumeIsDefault }} {{- end }} {{- if .Options.NfsClientEnable }} nfsClient: + enabled: true isDefaultClass: {{ .Options.NfsClientIsDefault }} nfsServer: "" nfsPath: "" diff --git a/docs/config-example.md b/docs/config-example.md index cbe2be49..30cad8cf 100644 --- a/docs/config-example.md +++ b/docs/config-example.md @@ -33,11 +33,13 @@ spec: insecureRegistries: [] plugins: localVolume: + enabled: true isDefaultClass: true nfsClient: + enabled: true isDefaultClass: false - nfsServer: "" - nfsPath: "" + nfsServer: 172.16.0.2 + nfsPath: /mnt/nfs nfsVrs3Enabled: false nfsArchiveOnDelete: false ``` \ No newline at end of file diff --git a/pkg/apis/kubekey/v1alpha1/k2cluster_types.go b/pkg/apis/kubekey/v1alpha1/k2cluster_types.go index bc538c51..728e503c 100644 --- a/pkg/apis/kubekey/v1alpha1/k2cluster_types.go +++ b/pkg/apis/kubekey/v1alpha1/k2cluster_types.go @@ -19,6 +19,7 @@ type K2ClusterSpec struct { KubeCluster KubeCluster `yaml:"kubeCluster" json:"kubeCluster,omitempty"` Network NetworkConfig `yaml:"network" json:"network,omitempty"` Registry RegistryConfig `yaml:"registry" json:"registry,omitempty"` + Plugins PluginsCfg `yaml:"plugins" json:"plugins,omitempty"` } type KubeCluster struct { diff --git a/pkg/apis/kubekey/v1alpha1/plugins_types.go b/pkg/apis/kubekey/v1alpha1/plugins_types.go index adfab75c..48248d74 100644 --- a/pkg/apis/kubekey/v1alpha1/plugins_types.go +++ b/pkg/apis/kubekey/v1alpha1/plugins_types.go @@ -1,10 +1,19 @@ package v1alpha1 +type PluginsCfg struct { + LocalVolume LocalVolume `yaml:"localVolume" json:"localVolume,omitempty"` + NfsClient NfsClient `yaml:"nfsClient" json:"nfsClient,omitempty"` + GlusterFS GlusterFS `yaml:"glusterFS" json:"glusterFS,omitempty"` + CephRBD CephRBD `yaml:"cephRBD" json:"cephRBD,omitempty"` +} + type LocalVolume struct { + Enabled bool `yaml:"enabled" json:"enabled,omitempty"` IsDefaultClass bool `yaml:"isDefaultClass" json:"isDefaultClass,omitempty"` } type NfsClient struct { + Enabled bool `yaml:"enabled" json:"enabled,omitempty"` IsDefaultClass bool `yaml:"isDefaultClass" json:"isDefaultClass,omitempty"` NfsServer string `yaml:"nfsServer" json:"nfsServer,omitempty"` NfsPath string `yaml:"nfsPath" json:"nfsPath,omitempty"` @@ -13,6 +22,7 @@ type NfsClient struct { } type GlusterFS struct { + Enabled bool `yaml:"enabled" json:"enabled,omitempty"` IsDefaultClass bool RestAuthEnabled bool RestUrl string @@ -26,6 +36,7 @@ type GlusterFS struct { } type CephRBD struct { + Enabled bool `yaml:"enabled" json:"enabled,omitempty"` IsDefaultClass bool Monitors []string AdminID string diff --git a/pkg/config/config.go b/pkg/config/config.go index d131979e..6c1a0fca 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -59,6 +59,7 @@ func SetDefaultK2ClusterSpec(cfg *kubekeyapi.K2ClusterSpec) kubekeyapi.K2Cluster clusterCfg.Network = SetDefaultNetworkCfg(cfg) clusterCfg.KubeCluster = SetDefaultClusterCfg(cfg) clusterCfg.Registry = cfg.Registry + clusterCfg.Plugins = cfg.Plugins if cfg.KubeCluster.ImageRepo == "" { clusterCfg.KubeCluster.ImageRepo = kubekeyapi.DefaultKubeImageRepo } diff --git a/pkg/install/install.go b/pkg/install/install.go index 9b020701..8abb89a5 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -9,6 +9,7 @@ import ( "github.com/pixiake/kubekey/pkg/container-engine/docker" "github.com/pixiake/kubekey/pkg/images" "github.com/pixiake/kubekey/pkg/plugins/network" + "github.com/pixiake/kubekey/pkg/plugins/storage" "github.com/pixiake/kubekey/pkg/util/manager" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -42,6 +43,7 @@ func ExecTasks(mgr *manager.Manager) error { {Task: network.DeployNetworkPlugin, ErrMsg: "failed to deploy network plugin"}, {Task: kubernetes.GetJoinNodesCmd, ErrMsg: "failed to get join cmd"}, {Task: kubernetes.JoinNodesToCluster, ErrMsg: "failed to join node"}, + {Task: storage.DeployStoragePlugins, ErrMsg: "failed to deploy storage plugin"}, } for _, step := range createTasks { diff --git a/pkg/plugins/network/network.go b/pkg/plugins/network/network.go index 40c3cf3b..c296e629 100644 --- a/pkg/plugins/network/network.go +++ b/pkg/plugins/network/network.go @@ -21,14 +21,17 @@ func deployNetworkPlugin(mgr *manager.Manager, node *kubekeyapi.HostCfg, conn ss if mgr.Runner.Index == 0 { switch mgr.Cluster.Network.Plugin { case "calico": - err := deployCalico(mgr, node) - return err + if err := deployCalico(mgr, node); err != nil { + return err + } case "flannel": - err := deployFlannel(mgr) - return err + if err := deployFlannel(mgr); err != nil { + return err + } case "macvlan": - err := deployMacvlan(mgr) - return err + if err := deployMacvlan(mgr); err != nil { + return err + } default: return errors.New(fmt.Sprintf("This network plugin is not supported: %s", mgr.Cluster.Network.Plugin)) } diff --git a/pkg/plugins/storage/Local/openebs.go b/pkg/plugins/storage/local-volume/openebs.go similarity index 97% rename from pkg/plugins/storage/Local/openebs.go rename to pkg/plugins/storage/local-volume/openebs.go index 7c4d9484..45681859 100644 --- a/pkg/plugins/storage/Local/openebs.go +++ b/pkg/plugins/storage/local-volume/openebs.go @@ -1,4 +1,4 @@ -package Local +package local_volume import ( "github.com/lithammer/dedent" @@ -282,7 +282,7 @@ spec: - name: OPERATOR_NAME value: "node-disk-operator" - name: CLEANUP_JOB_IMAGE - value: quay.azk8s.cn/openebs/linux-utils:3.9 + value: kubesphere/linux-utils:3.9 --- apiVersion: apps/v1 kind: Deployment @@ -325,7 +325,7 @@ spec: - name: OPENEBS_IO_ENABLE_ANALYTICS value: "true" - name: OPENEBS_IO_HELPER_IMAGE - value: quay.azk8s.cn/openebs/openebs-tools:3.8 + value: kubesphere/openebs-tools:3.8 - name: OPENEBS_IO_INSTALLER_TYPE value: "openebs-operator-lite" livenessProbe: @@ -338,7 +338,5 @@ spec: `))) func GenerateOpenebsManifests(mgr *manager.Manager) (string, error) { - return util.Render(OpenebsTempl, util.Data{ - "ClusterIP": mgr.Cluster.ClusterIP(), - }) + return util.Render(OpenebsTempl, util.Data{}) } diff --git a/pkg/plugins/storage/Local/openebs.yaml b/pkg/plugins/storage/local-volume/openebs.yaml similarity index 100% rename from pkg/plugins/storage/Local/openebs.yaml rename to pkg/plugins/storage/local-volume/openebs.yaml diff --git a/pkg/plugins/storage/nfs-client/nfs-client.go b/pkg/plugins/storage/nfs-client/nfs-client.go new file mode 100644 index 00000000..6f91a429 --- /dev/null +++ b/pkg/plugins/storage/nfs-client/nfs-client.go @@ -0,0 +1,96 @@ +package nfs_client + +import ( + "github.com/lithammer/dedent" + "github.com/pixiake/kubekey/pkg/util" + "github.com/pixiake/kubekey/pkg/util/manager" + "text/template" +) + +var NfsClientTempl = template.Must(template.New("nfs-client").Parse( + dedent.Dedent(`# Default values for nfs-client-provisioner. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 +strategyType: Recreate + +image: + repository: kubesphere/nfs-client-provisioner + tag: v3.1.0-k8s1.11 + pullPolicy: IfNotPresent + +nfs: + server: {{ .NfsClient.NfsServer }} + path: {{ .NfsClient.NfsPath }} + mountOptions: + {{- if .NfsClient.NfsVrs3Enabled }} + - 'nfsvers=3' + {{- end }} + +# For creating the StorageClass automatically: +storageClass: + create: true + + # Set a provisioner name. If unset, a name will be generated. + # provisionerName: + + # Set StorageClass as the default StorageClass + # Ignored if storageClass.create is false + defaultClass: {{ .NfsClient.IsDefaultClass }} + + # Set a StorageClass name + # Ignored if storageClass.create is false + name: nfs-client + + # Allow volume to be expanded dynamically + allowVolumeExpansion: true + + # Method used to reclaim an obsoleted volume + reclaimPolicy: Delete + + # When set to false your PVs will not be archived by the provisioner upon deletion of the PVC. + archiveOnDelete: {{ .NfsClient.NfsArchiveOnDelete }} + +## For RBAC support: +rbac: + # Specifies whether RBAC resources should be created + create: true + +# If true, create & use Pod Security Policy resources +# https://kubernetes.io/docs/concepts/policy/pod-security-policy/ +podSecurityPolicy: + enabled: false + +## Set pod priorityClassName +# priorityClassName: "" + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + +resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} + + `))) + +func GenerateNfsClientValuesFile(mgr *manager.Manager) (string, error) { + return util.Render(NfsClientTempl, util.Data{ + "NfsClient": mgr.Cluster.Plugins.NfsClient, + }) +} diff --git a/pkg/plugins/storage/storage.go b/pkg/plugins/storage/storage.go new file mode 100644 index 00000000..32c5541c --- /dev/null +++ b/pkg/plugins/storage/storage.go @@ -0,0 +1,77 @@ +package storage + +import ( + "encoding/base64" + "fmt" + kubekeyapi "github.com/pixiake/kubekey/pkg/apis/kubekey/v1alpha1" + local_volume "github.com/pixiake/kubekey/pkg/plugins/storage/local-volume" + nfs_client "github.com/pixiake/kubekey/pkg/plugins/storage/nfs-client" + "github.com/pixiake/kubekey/pkg/util/manager" + "github.com/pixiake/kubekey/pkg/util/ssh" + "github.com/pkg/errors" +) + +func DeployStoragePlugins(mgr *manager.Manager) error { + mgr.Logger.Infoln("Deploy storage plugin") + + return mgr.RunTaskOnMasterNodes(deployStoragePlugins, true) +} + +func deployStoragePlugins(mgr *manager.Manager, node *kubekeyapi.HostCfg, conn ssh.Connection) error { + if mgr.Runner.Index == 0 { + mgr.Runner.RunCmd("sudo -E /bin/sh -c \"mkdir -p /etc/kubernetes/addons\" && /usr/local/bin/helm repo add kubesphere https://charts.kubesphere.io/qingcloud") + if mgr.Cluster.Plugins.LocalVolume.Enabled { + if err := DeployLocalVolume(mgr); err != nil { + return err + } + } + if mgr.Cluster.Plugins.NfsClient.Enabled { + if err := DeployNfsClient(mgr); err != nil { + return err + } + } + } + return nil +} + +func DeployLocalVolume(mgr *manager.Manager) error { + localVolumeFile, err := local_volume.GenerateOpenebsManifests(mgr) + if err != nil { + return err + } + localVolumeFileBase64 := base64.StdEncoding.EncodeToString([]byte(localVolumeFile)) + _, err1 := mgr.Runner.RunCmd(fmt.Sprintf("sudo -E /bin/sh -c \"echo %s | base64 -d > /etc/kubernetes/addons/local-volume.yaml\"", localVolumeFileBase64)) + if err1 != nil { + return errors.Wrap(errors.WithStack(err1), "failed to generate local-volume file") + } + + _, err2 := mgr.Runner.RunCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/addons/local-volume.yaml") + if err2 != nil { + return errors.Wrap(errors.WithStack(err2), "failed to deploy local-volume.yaml") + } + return nil +} + +func DeployNfsClient(mgr *manager.Manager) error { + + _, err1 := mgr.Runner.RunCmd("sudo -E /bin/sh -c \"rm -rf /etc/kubernetes/addons/nfs-client-provisioner && /usr/local/bin/helm fetch kubesphere/nfs-client-provisioner -d /etc/kubernetes/addons --untar\"") + if err1 != nil { + return errors.Wrap(errors.WithStack(err1), "failed to fetch nfs-client-provisioner chart") + } + + nfsClientValuesFile, err := nfs_client.GenerateNfsClientValuesFile(mgr) + if err != nil { + return err + } + nfsClientValuesFileBase64 := base64.StdEncoding.EncodeToString([]byte(nfsClientValuesFile)) + _, err2 := mgr.Runner.RunCmd(fmt.Sprintf("sudo -E /bin/sh -c \"echo %s | base64 -d > /etc/kubernetes/addons/custom-values-nfs-client.yaml\"", nfsClientValuesFileBase64)) + if err2 != nil { + return errors.Wrap(errors.WithStack(err2), "failed to generate nfs-client values file") + } + + _, err3 := mgr.Runner.RunCmd("sudo -E /bin/sh -c \"/usr/local/bin/helm upgrade -i nfs-client /etc/kubernetes/addons/nfs-client-provisioner -f /etc/kubernetes/addons/custom-values-nfs-client.yaml -n kube-system\"") + if err3 != nil { + return errors.Wrap(errors.WithStack(err3), "failed to deploy nfs-client-provisioner") + } + return nil +} diff --git a/pkg/reset/reset.go b/pkg/reset/reset.go index 9248278a..4182bdb1 100644 --- a/pkg/reset/reset.go +++ b/pkg/reset/reset.go @@ -1,7 +1,6 @@ package reset import ( - "encoding/json" "fmt" kubekeyapi "github.com/pixiake/kubekey/pkg/apis/kubekey/v1alpha1" "github.com/pixiake/kubekey/pkg/cluster/preinstall" @@ -18,8 +17,8 @@ func ResetCluster(clusterCfgFile string, logger *log.Logger) error { return errors.Wrap(err, "failed to download cluster config") } - out, _ := json.MarshalIndent(cfg, "", " ") - fmt.Println(string(out)) + //out, _ := json.MarshalIndent(cfg, "", " ") + //fmt.Println(string(out)) if err := preinstall.Prepare(&cfg.Spec, logger); err != nil { return errors.Wrap(err, "failed to load kube binarys") }