add storage plugins

cmd/create/config.go

@@ -51,10 +51,12 @@ spec:
   plugins:
     {{- if .Options.LocalVolumeEnable }}
     localVolume:
+      enabled: true
       isDefaultClass: {{ .Options.LocalVolumeIsDefault }}
     {{- end }}
     {{- if .Options.NfsClientEnable }}
     nfsClient:
+      enabled: true
       isDefaultClass: {{ .Options.NfsClientIsDefault }}
       nfsServer: ""
       nfsPath: ""

docs/config-example.md

@@ -33,11 +33,13 @@ spec:
     insecureRegistries: []
   plugins:
     localVolume:
+      enabled: true
       isDefaultClass: true
     nfsClient:
+      enabled: true
       isDefaultClass: false
-      nfsServer: ""
-      nfsPath: ""
+      nfsServer: 172.16.0.2
+      nfsPath: /mnt/nfs
       nfsVrs3Enabled: false
       nfsArchiveOnDelete: false
 ```

pkg/apis/kubekey/v1alpha1/k2cluster_types.go

@@ -19,6 +19,7 @@ type K2ClusterSpec struct {
 	KubeCluster     KubeCluster        `yaml:"kubeCluster" json:"kubeCluster,omitempty"`
 	Network         NetworkConfig      `yaml:"network" json:"network,omitempty"`
 	Registry        RegistryConfig     `yaml:"registry" json:"registry,omitempty"`
+	Plugins         PluginsCfg         `yaml:"plugins" json:"plugins,omitempty"`
 }
 
 type KubeCluster struct {

pkg/apis/kubekey/v1alpha1/plugins_types.go

@@ -1,10 +1,19 @@
 package v1alpha1
 
+type PluginsCfg struct {
+	LocalVolume LocalVolume `yaml:"localVolume" json:"localVolume,omitempty"`
+	NfsClient   NfsClient   `yaml:"nfsClient" json:"nfsClient,omitempty"`
+	GlusterFS   GlusterFS   `yaml:"glusterFS" json:"glusterFS,omitempty"`
+	CephRBD     CephRBD     `yaml:"cephRBD" json:"cephRBD,omitempty"`
+}
+
 type LocalVolume struct {
+	Enabled        bool `yaml:"enabled" json:"enabled,omitempty"`
 	IsDefaultClass bool `yaml:"isDefaultClass" json:"isDefaultClass,omitempty"`
 }
 
 type NfsClient struct {
+	Enabled            bool   `yaml:"enabled" json:"enabled,omitempty"`
 	IsDefaultClass     bool   `yaml:"isDefaultClass" json:"isDefaultClass,omitempty"`
 	NfsServer          string `yaml:"nfsServer" json:"nfsServer,omitempty"`
 	NfsPath            string `yaml:"nfsPath" json:"nfsPath,omitempty"`
@@ -13,6 +22,7 @@ type NfsClient struct {
 }
 
 type GlusterFS struct {
+	Enabled         bool `yaml:"enabled" json:"enabled,omitempty"`
 	IsDefaultClass  bool
 	RestAuthEnabled bool
 	RestUrl         string
@@ -26,6 +36,7 @@ type GlusterFS struct {
 }
 
 type CephRBD struct {
+	Enabled        bool `yaml:"enabled" json:"enabled,omitempty"`
 	IsDefaultClass bool
 	Monitors       []string
 	AdminID        string

pkg/config/config.go

@@ -59,6 +59,7 @@ func SetDefaultK2ClusterSpec(cfg *kubekeyapi.K2ClusterSpec) kubekeyapi.K2Cluster
 	clusterCfg.Network = SetDefaultNetworkCfg(cfg)
 	clusterCfg.KubeCluster = SetDefaultClusterCfg(cfg)
 	clusterCfg.Registry = cfg.Registry
+	clusterCfg.Plugins = cfg.Plugins
 	if cfg.KubeCluster.ImageRepo == "" {
 		clusterCfg.KubeCluster.ImageRepo = kubekeyapi.DefaultKubeImageRepo
 	}

pkg/install/install.go

@@ -9,6 +9,7 @@ import (
 	"github.com/pixiake/kubekey/pkg/container-engine/docker"
 	"github.com/pixiake/kubekey/pkg/images"
 	"github.com/pixiake/kubekey/pkg/plugins/network"
+	"github.com/pixiake/kubekey/pkg/plugins/storage"
 	"github.com/pixiake/kubekey/pkg/util/manager"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
@@ -42,6 +43,7 @@ func ExecTasks(mgr *manager.Manager) error {
 		{Task: network.DeployNetworkPlugin, ErrMsg: "failed to deploy network plugin"},
 		{Task: kubernetes.GetJoinNodesCmd, ErrMsg: "failed to get join cmd"},
 		{Task: kubernetes.JoinNodesToCluster, ErrMsg: "failed to join node"},
+		{Task: storage.DeployStoragePlugins, ErrMsg: "failed to deploy storage plugin"},
 	}
 
 	for _, step := range createTasks {

pkg/plugins/network/network.go

@@ -21,14 +21,17 @@ func deployNetworkPlugin(mgr *manager.Manager, node *kubekeyapi.HostCfg, conn ss
 	if mgr.Runner.Index == 0 {
 		switch mgr.Cluster.Network.Plugin {
 		case "calico":
-			err := deployCalico(mgr, node)
-			return err
+			if err := deployCalico(mgr, node); err != nil {
+				return err
+			}
 		case "flannel":
-			err := deployFlannel(mgr)
-			return err
+			if err := deployFlannel(mgr); err != nil {
+				return err
+			}
 		case "macvlan":
-			err := deployMacvlan(mgr)
-			return err
+			if err := deployMacvlan(mgr); err != nil {
+				return err
+			}
 		default:
 			return errors.New(fmt.Sprintf("This network plugin is not supported: %s", mgr.Cluster.Network.Plugin))
 		}

pkg/plugins/storage/local-volume/openebs.go

@@ -1,4 +1,4 @@
-package Local
+package local_volume
 
 import (
 	"github.com/lithammer/dedent"
@@ -282,7 +282,7 @@ spec:
             - name: OPERATOR_NAME
               value: "node-disk-operator"
             - name: CLEANUP_JOB_IMAGE
-              value: quay.azk8s.cn/openebs/linux-utils:3.9
+              value: kubesphere/linux-utils:3.9
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -325,7 +325,7 @@ spec:
             - name: OPENEBS_IO_ENABLE_ANALYTICS
               value: "true"
             - name: OPENEBS_IO_HELPER_IMAGE
-              value: quay.azk8s.cn/openebs/openebs-tools:3.8
+              value: kubesphere/openebs-tools:3.8
             - name: OPENEBS_IO_INSTALLER_TYPE
               value: "openebs-operator-lite"
           livenessProbe:
@@ -338,7 +338,5 @@ spec:
     `)))
 
 func GenerateOpenebsManifests(mgr *manager.Manager) (string, error) {
-	return util.Render(OpenebsTempl, util.Data{
-		"ClusterIP": mgr.Cluster.ClusterIP(),
-	})
+	return util.Render(OpenebsTempl, util.Data{})
 }

pkg/plugins/storage/nfs-client/nfs-client.go

@@ -0,0 +1,96 @@
+package nfs_client
+
+import (
+	"github.com/lithammer/dedent"
+	"github.com/pixiake/kubekey/pkg/util"
+	"github.com/pixiake/kubekey/pkg/util/manager"
+	"text/template"
+)
+
+var NfsClientTempl = template.Must(template.New("nfs-client").Parse(
+	dedent.Dedent(`# Default values for nfs-client-provisioner.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 1
+strategyType: Recreate
+
+image:
+  repository: kubesphere/nfs-client-provisioner
+  tag: v3.1.0-k8s1.11
+  pullPolicy: IfNotPresent
+
+nfs:
+  server: {{ .NfsClient.NfsServer }}
+  path: {{ .NfsClient.NfsPath }}
+  mountOptions:
+  {{- if .NfsClient.NfsVrs3Enabled }}
+  - 'nfsvers=3'
+  {{- end }}
+
+# For creating the StorageClass automatically:
+storageClass:
+  create: true
+
+  # Set a provisioner name. If unset, a name will be generated.
+  # provisionerName:
+
+  # Set StorageClass as the default StorageClass
+  # Ignored if storageClass.create is false
+  defaultClass: {{ .NfsClient.IsDefaultClass }}
+
+  # Set a StorageClass name
+  # Ignored if storageClass.create is false
+  name: nfs-client
+
+  # Allow volume to be expanded dynamically
+  allowVolumeExpansion: true
+
+  # Method used to reclaim an obsoleted volume
+  reclaimPolicy: Delete
+
+  # When set to false your PVs will not be archived by the provisioner upon deletion of the PVC.
+  archiveOnDelete: {{ .NfsClient.NfsArchiveOnDelete }}
+
+## For RBAC support:
+rbac:
+  # Specifies whether RBAC resources should be created
+  create: true
+
+# If true, create & use Pod Security Policy resources
+# https://kubernetes.io/docs/concepts/policy/pod-security-policy/
+podSecurityPolicy:
+  enabled: false
+
+## Set pod priorityClassName
+# priorityClassName: ""
+
+serviceAccount:
+  # Specifies whether a ServiceAccount should be created
+  create: true
+
+  # The name of the ServiceAccount to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name:
+
+resources: {}
+  # limits:
+  #  cpu: 100m
+  #  memory: 128Mi
+  # requests:
+  #  cpu: 100m
+  #  memory: 128Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+
+    `)))
+
+func GenerateNfsClientValuesFile(mgr *manager.Manager) (string, error) {
+	return util.Render(NfsClientTempl, util.Data{
+		"NfsClient": mgr.Cluster.Plugins.NfsClient,
+	})
+}

pkg/plugins/storage/storage.go

@@ -0,0 +1,77 @@
+package storage
+
+import (
+	"encoding/base64"
+	"fmt"
+	kubekeyapi "github.com/pixiake/kubekey/pkg/apis/kubekey/v1alpha1"
+	local_volume "github.com/pixiake/kubekey/pkg/plugins/storage/local-volume"
+	nfs_client "github.com/pixiake/kubekey/pkg/plugins/storage/nfs-client"
+	"github.com/pixiake/kubekey/pkg/util/manager"
+	"github.com/pixiake/kubekey/pkg/util/ssh"
+	"github.com/pkg/errors"
+)
+
+func DeployStoragePlugins(mgr *manager.Manager) error {
+	mgr.Logger.Infoln("Deploy storage plugin")
+
+	return mgr.RunTaskOnMasterNodes(deployStoragePlugins, true)
+}
+
+func deployStoragePlugins(mgr *manager.Manager, node *kubekeyapi.HostCfg, conn ssh.Connection) error {
+	if mgr.Runner.Index == 0 {
+		mgr.Runner.RunCmd("sudo -E /bin/sh -c \"mkdir -p /etc/kubernetes/addons\" && /usr/local/bin/helm repo add kubesphere https://charts.kubesphere.io/qingcloud")
+		if mgr.Cluster.Plugins.LocalVolume.Enabled {
+			if err := DeployLocalVolume(mgr); err != nil {
+				return err
+			}
+		}
+		if mgr.Cluster.Plugins.NfsClient.Enabled {
+			if err := DeployNfsClient(mgr); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func DeployLocalVolume(mgr *manager.Manager) error {
+	localVolumeFile, err := local_volume.GenerateOpenebsManifests(mgr)
+	if err != nil {
+		return err
+	}
+	localVolumeFileBase64 := base64.StdEncoding.EncodeToString([]byte(localVolumeFile))
+	_, err1 := mgr.Runner.RunCmd(fmt.Sprintf("sudo -E /bin/sh -c \"echo %s | base64 -d > /etc/kubernetes/addons/local-volume.yaml\"", localVolumeFileBase64))
+	if err1 != nil {
+		return errors.Wrap(errors.WithStack(err1), "failed to generate local-volume file")
+	}
+
+	_, err2 := mgr.Runner.RunCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/addons/local-volume.yaml")
+	if err2 != nil {
+		return errors.Wrap(errors.WithStack(err2), "failed to deploy local-volume.yaml")
+	}
+	return nil
+}
+
+func DeployNfsClient(mgr *manager.Manager) error {
+
+	_, err1 := mgr.Runner.RunCmd("sudo -E /bin/sh -c \"rm -rf /etc/kubernetes/addons/nfs-client-provisioner && /usr/local/bin/helm fetch kubesphere/nfs-client-provisioner -d /etc/kubernetes/addons --untar\"")
+	if err1 != nil {
+		return errors.Wrap(errors.WithStack(err1), "failed to fetch nfs-client-provisioner chart")
+	}
+
+	nfsClientValuesFile, err := nfs_client.GenerateNfsClientValuesFile(mgr)
+	if err != nil {
+		return err
+	}
+	nfsClientValuesFileBase64 := base64.StdEncoding.EncodeToString([]byte(nfsClientValuesFile))
+	_, err2 := mgr.Runner.RunCmd(fmt.Sprintf("sudo -E /bin/sh -c \"echo %s | base64 -d > /etc/kubernetes/addons/custom-values-nfs-client.yaml\"", nfsClientValuesFileBase64))
+	if err2 != nil {
+		return errors.Wrap(errors.WithStack(err2), "failed to generate nfs-client values file")
+	}
+
+	_, err3 := mgr.Runner.RunCmd("sudo -E /bin/sh -c \"/usr/local/bin/helm upgrade -i nfs-client /etc/kubernetes/addons/nfs-client-provisioner -f /etc/kubernetes/addons/custom-values-nfs-client.yaml -n kube-system\"")
+	if err3 != nil {
+		return errors.Wrap(errors.WithStack(err3), "failed to deploy nfs-client-provisioner")
+	}
+	return nil
+}

pkg/reset/reset.go

@@ -1,7 +1,6 @@
 package reset
 
 import (
-	"encoding/json"
 	"fmt"
 	kubekeyapi "github.com/pixiake/kubekey/pkg/apis/kubekey/v1alpha1"
 	"github.com/pixiake/kubekey/pkg/cluster/preinstall"
@@ -18,8 +17,8 @@ func ResetCluster(clusterCfgFile string, logger *log.Logger) error {
 		return errors.Wrap(err, "failed to download cluster config")
 	}
 
-	out, _ := json.MarshalIndent(cfg, "", "  ")
-	fmt.Println(string(out))
+	//out, _ := json.MarshalIndent(cfg, "", "  ")
+	//fmt.Println(string(out))
 	if err := preinstall.Prepare(&cfg.Spec, logger); err != nil {
 		return errors.Wrap(err, "failed to load kube binarys")
 	}