From efc63740a96c026f0d3bc9fe2a6655aaaffac1fb Mon Sep 17 00:00:00 2001 From: pixiake Date: Sat, 10 Oct 2020 14:21:38 +0800 Subject: [PATCH] addons charts Signed-off-by: pixiake --- Dockerfile | 6 +- api/v1alpha1/cluster_types.go | 1 + .../bases/kubekey.kubesphere.io_clusters.yaml | 2 + config/manager/kustomization.yaml | 4 +- config/manager/manager.yaml | 4 +- config/rbac/role.yaml | 160 ++++- controllers/cluster_controller.go | 44 +- controllers/install.go | 28 +- manifests/deploy.yaml | 619 ++++++++++++++++++ pkg/add/add.go | 4 +- pkg/addons/install.go | 3 +- pkg/bootstrap/init.go | 4 +- pkg/cluster/kubernetes/master.go | 46 +- pkg/config/parse.go | 43 +- pkg/delete/delete.go | 16 +- pkg/install/install.go | 5 +- pkg/upgrade/upgrade.go | 4 +- pkg/util/executor/executor.go | 5 +- pkg/util/manager/manager.go | 1 + pkg/util/ssh/dialer.go | 13 +- 20 files changed, 940 insertions(+), 72 deletions(-) create mode 100644 manifests/deploy.yaml diff --git a/Dockerfile b/Dockerfile index 2e85b415..bca2af56 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,8 +19,12 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager FROM debian:stable RUN useradd -m kubekey && apt-get update && apt-get install bash curl -y; apt-get autoclean; rm -rf /var/lib/apt/lists/* - WORKDIR /home/kubekey +#ADD addons /home/kubekey/addons + +#ADD kubekey /home/kubekey/kubekey +#RUN chown kubekey:kubekey -R /home/kubekey/kubekey + COPY --from=builder /workspace/manager /home/kubekey USER kubekey:kubekey diff --git a/api/v1alpha1/cluster_types.go b/api/v1alpha1/cluster_types.go index 71ce7dca..3f1b5261 100644 --- a/api/v1alpha1/cluster_types.go +++ b/api/v1alpha1/cluster_types.go @@ -106,6 +106,7 @@ type HostCfg struct { Port string `yaml:"port,omitempty" json:"port,omitempty"` User string `yaml:"user,omitempty" json:"user,omitempty"` Password string `yaml:"password,omitempty" json:"password,omitempty"` + PrivateKey string `yaml:"privateKey,omitempty" json:"privateKey,omitempty"` PrivateKeyPath string `yaml:"privateKeyPath,omitempty" json:"privateKeyPath,omitempty"` Arch string `yaml:"arch,omitempty" json:"arch,omitempty"` ID int `json:"-"` diff --git a/config/crd/bases/kubekey.kubesphere.io_clusters.yaml b/config/crd/bases/kubekey.kubesphere.io_clusters.yaml index 20676aaf..1a0dc76e 100644 --- a/config/crd/bases/kubekey.kubesphere.io_clusters.yaml +++ b/config/crd/bases/kubekey.kubesphere.io_clusters.yaml @@ -102,6 +102,8 @@ spec: type: string port: type: string + privateKey: + type: string privateKeyPath: type: string user: diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index b828738b..2fb6afa6 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: controller - newName: pixiake/kubekey - newTag: test + newName: kubespheredev/kubekey + newTag: latest diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index d638b6ba..ae831441 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -32,8 +32,8 @@ spec: name: manager resources: limits: - cpu: 100m - memory: 30Mi + cpu: 300m + memory: 200Mi requests: cpu: 100m memory: 20Mi diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 2995a628..364b2d2c 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -7,9 +7,27 @@ metadata: name: manager-role rules: - apiGroups: - - "" + - admissionregistration.k8s.io resources: - - secrets + - '*' + verbs: + - '*' +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - '*' +- apiGroups: + - apiregistration.k8s.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - apps + resources: + - deployments verbs: - create - delete @@ -18,6 +36,90 @@ rules: - patch - update - watch +- apiGroups: + - auditing.kubesphere.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - autoscaling + resources: + - '*' + verbs: + - '*' +- apiGroups: + - batch + resources: + - '*' + verbs: + - '*' +- apiGroups: + - certificates.k8s.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - config.istio.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - "" + resources: + - '*' + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - core.kubefed.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - devops.kubesphere.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - events.kubesphere.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - extensions + resources: + - '*' + verbs: + - '*' +- apiGroups: + - iam.kubesphere.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - installer.kubesphere.io + resources: + - clusterconfigurations + verbs: + - '*' +- apiGroups: + - jaegertracing.io + resources: + - '*' + verbs: + - '*' - apiGroups: - kubekey.kubesphere.io resources: @@ -38,3 +140,57 @@ rules: - get - patch - update +- apiGroups: + - logging.kubesphere.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - monitoring.coreos.com + resources: + - '*' + verbs: + - '*' +- apiGroups: + - networking.istio.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - notification.kubesphere.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - policy + resources: + - '*' + verbs: + - '*' +- apiGroups: + - rbac.authorization.k8s.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - storage.k8s.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - storage.kubesphere.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - tenant.kubesphere.io + resources: + - '*' + verbs: + - '*' diff --git a/controllers/cluster_controller.go b/controllers/cluster_controller.go index 00a75f3a..6c17fd90 100644 --- a/controllers/cluster_controller.go +++ b/controllers/cluster_controller.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "github.com/kubesphere/kubekey/pkg/addons" "github.com/kubesphere/kubekey/pkg/cluster/etcd" "github.com/kubesphere/kubekey/pkg/cluster/kubernetes" "github.com/kubesphere/kubekey/pkg/cluster/preinstall" @@ -46,7 +47,32 @@ type ClusterReconciler struct { // +kubebuilder:rbac:groups=kubekey.kubesphere.io,resources=clusters,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=kubekey.kubesphere.io,resources=clusters/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=*,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=storage.k8s.io,resources=*,verbs=* +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=*,verbs=* +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=* +// +kubebuilder:rbac:groups=installer.kubesphere.io,resources=clusterconfigurations,verbs=* +// +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=*,verbs=* +// +kubebuilder:rbac:groups=apiregistration.k8s.io,resources=*,verbs=* +// +kubebuilder:rbac:groups=auditing.kubesphere.io,resources=*,verbs=* +// +kubebuilder:rbac:groups=autoscaling,resources=*,verbs=* +// +kubebuilder:rbac:groups=certificates.k8s.io,resources=*,verbs=* +// +kubebuilder:rbac:groups=config.istio.io,resources=*,verbs=* +// +kubebuilder:rbac:groups=core.kubefed.io,resources=*,verbs=* +// +kubebuilder:rbac:groups=devops.kubesphere.io,resources=*,verbs=* +// +kubebuilder:rbac:groups=events.kubesphere.io,resources=*,verbs=* +// +kubebuilder:rbac:groups=batch,resources=*,verbs=* +// +kubebuilder:rbac:groups=extensions,resources=*,verbs=* +// +kubebuilder:rbac:groups=iam.kubesphere.io,resources=*,verbs=* +// +kubebuilder:rbac:groups=jaegertracing.io,resources=*,verbs=* +// +kubebuilder:rbac:groups=logging.kubesphere.io,resources=*,verbs=* +// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=*,verbs=* +// +kubebuilder:rbac:groups=networking.istio.io,resources=*,verbs=* +// +kubebuilder:rbac:groups=notification.kubesphere.io,resources=*,verbs=* +// +kubebuilder:rbac:groups=policy,resources=*,verbs=* +// +kubebuilder:rbac:groups=storage.kubesphere.io,resources=*,verbs=* +// +kubebuilder:rbac:groups=tenant.kubesphere.io,resources=*,verbs=* func (r *ClusterReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { ctx := context.Background() @@ -64,7 +90,7 @@ func (r *ClusterReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { return ctrl.Result{}, err } - newExecutor := executor.NewExecutor(&cluster.Spec, logger, "", true, true, false, false) + newExecutor := executor.NewExecutor(&cluster.Spec, cluster.Name, logger, "", true, true, false, false) mgr, err := newExecutor.CreateManager() if err != nil { @@ -72,15 +98,19 @@ func (r *ClusterReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { } if len(cluster.Status.Conditions) == 0 || len(cluster.Status.Nodes) == 0 { - // install - if err := installTasks(r, ctx, cluster, mgr); err != nil { + // install cluster + if err := clusterTasksExecute(r, ctx, cluster, mgr); err != nil { + return ctrl.Result{}, err + } + // install addons + if err := AddonsTasksExecute(r, ctx, cluster, mgr); err != nil { return ctrl.Result{}, err } } if len(mgr.AllNodes) > cluster.Status.NodesCount { // add nodes - if err := installTasks(r, ctx, cluster, mgr); err != nil { + if err := clusterTasksExecute(r, ctx, cluster, mgr); err != nil { return ctrl.Result{}, err } } @@ -133,4 +163,8 @@ var ( joinNodesTasks = []manager.Task{ {Task: kubernetes.JoinNodesToCluster, ErrMsg: "Failed to join node"}, } + + installAddonsTasks = []manager.Task{ + {Task: addons.InstallAddons, ErrMsg: "Failed to deploy addons"}, + } ) diff --git a/controllers/install.go b/controllers/install.go index 419f4219..9df8407b 100644 --- a/controllers/install.go +++ b/controllers/install.go @@ -23,7 +23,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func installTasks(r *ClusterReconciler, ctx context.Context, cluster *kubekeyv1alpha1.Cluster, mgr *manager.Manager) error { +func clusterTasksExecute(r *ClusterReconciler, ctx context.Context, cluster *kubekeyv1alpha1.Cluster, mgr *manager.Manager) error { // init nodes cluster.Status.Conditions = []kubekeyv1alpha1.Condition{} if err := r.Status().Update(ctx, cluster); err != nil { @@ -164,3 +164,29 @@ func installTasks(r *ClusterReconciler, ctx context.Context, cluster *kubekeyv1a return nil } + +func AddonsTasksExecute(r *ClusterReconciler, ctx context.Context, cluster *kubekeyv1alpha1.Cluster, mgr *manager.Manager) error { + // install addons + installAddonsCondition := kubekeyv1alpha1.Condition{ + Step: "Install addons", + StartTime: metav1.Now(), + EndTime: metav1.Now(), + Status: false, + } + cluster.Status.Conditions = append(cluster.Status.Conditions, installAddonsCondition) + if err := r.Status().Update(ctx, cluster); err != nil { + return err + } + + if err := runTasks(mgr, installAddonsTasks); err != nil { + return err + } + + cluster.Status.Conditions[len(cluster.Status.Conditions)-1].EndTime = metav1.Now() + cluster.Status.Conditions[len(cluster.Status.Conditions)-1].Status = true + if err := r.Status().Update(ctx, cluster); err != nil { + return err + } + + return nil +} diff --git a/manifests/deploy.yaml b/manifests/deploy.yaml new file mode 100644 index 00000000..89918e4e --- /dev/null +++ b/manifests/deploy.yaml @@ -0,0 +1,619 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + name: kubekey-system +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.3.0 + creationTimestamp: null + name: clusters.kubekey.kubesphere.io +spec: + group: kubekey.kubesphere.io + names: + kind: Cluster + listKind: ClusterList + plural: clusters + singular: cluster + scope: Cluster + subresources: + status: {} + validation: + openAPIV3Schema: + description: Cluster is the Schema for the clusters API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ClusterSpec defines the desired state of Cluster + properties: + addons: + items: + properties: + delay: + type: integer + name: + type: string + namespace: + type: string + retries: + type: integer + sources: + properties: + chart: + properties: + name: + type: string + path: + type: string + repo: + type: string + values: + items: + type: string + type: array + valuesFile: + type: string + version: + type: string + type: object + yaml: + properties: + path: + items: + type: string + type: array + type: object + type: object + type: object + type: array + controlPlaneEndpoint: + properties: + address: + type: string + domain: + type: string + port: + type: string + type: object + hosts: + description: Foo is an example field of Cluster. Edit Cluster_types.go + to remove/update + items: + properties: + address: + type: string + arch: + type: string + internalAddress: + type: string + name: + type: string + password: + type: string + port: + type: string + privateKey: + type: string + privateKeyPath: + type: string + user: + type: string + type: object + type: array + kubernetes: + properties: + apiserverCertExtraSans: + items: + type: string + type: array + clusterName: + type: string + etcdBackupDir: + type: string + etcdBackupPeriod: + type: string + etcdBackupScript: + type: string + imageRepo: + type: string + keepBackupNumber: + type: string + masqueradeAll: + type: boolean + maxPods: + type: string + nodeCidrMaskSize: + type: string + proxyMode: + type: string + version: + type: string + type: object + kubesphere: + properties: + configurations: + type: string + enabled: + type: boolean + version: + type: string + type: object + network: + properties: + calico: + properties: + ipipMode: + type: string + vethMTU: + type: string + vxlanMode: + type: string + type: object + flannel: + properties: + backendMode: + type: string + type: object + kubePodsCIDR: + type: string + kubeServiceCIDR: + type: string + plugin: + type: string + type: object + registry: + properties: + insecureRegistries: + items: + type: string + type: array + privateRegistry: + type: string + registryMirrors: + items: + type: string + type: array + type: object + roleGroups: + properties: + etcd: + items: + type: string + type: array + master: + items: + type: string + type: array + worker: + items: + type: string + type: array + type: object + type: object + status: + description: ClusterStatus defines the observed state of Cluster + properties: + Conditions: + items: + properties: + endTime: + format: date-time + type: string + startTime: + format: date-time + type: string + status: + type: boolean + step: + type: string + type: object + type: array + etcdCount: + type: integer + masterCount: + type: integer + networkPlugin: + type: string + nodes: + items: + properties: + hostname: + type: string + internalIP: + type: string + roles: + additionalProperties: + type: boolean + type: object + type: object + type: array + nodesCount: + type: integer + version: + type: string + workerCount: + type: integer + type: object + type: object + version: v1alpha1 + versions: + - name: v1alpha1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kubekey-leader-election-role + namespace: kubekey-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - configmaps/status + verbs: + - get + - update + - patch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: kubekey-manager-role +rules: +- apiGroups: + - admissionregistration.k8s.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - '*' +- apiGroups: + - apiregistration.k8s.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - auditing.kubesphere.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - autoscaling + resources: + - '*' + verbs: + - '*' +- apiGroups: + - batch + resources: + - '*' + verbs: + - '*' +- apiGroups: + - certificates.k8s.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - config.istio.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - "" + resources: + - '*' + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - core.kubefed.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - devops.kubesphere.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - events.kubesphere.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - extensions + resources: + - '*' + verbs: + - '*' +- apiGroups: + - iam.kubesphere.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - installer.kubesphere.io + resources: + - clusterconfigurations + verbs: + - '*' +- apiGroups: + - jaegertracing.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - kubekey.kubesphere.io + resources: + - clusters + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubekey.kubesphere.io + resources: + - clusters/status + verbs: + - get + - patch + - update +- apiGroups: + - logging.kubesphere.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - monitoring.coreos.com + resources: + - '*' + verbs: + - '*' +- apiGroups: + - networking.istio.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - notification.kubesphere.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - policy + resources: + - '*' + verbs: + - '*' +- apiGroups: + - rbac.authorization.k8s.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - storage.k8s.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - storage.kubesphere.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - tenant.kubesphere.io + resources: + - '*' + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubekey-proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: kubekey-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kubekey-leader-election-rolebinding + namespace: kubekey-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubekey-leader-election-role +subjects: +- kind: ServiceAccount + name: default + namespace: kubekey-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubekey-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubekey-manager-role +subjects: +- kind: ServiceAccount + name: default + namespace: kubekey-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubekey-proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubekey-proxy-role +subjects: +- kind: ServiceAccount + name: default + namespace: kubekey-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + name: kubekey-controller-manager-metrics-service + namespace: kubekey-system +spec: + ports: + - name: https + port: 8443 + targetPort: https + selector: + control-plane: controller-manager +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + control-plane: controller-manager + name: kubekey-controller-manager + namespace: kubekey-system +spec: + replicas: 1 + selector: + matchLabels: + control-plane: controller-manager + template: + metadata: + labels: + control-plane: controller-manager + spec: + containers: + - args: + - --secure-listen-address=0.0.0.0:8443 + - --upstream=http://127.0.0.1:8080/ + - --logtostderr=true + - --v=10 + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0 + name: kube-rbac-proxy + ports: + - containerPort: 8443 + name: https + - args: + - --metrics-addr=127.0.0.1:8080 + - --enable-leader-election + command: + - /home/kubekey/manager + image: kubespheredev/kubekey:latest + name: manager + resources: + limits: + cpu: 300m + memory: 200Mi + requests: + cpu: 100m + memory: 20Mi + terminationGracePeriodSeconds: 10 diff --git a/pkg/add/add.go b/pkg/add/add.go index bb0ee96e..ee5aebd1 100644 --- a/pkg/add/add.go +++ b/pkg/add/add.go @@ -41,12 +41,12 @@ func AddNodes(clusterCfgFile, k8sVersion, ksVersion string, logger *log.Logger, return errors.Wrap(err, "Failed to create work dir") } - cfg, err := config.ParseClusterCfg(clusterCfgFile, k8sVersion, ksVersion, ksEnabled, logger) + cfg, objName, err := config.ParseClusterCfg(clusterCfgFile, k8sVersion, ksVersion, ksEnabled, logger) if err != nil { return errors.Wrap(err, "Failed to download cluster config") } - return Execute(executor.NewExecutor(&cfg.Spec, logger, "", verbose, skipCheck, skipPullImages, false)) + return Execute(executor.NewExecutor(&cfg.Spec, objName, logger, "", verbose, skipCheck, skipPullImages, false)) } func ExecTasks(mgr *manager.Manager) error { diff --git a/pkg/addons/install.go b/pkg/addons/install.go index 990564d2..08599355 100644 --- a/pkg/addons/install.go +++ b/pkg/addons/install.go @@ -17,6 +17,7 @@ limitations under the License. package addons import ( + "fmt" kubekeyapiv1alpha1 "github.com/kubesphere/kubekey/api/v1alpha1" "github.com/kubesphere/kubekey/pkg/addons/charts" "github.com/kubesphere/kubekey/pkg/addons/manifests" @@ -35,7 +36,7 @@ func InstallAddons(mgr *manager.Manager) error { if addonsNum != 0 { for index, addon := range mgr.Cluster.Addons { mgr.Logger.Infof("Installing addon [%v-%v]: %s", addonsNum, index+1, addon.Name) - if err := installAddon(mgr, &addon, filepath.Join(mgr.WorkDir, "config")); err != nil { + if err := installAddon(mgr, &addon, filepath.Join(mgr.WorkDir, fmt.Sprintf("config-%s", mgr.ObjName))); err != nil { return err } } diff --git a/pkg/bootstrap/init.go b/pkg/bootstrap/init.go index 21ceff13..ee6aec45 100644 --- a/pkg/bootstrap/init.go +++ b/pkg/bootstrap/init.go @@ -37,12 +37,12 @@ func Init(clusterCfgFile, sourcesDir string, addImagesRepo bool, logger *log.Log return errors.Wrap(err, "Failed to create work dir") } - cfg, err := config.ParseClusterCfg(clusterCfgFile, "", "", false, logger) + cfg, objName, err := config.ParseClusterCfg(clusterCfgFile, "", "", false, logger) if err != nil { return errors.Wrap(err, "Failed to download cluster config") } - return Execute(executor.NewExecutor(&cfg.Spec, logger, sourcesDir, true, true, true, addImagesRepo)) + return Execute(executor.NewExecutor(&cfg.Spec, objName, logger, sourcesDir, true, true, true, addImagesRepo)) } func Execute(executor *executor.Executor) error { diff --git a/pkg/cluster/kubernetes/master.go b/pkg/cluster/kubernetes/master.go index 8b4f79e9..c6c5f53e 100644 --- a/pkg/cluster/kubernetes/master.go +++ b/pkg/cluster/kubernetes/master.go @@ -50,25 +50,36 @@ func GetClusterStatus(mgr *manager.Manager) error { } func getClusterStatus(mgr *manager.Manager, _ *kubekeyapiv1alpha1.HostCfg) error { - if clusterStatus["clusterInfo"] == "" { - output, err := mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"[ -f /etc/kubernetes/admin.conf ] && echo 'Cluster already exists.' || echo 'Cluster will be created.'\"", 0, true) - if strings.Contains(output, "Cluster will be created") { - clusterIsExist = false - } else { - if err != nil { - return errors.Wrap(errors.WithStack(err), "Failed to find /etc/kubernetes/admin.conf") + if mgr.Runner.Index == 0 { + if clusterStatus["clusterInfo"] == "" { + output, err := mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"[ -f /etc/kubernetes/admin.conf ] && echo 'Cluster already exists.' || echo 'Cluster will be created.'\"", 0, true) + if strings.Contains(output, "Cluster will be created") { + clusterIsExist = false } else { - clusterIsExist = true - output, err := mgr.Runner.ExecuteCmd("sudo cat /etc/kubernetes/manifests/kube-apiserver.yaml | grep 'image:' | awk -F '[:]' '{print $(NF-0)}'", 0, true) if err != nil { - return errors.Wrap(errors.WithStack(err), "Failed to find current version") + return errors.Wrap(errors.WithStack(err), "Failed to find /etc/kubernetes/admin.conf") } else { - if !strings.Contains(output, "No such file or directory") { - clusterStatus["version"] = output + clusterIsExist = true + if output, err := mgr.Runner.ExecuteCmd("sudo cat /etc/kubernetes/manifests/kube-apiserver.yaml | grep 'image:' | awk -F '[:]' '{print $(NF-0)}'", 0, true); err != nil { + return errors.Wrap(errors.WithStack(err), "Failed to find current version") + } else { + if !strings.Contains(output, "No such file or directory") { + clusterStatus["version"] = output + } + } + kubeCfgBase64Cmd := "cat /etc/kubernetes/admin.conf | base64 --wrap=0" + kubeConfigStr, err1 := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", kubeCfgBase64Cmd), 1, false) + fmt.Println(kubeConfigStr) + if err1 != nil { + return errors.Wrap(errors.WithStack(err1), "Failed to get cluster kubeconfig") + } + clusterStatus["kubeConfig"] = kubeConfigStr + if err := loadKubeConfig(mgr); err != nil { + return err + } + if err := getJoinNodesCmd(mgr); err != nil { + return err } - } - if err := getJoinNodesCmd(mgr); err != nil { - return err } } } @@ -272,6 +283,7 @@ func joinNodesToCluster(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) } } } + clusterStatus["clusterInfo"] = "" return nil } @@ -326,12 +338,12 @@ func addWorker(mgr *manager.Manager) error { } func loadKubeConfig(mgr *manager.Manager) error { - kubeConfigPath := filepath.Join(mgr.WorkDir, "config") + kubeConfigPath := filepath.Join(mgr.WorkDir, fmt.Sprintf("config-%s", mgr.ObjName)) loadKubeConfigCmd := fmt.Sprintf("echo %s | base64 -d > %s", clusterStatus["kubeConfig"], kubeConfigPath) if output, err := exec.Command("/bin/sh", "-c", loadKubeConfigCmd).CombinedOutput(); err != nil { return errors.Wrap(err, string(output)) } - replaceApiServerAddCmd := fmt.Sprintf("sed -i '/server:/s/\\:.*/\\: %s/g' %s", fmt.Sprintf("https\\:\\/\\/%s\\:6443", mgr.MasterNodes[0].InternalAddress), kubeConfigPath) + replaceApiServerAddCmd := fmt.Sprintf("sed -i '/server:/s/\\:.*/\\: %s/g' %s", fmt.Sprintf("https\\:\\/\\/%s\\:6443", mgr.MasterNodes[0].Address), kubeConfigPath) if output, err := exec.Command("/bin/sh", "-c", replaceApiServerAddCmd).CombinedOutput(); err != nil { return errors.Wrap(err, string(output)) } diff --git a/pkg/config/parse.go b/pkg/config/parse.go index 951d6bb3..588eba35 100644 --- a/pkg/config/parse.go +++ b/pkg/config/parse.go @@ -34,37 +34,42 @@ import ( "strings" ) -func ParseClusterCfg(clusterCfgPath, k8sVersion, ksVersion string, ksEnabled bool, logger *log.Logger) (*kubekeyapiv1alpha1.Cluster, error) { - var clusterCfg *kubekeyapiv1alpha1.Cluster +func ParseClusterCfg(clusterCfgPath, k8sVersion, ksVersion string, ksEnabled bool, logger *log.Logger) (*kubekeyapiv1alpha1.Cluster, string, error) { + var ( + clusterCfg *kubekeyapiv1alpha1.Cluster + objName string + ) if len(clusterCfgPath) == 0 { user, _ := user.Current() if user.Username != "root" { - return nil, errors.New(fmt.Sprintf("Current user is %s. Please use root!", user.Username)) + return nil, "", errors.New(fmt.Sprintf("Current user is %s. Please use root!", user.Username)) } - clusterCfg = AllinoneCfg(user, k8sVersion, ksVersion, ksEnabled, logger) + clusterCfg, objName = AllinoneCfg(user, k8sVersion, ksVersion, ksEnabled, logger) } else { - cfg, err := ParseCfg(clusterCfgPath, k8sVersion, ksVersion, ksEnabled) + cfg, name, err := ParseCfg(clusterCfgPath, k8sVersion, ksVersion, ksEnabled) if err != nil { - return nil, err + return nil, "", err } clusterCfg = cfg + objName = name } - return clusterCfg, nil + return clusterCfg, objName, nil } -func ParseCfg(clusterCfgPath, k8sVersion, ksVersion string, ksEnabled bool) (*kubekeyapiv1alpha1.Cluster, error) { +func ParseCfg(clusterCfgPath, k8sVersion, ksVersion string, ksEnabled bool) (*kubekeyapiv1alpha1.Cluster, string, error) { + var objName string clusterCfg := kubekeyapiv1alpha1.Cluster{} fp, err := filepath.Abs(clusterCfgPath) if err != nil { - return nil, errors.Wrap(err, "Failed to look up current directory") + return nil, "", errors.Wrap(err, "Failed to look up current directory") } if len(k8sVersion) != 0 { _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("sed -i \"/version/s/\\:.*/\\: %s/g\" %s", k8sVersion, fp)).Run() } file, err := os.Open(fp) if err != nil { - return nil, errors.Wrap(err, "Unable to open the given cluster configuration file") + return nil, "", errors.Wrap(err, "Unable to open the given cluster configuration file") } defer file.Close() b1 := bufio.NewReader(file) @@ -75,16 +80,18 @@ func ParseCfg(clusterCfgPath, k8sVersion, ksVersion string, ksEnabled bool) (*ku break } if err != nil { - return nil, errors.Wrap(err, "Unable to read the given cluster configuration file") + return nil, "", errors.Wrap(err, "Unable to read the given cluster configuration file") } err = yaml.Unmarshal(content, &result) if err != nil { - return nil, errors.Wrap(err, "Unable to unmarshal the given cluster configuration file") + return nil, "", errors.Wrap(err, "Unable to unmarshal the given cluster configuration file") } if result["kind"] == "Cluster" { if err := yaml.Unmarshal(content, &clusterCfg); err != nil { - return nil, errors.Wrap(err, "Unable to convert file to yaml") + return nil, "", errors.Wrap(err, "Unable to convert file to yaml") } + metadata := result["metadata"].(map[interface{}]interface{}) + objName = metadata["name"].(string) } if result["kind"] == "ConfigMap" || result["kind"] == "ClusterConfiguration" { @@ -101,7 +108,7 @@ func ParseCfg(clusterCfgPath, k8sVersion, ksVersion string, ksEnabled bool) (*ku clusterCfg.Spec.KubeSphere.Configurations = "---\n" + string(content) clusterCfg.Spec.KubeSphere.Version = "v2.1.1" default: - return nil, errors.Wrap(err, fmt.Sprintf("Unsupported versions: %s", labels["version"])) + return nil, "", errors.Wrap(err, fmt.Sprintf("Unsupported versions: %s", labels["version"])) } } } @@ -120,14 +127,14 @@ func ParseCfg(clusterCfgPath, k8sVersion, ksVersion string, ksEnabled bool) (*ku clusterCfg.Spec.KubeSphere.Version = "v2.1.1" clusterCfg.Spec.KubeSphere.Configurations = kubesphere.V2_1_1 default: - return nil, errors.New(fmt.Sprintf("Unsupported version: %s", strings.TrimSpace(ksVersion))) + return nil, "", errors.New(fmt.Sprintf("Unsupported version: %s", strings.TrimSpace(ksVersion))) } } - return &clusterCfg, nil + return &clusterCfg, objName, nil } -func AllinoneCfg(user *user.User, k8sVersion, ksVersion string, ksEnabled bool, logger *log.Logger) *kubekeyapiv1alpha1.Cluster { +func AllinoneCfg(user *user.User, k8sVersion, ksVersion string, ksEnabled bool, logger *log.Logger) (*kubekeyapiv1alpha1.Cluster, string) { allinoneCfg := kubekeyapiv1alpha1.Cluster{} if output, err := exec.Command("/bin/sh", "-c", "if [ ! -f \"$HOME/.ssh/id_rsa\" ]; then ssh-keygen -t rsa -P \"\" -f $HOME/.ssh/id_rsa && ls $HOME/.ssh;fi;").CombinedOutput(); err != nil { log.Fatalf("Failed to generate public key: %v\n%s", err, string(output)) @@ -184,5 +191,5 @@ func AllinoneCfg(user *user.User, k8sVersion, ksVersion string, ksEnabled bool, } } - return &allinoneCfg + return &allinoneCfg, hostname } diff --git a/pkg/delete/delete.go b/pkg/delete/delete.go index e0bdfc42..1fbcb672 100644 --- a/pkg/delete/delete.go +++ b/pkg/delete/delete.go @@ -34,12 +34,12 @@ import ( ) func ResetCluster(clusterCfgFile string, logger *log.Logger, verbose bool) error { - cfg, err := config.ParseClusterCfg(clusterCfgFile, "", "", false, logger) + cfg, objName, err := config.ParseClusterCfg(clusterCfgFile, "", "", false, logger) if err != nil { return errors.Wrap(err, "Failed to download cluster config") } - return Execute(executor.NewExecutor(&cfg.Spec, logger, "", verbose, false, true, false)) + return Execute(executor.NewExecutor(&cfg.Spec, objName, logger, "", verbose, false, true, false)) } func ResetNode(clusterCfgFile string, logger *log.Logger, verbose bool, nodeName string) error { @@ -55,16 +55,16 @@ func ResetNode(clusterCfgFile string, logger *log.Logger, verbose bool, nodeName if string(nodeNameNum) == "2\n" { cmd := fmt.Sprintf("sed -i /%s/d %s", nodeName, fp) _ = exec.Command("/bin/sh", "-c", cmd).Run() - cfg, _ := config.ParseClusterCfg(clusterCfgFile, "", "", false, logger) - return Execute1(executor.NewExecutor(&cfg.Spec, logger, "", verbose, false, true, false)) + cfg, objName, _ := config.ParseClusterCfg(clusterCfgFile, "", "", false, logger) + return Execute1(executor.NewExecutor(&cfg.Spec, objName, logger, "", verbose, false, true, false)) } else if string(nodeNameNum) == "1\n" { cmd := fmt.Sprintf("sed -i /%s/d %s", nodeName, fp) _ = exec.Command("/bin/sh", "-c", cmd).Run() - cfg, err := config.ParseClusterCfg(clusterCfgFile, "", "", false, logger) + cfg, objName, err := config.ParseClusterCfg(clusterCfgFile, "", "", false, logger) if err != nil { return errors.Wrap(err, "Failed to download cluster config") } - mgr, err1 := executor.NewExecutor(&cfg.Spec, logger, "", verbose, false, true, false).CreateManager() + mgr, err1 := executor.NewExecutor(&cfg.Spec, objName, logger, "", verbose, false, true, false).CreateManager() if err1 != nil { return errors.Wrap(err1, "Failed to get cluster config") } @@ -99,8 +99,8 @@ func ResetNode(clusterCfgFile string, logger *log.Logger, verbose bool, nodeName cmd2 := fmt.Sprintf("sed -i '/worker/a\\ \\ \\ \\ \\- %s' %s", workPar1, fp) _ = exec.Command("/bin/sh", "-c", cmd2).Run() } - cfg1, _ := config.ParseClusterCfg(clusterCfgFile, "", "", false, logger) - return Execute1(executor.NewExecutor(&cfg1.Spec, logger, "", verbose, false, true, false)) + cfg1, objName, _ := config.ParseClusterCfg(clusterCfgFile, "", "", false, logger) + return Execute1(executor.NewExecutor(&cfg1.Spec, objName, logger, "", verbose, false, true, false)) } else { fmt.Println("Please check the node name in the config-sample.yaml or do not support to delete master") os.Exit(0) diff --git a/pkg/install/install.go b/pkg/install/install.go index 46f14b29..f799a145 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -46,7 +46,7 @@ func CreateCluster(clusterCfgFile, k8sVersion, ksVersion string, logger *log.Log return errors.Wrap(err, "Failed to create work dir") } - cfg, err := config.ParseClusterCfg(clusterCfgFile, k8sVersion, ksVersion, ksEnabled, logger) + cfg, objName, err := config.ParseClusterCfg(clusterCfgFile, k8sVersion, ksVersion, ksEnabled, logger) if err != nil { return errors.Wrap(err, "Failed to download cluster config") } @@ -55,7 +55,8 @@ func CreateCluster(clusterCfgFile, k8sVersion, ksVersion string, logger *log.Log return errors.New("Please do not use uppercase letters in hostname: " + host.Name) } } - return Execute(executor.NewExecutor(&cfg.Spec, logger, "", verbose, skipCheck, skipPullImages, false)) + fmt.Println(objName) + return Execute(executor.NewExecutor(&cfg.Spec, objName, logger, "", verbose, skipCheck, skipPullImages, false)) } func ExecTasks(mgr *manager.Manager) error { diff --git a/pkg/upgrade/upgrade.go b/pkg/upgrade/upgrade.go index a15d2718..e12b3df4 100644 --- a/pkg/upgrade/upgrade.go +++ b/pkg/upgrade/upgrade.go @@ -39,12 +39,12 @@ func UpgradeCluster(clusterCfgFile, k8sVersion, ksVersion string, logger *log.Lo return errors.Wrap(err, "Failed to create work dir") } - cfg, err := config.ParseClusterCfg(clusterCfgFile, k8sVersion, ksVersion, ksEnabled, logger) + cfg, objName, err := config.ParseClusterCfg(clusterCfgFile, k8sVersion, ksVersion, ksEnabled, logger) if err != nil { return errors.Wrap(err, "Failed to download cluster config") } - return Execute(executor.NewExecutor(&cfg.Spec, logger, "", verbose, true, skipPullImages, false)) + return Execute(executor.NewExecutor(&cfg.Spec, objName, logger, "", verbose, true, skipPullImages, false)) } func ExecTasks(mgr *manager.Manager) error { diff --git a/pkg/util/executor/executor.go b/pkg/util/executor/executor.go index 25523a88..40ccaebc 100644 --- a/pkg/util/executor/executor.go +++ b/pkg/util/executor/executor.go @@ -28,6 +28,7 @@ import ( ) type Executor struct { + ObjName string Cluster *kubekeyapiv1alpha1.ClusterSpec Logger *log.Logger SourcesDir string @@ -37,8 +38,9 @@ type Executor struct { AddImagesRepo bool } -func NewExecutor(cluster *kubekeyapiv1alpha1.ClusterSpec, logger *log.Logger, sourcesDir string, debug, skipCheck, skipPullImages, addImagesRepo bool) *Executor { +func NewExecutor(cluster *kubekeyapiv1alpha1.ClusterSpec, objName string, logger *log.Logger, sourcesDir string, debug, skipCheck, skipPullImages, addImagesRepo bool) *Executor { return &Executor{ + ObjName: objName, Cluster: cluster, Logger: logger, SourcesDir: sourcesDir, @@ -70,6 +72,7 @@ func (executor *Executor) CreateManager() (*manager.Manager, error) { mgr.SkipPullImages = executor.SkipPullImages mgr.SourcesDir = executor.SourcesDir mgr.AddImagesRepo = executor.AddImagesRepo + mgr.ObjName = executor.ObjName return mgr, nil } diff --git a/pkg/util/manager/manager.go b/pkg/util/manager/manager.go index 0d0b90f1..70506409 100644 --- a/pkg/util/manager/manager.go +++ b/pkg/util/manager/manager.go @@ -24,6 +24,7 @@ import ( ) type Manager struct { + ObjName string Cluster *kubekeyapiv1alpha1.ClusterSpec Logger log.FieldLogger Connector *ssh.Dialer diff --git a/pkg/util/ssh/dialer.go b/pkg/util/ssh/dialer.go index 9e8803ee..2b446205 100644 --- a/pkg/util/ssh/dialer.go +++ b/pkg/util/ssh/dialer.go @@ -43,12 +43,13 @@ func (dialer *Dialer) Connect(host kubekeyapiv1alpha1.HostCfg) (Connection, erro conn, _ := dialer.connections[host.ID] port, _ := strconv.Atoi(host.Port) opts := Cfg{ - Username: host.User, - Port: port, - Address: host.Address, - Password: host.Password, - KeyFile: host.PrivateKeyPath, - Timeout: 30 * time.Second, + Username: host.User, + Port: port, + Address: host.Address, + Password: host.Password, + PrivateKey: host.PrivateKey, + KeyFile: host.PrivateKeyPath, + Timeout: 30 * time.Second, } conn, err = NewConnection(opts) if err != nil {