diff --git a/builtin/core/playbooks/add_nodes.yaml b/builtin/core/playbooks/add_nodes.yaml index d59582ca..30d084da 100644 --- a/builtin/core/playbooks/add_nodes.yaml +++ b/builtin/core/playbooks/add_nodes.yaml @@ -98,6 +98,6 @@ when: - or (.add_nodes | default list | empty) (.add_nodes | default list | has .inventory_hostname) - .groups.kube_control_plane | default list | has .inventory_hostname - - .kubernetes.renew_certs.enabled + - .kubernetes.certs.renew - import_playbook: hook/post_install.yaml \ No newline at end of file diff --git a/builtin/core/playbooks/create_cluster.yaml b/builtin/core/playbooks/create_cluster.yaml index 447fb1c2..87553bfc 100644 --- a/builtin/core/playbooks/create_cluster.yaml +++ b/builtin/core/playbooks/create_cluster.yaml @@ -68,7 +68,7 @@ - kubernetes/join-kubernetes - role: kubernetes/certs when: - - .kubernetes.renew_certs.enabled + - .kubernetes.certs.renew - .groups.kube_control_plane | default list | has .inventory_hostname post_tasks: - name: Add custom label to cluster diff --git a/builtin/core/playbooks/vars/create_cluster_kubernetes.yaml b/builtin/core/playbooks/vars/create_cluster_kubernetes.yaml index e9dbff1d..4ef0a9a4 100644 --- a/builtin/core/playbooks/vars/create_cluster_kubernetes.yaml +++ b/builtin/core/playbooks/vars/create_cluster_kubernetes.yaml @@ -202,8 +202,26 @@ kubernetes: tag: 3.5.0 # custom_labels: {} # if auto renew kubernetes certs - renew_certs: - enabled: true + certs: + # Certificate Authority (CA) files in Kubernetes come from three sources: + # 1. kubeadm: ca_cert and ca_key set empty. Automatically generated by kubeadm. + # These certificates have a 10-year expiration period and remain unchanged. + # 2. kubekey: ca_cert set to {{ .binary_dir }}/pki/ca.cert and ca_key set to {{ .binary_dir }}/pki/ca.key. + # Automatically generated by kubekey. These certificates also have a 10-year validity period + # and can be modified using `cert.ca_date`. + # 3. custom: ca_cert and ca_key set to existing files. Custom CA files provided manually. + # + # If you want to use custom CA files, specify the absolute paths to your ca_cert and ca_key files below. + # If left empty, the default behavior (kubeadm or kubekey) will be used. + ca_cert: "" + ca_key: "" + # The following fields are for the Kubernetes front-proxy CA certificate and key. + # If you want to use custom front-proxy CA files, specify the absolute paths below. + # If left empty, the default behavior will be used. + front_proxy_cert: "" + front_proxy_key: "" + renew: true + localDNS: - /etc/hosts \ No newline at end of file diff --git a/builtin/core/roles/init/init-cert/defaults/main.yaml b/builtin/core/roles/init/init-cert/defaults/main.yaml index 53fb0a55..7c24bd91 100644 --- a/builtin/core/roles/init/init-cert/defaults/main.yaml +++ b/builtin/core/roles/init/init-cert/defaults/main.yaml @@ -1,3 +1,24 @@ -artifact: - # how to generate cert file.support: IfNotPresent, Always - gen_cert_policy: IfNotPresent \ No newline at end of file +certs: + # CA cert + ca: + date: 87600h + # how to generate cert file.support: IfNotPresent, Always + gen_cert_policy: IfNotPresent + kubernetes_ca: + date: 87600h + # how to generate cert file.support: IfNotPresent, Always + gen_cert_policy: IfNotPresent + front_proxy_ca: + date: 87600h + # how to generate cert file.support: IfNotPresent, Always + gen_cert_policy: IfNotPresent + # etcd cert + etcd: + date: 87600h + # how to generate cert file.support: IfNotPresent, Always + gen_cert_policy: IfNotPresent + # image_registry cert date + image_registry: + date: 87600h + # how to generate cert file.support: IfNotPresent, Always + gen_cert_policy: IfNotPresent \ No newline at end of file diff --git a/builtin/core/roles/init/init-cert/tasks/main.yaml b/builtin/core/roles/init/init-cert/tasks/main.yaml index 5456e1bf..1013e380 100644 --- a/builtin/core/roles/init/init-cert/tasks/main.yaml +++ b/builtin/core/roles/init/init-cert/tasks/main.yaml @@ -2,13 +2,44 @@ - name: Generate root ca file gen_cert: cn: root - date: 87600h - policy: "{{ .artifact.gen_cert_policy }}" + date: "{{ .certs.ca.date }}" + policy: "{{ .certs.ca.gen_cert_policy }}" out_key: >- {{ .binary_dir }}/pki/root.key out_cert: >- {{ .binary_dir }}/pki/root.crt +- name: Generate kubernetes ca file + block: + - name: Generate ca file for kubernetes + gen_cert: + root_key: >- + {{ .binary_dir }}/pki/root.key + root_cert: >- + {{ .binary_dir }}/pki/root.crt + cn: kubernetes-ca + is_ca: true + date: "{{ .certs.kubernetes_ca.date }}" + policy: "{{ .certs.kubernetes_ca.gen_cert_policy }}" + out_key: >- + {{ .binary_dir }}/pki/kubernetes.key + out_cert: >- + {{ .binary_dir }}/pki/kubernetes.crt + - name: Generate front-proxy ca file for kubernetes + gen_cert: + root_key: >- + {{ .binary_dir }}/pki/root.key + root_cert: >- + {{ .binary_dir }}/pki/root.crt + cn: front-proxy-ca + is_ca: true + date: "{{ .certs.front_proxy_ca.date }}" + policy: "{{ .certs.front_proxy_ca.gen_cert_policy }}" + out_key: >- + {{ .binary_dir }}/pki/front-proxy.key + out_cert: >- + {{ .binary_dir }}/pki/front-proxy.crt + - name: Generate etcd cert file gen_cert: root_key: >- @@ -29,8 +60,8 @@ {{- end -}} {{- end -}} {{ $ips | toJson }} - date: 87600h - policy: "{{ .artifact.gen_cert_policy }}" + date: "{{ .certs.etcd.date }}" + policy: "{{ .certs.etcd.gen_cert_policy }}" out_key: >- {{ .binary_dir }}/pki/etcd.key out_cert: >- @@ -60,8 +91,8 @@ {{- end -}} {{- end -}} {{ $ips | toJson }} - date: 87600h - policy: "{{ .artifact.gen_cert_policy }}" + date: "{{ .certs.image_registry.date }}" + policy: "{{ .certs.image_registry.gen_cert_policy }}" out_key: >- {{ .binary_dir }}/pki/image_registry.key out_cert: >- diff --git a/builtin/core/roles/kubernetes/pre-kubernetes/tasks/main.yaml b/builtin/core/roles/kubernetes/pre-kubernetes/tasks/main.yaml index 7b267264..03f81d8a 100644 --- a/builtin/core/roles/kubernetes/pre-kubernetes/tasks/main.yaml +++ b/builtin/core/roles/kubernetes/pre-kubernetes/tasks/main.yaml @@ -26,4 +26,38 @@ copy: src: audit dest: /etc/kubernetes/audit/ - when: .kubernetes.audit \ No newline at end of file + when: .kubernetes.audit + +- name: Sync ca file to kube_control_plane + when: + - .kubernetes.certs.ca_cert | empty | not + - .kubernetes.certs.ca_key | empty | not + - .groups.kube_control_plane | has .inventory_hostname + block: + - name: Sync ca cert to kube_control_plane + copy: + src: >- + {{ .kubernetes.certs.ca_cert }} + dest: /etc/kubernetes/pki/ca.crt + - name: Sync ca key to kube_control_plane + copy: + src: >- + {{ .kubernetes.certs.ca_key }} + dest: /etc/kubernetes/pki/ca.key + +- name: Sync front-proxy ca file to kube_control_plane + when: + - .kubernetes.certs.front_proxy_cert | empty | not + - .kubernetes.certs.front_proxy_key | empty | not + - .groups.kube_control_plane | has .inventory_hostname + block: + - name: Sync front-proxy cert to kube_control_plane + copy: + src: >- + {{ .kubernetes.certs.front_proxy_cert }} + dest: /etc/kubernetes/pki/front-proxy-ca.crt + - name: Sync front-proxy key to kube_control_plane + copy: + src: >- + {{ .kubernetes.certs.front_proxy_key }} + dest: /etc/kubernetes/pki/front-proxy-ca.key \ No newline at end of file diff --git a/cmd/kk/app/options/builtin/add.go b/cmd/kk/app/options/builtin/add.go index 48e5f8a6..995dbec2 100644 --- a/cmd/kk/app/options/builtin/add.go +++ b/cmd/kk/app/options/builtin/add.go @@ -124,7 +124,7 @@ func (o *AddNodeOptions) complete() error { if o.ControlPlane != "" { for _, node := range strings.Split(o.ControlPlane, ",") { if !slices.Contains(groups[_const.VariableGroupsAll], node) { - return fmt.Errorf("%q is not defined in inventory.", node) + return errors.Errorf("%q is not defined in inventory.", node) } if !slices.Contains(groups[defaultGroupControlPlane], node) { group := o.Inventory.Spec.Groups[defaultGroupControlPlane] @@ -138,7 +138,7 @@ func (o *AddNodeOptions) complete() error { if o.Worker != "" { for _, node := range strings.Split(o.ControlPlane, ",") { if !slices.Contains(groups[_const.VariableGroupsAll], node) { - return fmt.Errorf("%q is not defined in inventory.", node) + return errors.Errorf("%q is not defined in inventory.", node) } if !slices.Contains(groups[defaultGroupWorker], node) { group := o.Inventory.Spec.Groups[defaultGroupWorker] diff --git a/pkg/modules/gen_cert.go b/pkg/modules/gen_cert.go index ba18cf24..033ff72d 100644 --- a/pkg/modules/gen_cert.go +++ b/pkg/modules/gen_cert.go @@ -25,6 +25,7 @@ import ( "k8s.io/client-go/util/keyutil" "k8s.io/klog/v2" netutils "k8s.io/utils/net" + "k8s.io/utils/ptr" "github.com/kubesphere/kubekey/v4/pkg/variable" ) @@ -107,6 +108,7 @@ type genCertArgs struct { cn string outKey string outCert string + isCA *bool } // signedCertificate generate certificate signed by root certificate @@ -151,7 +153,7 @@ NEW: if err != nil { return "", fmt.Sprintf("generate rsa key error: %v", err) } - newCert, err := NewSignedCert(*cfg, gca.date, newKey, parentCert, parentKey, true) + newCert, err := NewSignedCert(*cfg, gca.date, newKey, parentCert, parentKey, ptr.Deref(gca.isCA, false)) if err != nil { return "", fmt.Sprintf("failed to generate certificate: %v", err) } @@ -201,6 +203,7 @@ func newGenCertArgs(_ context.Context, raw runtime.RawExtension, vars map[string gca.cn, _ = variable.StringVar(vars, args, "cn") gca.outKey, _ = variable.StringVar(vars, args, "out_key") gca.outCert, _ = variable.StringVar(vars, args, "out_cert") + gca.isCA, _ = variable.BoolVar(vars, args, "is_ca") // check args if gca.policy != policyAlways && gca.policy != policyIfNotPresent { return nil, errors.New("\"policy\" should be one of [Always, IfNotPresent]") diff --git a/pkg/variable/yaml.go b/pkg/variable/yaml.go index d4c4c870..c3cb5b1d 100644 --- a/pkg/variable/yaml.go +++ b/pkg/variable/yaml.go @@ -154,7 +154,7 @@ func setContextValue(ctx map[string]any, value any, path ...string) error { case seqTag: current, err = handleSlice(current, val, isLast, value, &parents, &keys, path, i) default: - return fmt.Errorf("unsupported tag: %s", tag) + return errors.Errorf("unsupported tag: %s", tag) } if err != nil { return err @@ -171,7 +171,17 @@ func setContextValue(ctx map[string]any, value any, path ...string) error { func handleMap(current reflect.Value, key string, isLast bool, value any, parents *[]reflect.Value, keys *[]any, path []string, i int) (reflect.Value, error) { if current.Kind() != reflect.Map { - return reflect.Value{}, fmt.Errorf("expected map, got %s", current.Kind()) + return reflect.Value{}, errors.Errorf("expected map, got %s of path %q", current.Kind(), strings.Join(func() []string { + out := make([]string, len(path)) + for i, p := range path { + if len(p) > 5 { + out[i] = p[5:] + } else { + out[i] = "" + } + } + return out + }(), ".")) } rKey := reflect.ValueOf(key) existing := current.MapIndex(rKey) @@ -191,7 +201,7 @@ func handleMap(current reflect.Value, key string, isLast bool, value any, var next reflect.Value if !existing.IsValid() || isNil(existing) { if i+1 >= len(path) { - return reflect.Value{}, fmt.Errorf("path incomplete after index %d", i) + return reflect.Value{}, errors.Errorf("path incomplete after index %d", i) } if strings.HasPrefix(path[i+1], mapTag) { next = reflect.ValueOf(make(map[string]any)) @@ -216,10 +226,10 @@ func handleSlice(current reflect.Value, val string, isLast bool, value any, // Parse index from path value index, err := strconv.Atoi(val) if err != nil { - return reflect.Value{}, fmt.Errorf("invalid index %s: %w", val, err) + return reflect.Value{}, errors.Errorf("invalid index %s: %w", val, err) } if current.Kind() != reflect.Slice { - return reflect.Value{}, fmt.Errorf("expected slice, got %s", current.Kind()) + return reflect.Value{}, errors.Errorf("expected slice, got %s", current.Kind()) } // Grow slice if requested index is beyond current length if index >= current.Len() { @@ -249,7 +259,7 @@ func handleSlice(current reflect.Value, val string, isLast bool, value any, item := current.Index(index) if isNil(item) { if i+1 >= len(path) { - return reflect.Value{}, fmt.Errorf("path incomplete after index %d", i) + return reflect.Value{}, errors.Errorf("path incomplete after index %d", i) } var newItem reflect.Value