feat: more expiration for ca file (#2650)

Signed-off-by: joyceliu <joyceliu@yunify.com>

builtin/core/playbooks/add_nodes.yaml

@@ -98,6 +98,6 @@
       when: 
         - or (.add_nodes | default list | empty) (.add_nodes | default list | has .inventory_hostname)
         - .groups.kube_control_plane | default list | has .inventory_hostname
-        - .kubernetes.renew_certs.enabled    
+        - .kubernetes.certs.renew  
 
 - import_playbook: hook/post_install.yaml

builtin/core/playbooks/create_cluster.yaml

@@ -68,7 +68,7 @@
     - kubernetes/join-kubernetes
     - role: kubernetes/certs
       when: 
-        - .kubernetes.renew_certs.enabled
+        - .kubernetes.certs.renew
         - .groups.kube_control_plane | default list | has .inventory_hostname
   post_tasks:
     - name: Add custom label to cluster

builtin/core/playbooks/vars/create_cluster_kubernetes.yaml

@@ -202,8 +202,26 @@ kubernetes:
       tag: 3.5.0
   # custom_labels: {}
   # if auto renew kubernetes certs
-  renew_certs:
-    enabled: true
+  certs:
+    # Certificate Authority (CA) files in Kubernetes come from three sources:
+    # 1. kubeadm: ca_cert and ca_key set empty. Automatically generated by kubeadm. 
+    #    These certificates have a 10-year expiration period and remain unchanged.
+    # 2. kubekey: ca_cert set to {{ .binary_dir }}/pki/ca.cert and ca_key set to {{ .binary_dir }}/pki/ca.key. 
+    #    Automatically generated by kubekey. These certificates also have a 10-year validity period 
+    #    and can be modified using `cert.ca_date`.
+    # 3. custom: ca_cert and ca_key set to existing files. Custom CA files provided manually.
+    # 
+    # If you want to use custom CA files, specify the absolute paths to your ca_cert and ca_key files below.
+    # If left empty, the default behavior (kubeadm or kubekey) will be used.
+    ca_cert: ""
+    ca_key: ""
+    # The following fields are for the Kubernetes front-proxy CA certificate and key.
+    # If you want to use custom front-proxy CA files, specify the absolute paths below.
+    # If left empty, the default behavior will be used.
+    front_proxy_cert: ""
+    front_proxy_key: ""
+    renew: true
+  
 
 localDNS:
   - /etc/hosts

builtin/core/roles/init/init-cert/defaults/main.yaml

@@ -1,3 +1,24 @@
-artifact:
-  # how to generate cert file.support: IfNotPresent, Always
-  gen_cert_policy: IfNotPresent
+certs:
+  # CA cert
+  ca:
+    date: 87600h
+    # how to generate cert file.support: IfNotPresent, Always
+    gen_cert_policy: IfNotPresent
+  kubernetes_ca:
+      date: 87600h
+      # how to generate cert file.support: IfNotPresent, Always
+      gen_cert_policy: IfNotPresent
+  front_proxy_ca:
+    date: 87600h
+    # how to generate cert file.support: IfNotPresent, Always
+    gen_cert_policy: IfNotPresent
+  # etcd cert
+  etcd:
+    date: 87600h
+    # how to generate cert file.support: IfNotPresent, Always
+    gen_cert_policy: IfNotPresent
+  # image_registry cert date
+  image_registry:
+    date: 87600h
+    # how to generate cert file.support: IfNotPresent, Always
+    gen_cert_policy: IfNotPresent

builtin/core/roles/init/init-cert/tasks/main.yaml

@@ -2,13 +2,44 @@
 - name: Generate root ca file
   gen_cert:
     cn: root
-    date: 87600h
-    policy: "{{ .artifact.gen_cert_policy }}"
+    date: "{{ .certs.ca.date }}"
+    policy: "{{ .certs.ca.gen_cert_policy }}"
     out_key: >-
       {{ .binary_dir }}/pki/root.key
     out_cert: >-
       {{ .binary_dir }}/pki/root.crt
 
+- name: Generate kubernetes ca file
+  block:
+    - name: Generate ca file for kubernetes
+      gen_cert:
+        root_key: >-
+          {{ .binary_dir }}/pki/root.key
+        root_cert: >-
+          {{ .binary_dir }}/pki/root.crt
+        cn: kubernetes-ca
+        is_ca: true
+        date: "{{ .certs.kubernetes_ca.date }}"
+        policy: "{{ .certs.kubernetes_ca.gen_cert_policy }}"
+        out_key: >-
+          {{ .binary_dir }}/pki/kubernetes.key
+        out_cert: >-
+          {{ .binary_dir }}/pki/kubernetes.crt
+    - name: Generate front-proxy ca file for kubernetes
+      gen_cert:
+        root_key: >-
+          {{ .binary_dir }}/pki/root.key
+        root_cert: >-
+          {{ .binary_dir }}/pki/root.crt
+        cn: front-proxy-ca
+        is_ca: true
+        date: "{{ .certs.front_proxy_ca.date }}"
+        policy: "{{ .certs.front_proxy_ca.gen_cert_policy }}"
+        out_key: >-
+          {{ .binary_dir }}/pki/front-proxy.key
+        out_cert: >-
+          {{ .binary_dir }}/pki/front-proxy.crt
+
 - name: Generate etcd cert file
   gen_cert:
     root_key: >-
@@ -29,8 +60,8 @@
         {{- end -}}
       {{- end -}}
       {{ $ips | toJson }}
-    date: 87600h
-    policy: "{{ .artifact.gen_cert_policy }}"
+    date: "{{ .certs.etcd.date }}"
+    policy: "{{ .certs.etcd.gen_cert_policy }}"
     out_key: >-
       {{ .binary_dir }}/pki/etcd.key
     out_cert: >-
@@ -60,8 +91,8 @@
         {{- end -}}
       {{- end -}}
       {{ $ips | toJson }}
-    date: 87600h
-    policy: "{{ .artifact.gen_cert_policy }}"
+    date: "{{ .certs.image_registry.date }}"
+    policy: "{{ .certs.image_registry.gen_cert_policy }}"
     out_key: >-
       {{ .binary_dir }}/pki/image_registry.key
     out_cert: >-

builtin/core/roles/kubernetes/pre-kubernetes/tasks/main.yaml

@@ -26,4 +26,38 @@
   copy:
     src: audit
     dest: /etc/kubernetes/audit/
-  when: .kubernetes.audit
+  when: .kubernetes.audit
+
+- name: Sync ca file to kube_control_plane
+  when: 
+    - .kubernetes.certs.ca_cert | empty | not
+    - .kubernetes.certs.ca_key | empty | not
+    - .groups.kube_control_plane | has .inventory_hostname
+  block:
+    - name: Sync ca cert to kube_control_plane
+      copy: 
+        src: >-
+          {{ .kubernetes.certs.ca_cert }}
+        dest: /etc/kubernetes/pki/ca.crt
+    - name: Sync ca key to kube_control_plane
+      copy:
+        src: >-
+          {{ .kubernetes.certs.ca_key }}
+        dest: /etc/kubernetes/pki/ca.key
+
+- name: Sync front-proxy ca file to kube_control_plane
+  when:
+    - .kubernetes.certs.front_proxy_cert | empty | not
+    - .kubernetes.certs.front_proxy_key | empty | not
+    - .groups.kube_control_plane | has .inventory_hostname
+  block:
+    - name: Sync front-proxy cert to kube_control_plane
+      copy: 
+        src: >-
+          {{ .kubernetes.certs.front_proxy_cert }}
+        dest: /etc/kubernetes/pki/front-proxy-ca.crt
+    - name: Sync front-proxy key to kube_control_plane
+      copy:
+        src: >-
+          {{ .kubernetes.certs.front_proxy_key }}
+        dest: /etc/kubernetes/pki/front-proxy-ca.key

cmd/kk/app/options/builtin/add.go

@@ -124,7 +124,7 @@ func (o *AddNodeOptions) complete() error {
 	if o.ControlPlane != "" {
 		for _, node := range strings.Split(o.ControlPlane, ",") {
 			if !slices.Contains(groups[_const.VariableGroupsAll], node) {
-				return fmt.Errorf("%q is not defined in inventory.", node)
+				return errors.Errorf("%q is not defined in inventory.", node)
 			}
 			if !slices.Contains(groups[defaultGroupControlPlane], node) {
 				group := o.Inventory.Spec.Groups[defaultGroupControlPlane]
@@ -138,7 +138,7 @@ func (o *AddNodeOptions) complete() error {
 	if o.Worker != "" {
 		for _, node := range strings.Split(o.ControlPlane, ",") {
 			if !slices.Contains(groups[_const.VariableGroupsAll], node) {
-				return fmt.Errorf("%q is not defined in inventory.", node)
+				return errors.Errorf("%q is not defined in inventory.", node)
 			}
 			if !slices.Contains(groups[defaultGroupWorker], node) {
 				group := o.Inventory.Spec.Groups[defaultGroupWorker]

pkg/modules/gen_cert.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/client-go/util/keyutil"
 	"k8s.io/klog/v2"
 	netutils "k8s.io/utils/net"
+	"k8s.io/utils/ptr"
 
 	"github.com/kubesphere/kubekey/v4/pkg/variable"
 )
@@ -107,6 +108,7 @@ type genCertArgs struct {
 	cn       string
 	outKey   string
 	outCert  string
+	isCA     *bool
 }
 
 // signedCertificate generate certificate signed by root certificate
@@ -151,7 +153,7 @@ NEW:
 	if err != nil {
 		return "", fmt.Sprintf("generate rsa key error: %v", err)
 	}
-	newCert, err := NewSignedCert(*cfg, gca.date, newKey, parentCert, parentKey, true)
+	newCert, err := NewSignedCert(*cfg, gca.date, newKey, parentCert, parentKey, ptr.Deref(gca.isCA, false))
 	if err != nil {
 		return "", fmt.Sprintf("failed to generate certificate: %v", err)
 	}
@@ -201,6 +203,7 @@ func newGenCertArgs(_ context.Context, raw runtime.RawExtension, vars map[string
 	gca.cn, _ = variable.StringVar(vars, args, "cn")
 	gca.outKey, _ = variable.StringVar(vars, args, "out_key")
 	gca.outCert, _ = variable.StringVar(vars, args, "out_cert")
+	gca.isCA, _ = variable.BoolVar(vars, args, "is_ca")
 	// check args
 	if gca.policy != policyAlways && gca.policy != policyIfNotPresent {
 		return nil, errors.New("\"policy\" should be one of [Always, IfNotPresent]")

pkg/variable/yaml.go

@@ -154,7 +154,7 @@ func setContextValue(ctx map[string]any, value any, path ...string) error {
 		case seqTag:
 			current, err = handleSlice(current, val, isLast, value, &parents, &keys, path, i)
 		default:
-			return fmt.Errorf("unsupported tag: %s", tag)
+			return errors.Errorf("unsupported tag: %s", tag)
 		}
 		if err != nil {
 			return err
@@ -171,7 +171,17 @@ func setContextValue(ctx map[string]any, value any, path ...string) error {
 func handleMap(current reflect.Value, key string, isLast bool, value any,
 	parents *[]reflect.Value, keys *[]any, path []string, i int) (reflect.Value, error) {
 	if current.Kind() != reflect.Map {
-		return reflect.Value{}, fmt.Errorf("expected map, got %s", current.Kind())
+		return reflect.Value{}, errors.Errorf("expected map, got %s of path %q", current.Kind(), strings.Join(func() []string {
+			out := make([]string, len(path))
+			for i, p := range path {
+				if len(p) > 5 {
+					out[i] = p[5:]
+				} else {
+					out[i] = ""
+				}
+			}
+			return out
+		}(), "."))
 	}
 	rKey := reflect.ValueOf(key)
 	existing := current.MapIndex(rKey)
@@ -191,7 +201,7 @@ func handleMap(current reflect.Value, key string, isLast bool, value any,
 	var next reflect.Value
 	if !existing.IsValid() || isNil(existing) {
 		if i+1 >= len(path) {
-			return reflect.Value{}, fmt.Errorf("path incomplete after index %d", i)
+			return reflect.Value{}, errors.Errorf("path incomplete after index %d", i)
 		}
 		if strings.HasPrefix(path[i+1], mapTag) {
 			next = reflect.ValueOf(make(map[string]any))
@@ -216,10 +226,10 @@ func handleSlice(current reflect.Value, val string, isLast bool, value any,
 	// Parse index from path value
 	index, err := strconv.Atoi(val)
 	if err != nil {
-		return reflect.Value{}, fmt.Errorf("invalid index %s: %w", val, err)
+		return reflect.Value{}, errors.Errorf("invalid index %s: %w", val, err)
 	}
 	if current.Kind() != reflect.Slice {
-		return reflect.Value{}, fmt.Errorf("expected slice, got %s", current.Kind())
+		return reflect.Value{}, errors.Errorf("expected slice, got %s", current.Kind())
 	}
 	// Grow slice if requested index is beyond current length
 	if index >= current.Len() {
@@ -249,7 +259,7 @@ func handleSlice(current reflect.Value, val string, isLast bool, value any,
 	item := current.Index(index)
 	if isNil(item) {
 		if i+1 >= len(path) {
-			return reflect.Value{}, fmt.Errorf("path incomplete after index %d", i)
+			return reflect.Value{}, errors.Errorf("path incomplete after index %d", i)
 		}
 
 		var newItem reflect.Value