diff --git a/builtin/core/roles/kubernetes/init-kubernetes/tasks/init_kubernetes.yaml b/builtin/core/roles/kubernetes/init-kubernetes/tasks/init_kubernetes.yaml index 4e445083..6682d7a3 100644 --- a/builtin/core/roles/kubernetes/init-kubernetes/tasks/init_kubernetes.yaml +++ b/builtin/core/roles/kubernetes/init-kubernetes/tasks/init_kubernetes.yaml @@ -2,7 +2,9 @@ - name: Init | Generate kubeadm initialization configuration template: src: >- - {{- if .kubernetes.kube_version | semverCompare ">=v1.24.0" -}} + {{- if .kubernetes.kube_version | semverCompare ">=v1.31.0" -}} + kubeadm/kubeadm-init.v1beta4 + {{- else if .kubernetes.kube_version | semverCompare ">=v1.24.0" -}} kubeadm/kubeadm-init.v1beta3 {{- else -}} kubeadm/kubeadm-init.v1beta2 diff --git a/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta2 b/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta2 index ebb12725..cd44dc76 100644 --- a/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta2 +++ b/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta2 @@ -205,7 +205,7 @@ tlsCipherSuites: - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 {{- end }} -{{- if .kubernetes.kubelet.feature_gates | empty | not }} +{{- if .kubernetes.kubelet.feature_gates | empty | not }} featureGates: {{ .kubernetes.kubelet.feature_gates | toYaml | indent 2 }} {{- end }} diff --git a/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta3 b/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta3 index b594d9d3..a873c823 100644 --- a/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta3 +++ b/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta3 @@ -204,7 +204,7 @@ tlsCipherSuites: - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 {{- end }} -{{- if .kubernetes.kubelet.feature_gates | empty | not }} +{{- if .kubernetes.kubelet.feature_gates | empty | not }} featureGates: {{ .kubernetes.kubelet.feature_gates | toYaml | indent 2 }} {{- end }} diff --git a/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta4 b/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta4 new file mode 100644 index 00000000..da15281b --- /dev/null +++ b/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta4 @@ -0,0 +1,247 @@ +{{/* see: https://kubernetes.io/docs/reference/config-api/kubeadm-config.v1beta4/ */}} +{{- $internalIPv4 := .internal_ipv4 | default "" -}} +{{- $internalIPv6 := .internal_ipv6 | default "" -}} +--- +apiVersion: kubeadm.k8s.io/v1beta4 +kind: ClusterConfiguration +etcd: +{{- if .etcd.deployment_type | eq "internal" }} + local: + imageRepository: {{ .etcd.image.registry }} + imageTag: {{ .etcd.image.tag }} + serverCertSANs: + {{- range .groups.etcd | default list }} + - {{ index $.hostvars . "internal_ipv4" }} + {{- end }} +{{- else }} + external: + endpoints: + {{- range .groups.etcd | default list }} + - https://{{ index $.hostvars . "internal_ipv4" }}:2379 + {{- end }} + caFile: /etc/kubernetes/pki/etcd/ca.crt + certFile: /etc/kubernetes/pki/etcd/client.crt + keyFile: /etc/kubernetes/pki/etcd/client.key +{{- end }} +dns: + imageRepository: {{ .dns.dns_image.registry }}/{{ .dns.dns_image.repository }} + imageTag: {{ .dns.dns_image.tag }} +imageRepository: {{ .kubernetes.image_repository }} +kubernetesVersion: {{ .kubernetes.kube_version }} +certificatesDir: /etc/kubernetes/pki +clusterName: {{ .kubernetes.cluster_name }} +controlPlaneEndpoint: {{ .kubernetes.control_plane_endpoint.host }} +networking: + dnsDomain: {{ .dns.dns_domain }} + podSubnet: {{ .cni.pod_cidr }} + serviceSubnet: {{ .cni.service_cidr }} +apiServer: + extraArgs: +{{- if $internalIPv4 | empty | not }} + - name: bind-address + value: 0.0.0.0 +{{- else if $internalIPv6 | empty | not }} + - name: bind-address + value: :: +{{- end }} +{{- if .security_enhancement }} + - name: authorization-mode + value: Node,RBAC + - name: enable-admission-plugins + value: AlwaysPullImages,ServiceAccount,NamespaceLifecycle,NodeRestriction,LimitRanger,ResourceQuota,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,PodNodeSelector,PodSecurity + - name: profiling + value: false + - name: request-timeout + value: 120s + - name: service-account-lookup + value: true + - name: tls-min-version + value: VersionTLS12 + - name: tls-cipher-suites + value: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 +{{- end }} +{{- if .audit }} + - name: audit-log-format + value: json + - name: audit-log-maxbackup + value: 2 + - name: audit-log-maxsize + value: 200 + - name: audit-policy-file + value: /etc/kubernetes/audit/policy.yaml + - name: audit-webhook-config-file + value: /etc/kubernetes/audit/webhook.yaml +{{- end }} +{{- if .kubernetes.apiserver.extra_args | empty | not }} +{{ mapToNamedStringArgs .kubernetes.apiserver.extra_args | toYaml | indent 4 }} +{{- end }} + certSANs: + - localhost + - 127.0.0.1 + - ::1 + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - kubernetes.default.svc.{{ .kubernetes.cluster_name }} + - kubernetes.default.svc.{{ .kubernetes.cluster_name }}.{{ .dns.dns_domain }} + - {{ index (.cni.service_cidr | ipInCIDR) 0 }} + - {{ .kubernetes.control_plane_endpoint.host }} + {{- range .groups.k8s_cluster | default list }} + - {{ index $.hostvars . "hostname" }} + - {{ index $.hostvars . "hostname" }}.{{ $.kubernetes.cluster_name }} + - {{ index $.hostvars . "hostname" }}.{{ $.kubernetes.cluster_name }}.{{ $.dns.dns_domain }} + {{- $internalIPv4 := index $.hostvars . "internal_ipv4" | default "" }} + {{- $internalIPv6 := index $.hostvars . "internal_ipv6" | default "" }} + {{- if $internalIPv4 | empty | not }} + - {{ $internalIPv4 }} + {{- end }} + {{- if $internalIPv6 | empty | not }} + - {{ $internalIPv6 }} + {{- end }} + {{- end }} + {{- range .kubernetes.apiserver.certSANs }} + - {{ . }} + {{- end }} +{{- if .audit }} + extraVolumes: + - name: k8s-audit + hostPath: /etc/kubernetes/audit + mountPath: /etc/kubernetes/audit + pathType: DirectoryOrCreate +{{- end }} +controllerManager: + extraArgs: +{{- if eq (.cni.pod_cidr | splitList "," | first | ipFamily) "IPv4" }} + - name: node-cidr-mask-size-ipv4 + value: "{{ .cni.ipv4_mask_size }}" +{{- end }} +{{- if eq (.cni.pod_cidr | splitList "," | last | ipFamily) "IPv6" }} + - name: node-cidr-mask-size-ipv6 + value: "{{ .cni.ipv6_mask_size }}" +{{- end }} +{{- if .security_enhancement }} + {{- if $internalIPv4 | empty | not }} + - name: bind-address + value: 127.0.0.1 + {{- else if $internalIPv6 | empty | not }} + - name: bind-address + value: ::1 + {{- end }} + - name: profiling + value: false + - name: terminated-pod-gc-threshold + value: 50 + - name: use-service-account-credentials + value: true +{{- else }} + {{- if $internalIPv4 | empty | not }} + - name: bind-address + value: 0.0.0.0 + {{- else if $internalIPv6 | empty | not }} + - name: bind-address + value: :: + {{- end }} +{{- end }} +{{- if .kubernetes.controller_manager.extra_args }} +{{ mapToNamedStringArgs .kubernetes.controller_manager.extra_args | toYaml | indent 4 }} +{{- end }} + extraVolumes: + - name: host-time + hostPath: /etc/localtime + mountPath: /etc/localtime + readOnly: true +scheduler: + extraArgs: +{{- if .security_enhancement }} + {{- if $internalIPv4 | empty | not }} + - name: bind-address + value: 127.0.0.1 + {{- else if $internalIPv6 | empty | not }} + - name: bind-address + value: ::1 + {{- end }} + - name: profiling + value: false +{{- else }} + {{- if $internalIPv4 | empty | not }} + - name: bind-address + value: 0.0.0.0 + {{- else if $internalIPv6 | empty | not }} + - name: bind-address + value: :: + {{- end }} +{{- end }} +{{- if .kubernetes.scheduler.extra_args }} +{{ mapToNamedStringArgs .kubernetes.scheduler.extra_args | toYaml | indent 4 }} +{{- end }} +--- +apiVersion: kubeadm.k8s.io/v1beta4 +kind: InitConfiguration +localAPIEndpoint: +{{- if $internalIPv4 | empty | not }} + advertiseAddress: {{ $internalIPv4 }} +{{- else if $internalIPv6 | empty | not }} + advertiseAddress: {{ $internalIPv6 }} +{{- end }} + bindPort: {{ .kubernetes.apiserver.port }} +nodeRegistration: + criSocket: {{ .cri.cri_socket }} + kubeletExtraArgs: + - name: cgroup-driver + value: {{ .cri.cgroup_driver }} + - name: pod-infra-container-image + value: "{{ .cri.sandbox_image.registry }}/{{ .cri.sandbox_image.repository }}:{{ .cri.sandbox_image.tag }}" + +--- +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +kind: KubeProxyConfiguration +clusterCIDR: {{ .cni.pod_cidr }} +mode: {{ .kubernetes.kube_proxy.mode }} +{{- if .kubernetes.kube_proxy.config | empty | not }} +{{ .kubernetes.kube_proxy.config | toYaml }} +{{- end }} +--- +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +clusterDomain: {{ .dns.dns_domain }} +clusterDNS: + - {{ .dns.dns_cache_ip }} +maxPods: {{ .cni.max_pods }} +podPidsLimit: {{ .kubernetes.kubelet.pod_pids_limit }} +rotateCertificates: true +kubeReserved: + cpu: 200m + memory: 250Mi +systemReserved: + cpu: 200m + memory: 250Mi +evictionHard: + memory.available: 5% + pid.available: 10% +evictionSoft: + memory.available: 10% +evictionSoftGracePeriod: + memory.available: 2m +evictionMaxPodGracePeriod: 120 +evictionPressureTransitionPeriod: 30s +{{- if .security_enhancement }} +readOnlyPort: 0 +protectKernelDefaults: true +eventRecordQPS: 1 +streamingConnectionIdleTimeout: 5m +makeIPTablesUtilChains: true +tlsCipherSuites: + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 +{{- end }} +{{- if .kubernetes.kubelet.feature_gates | empty | not }} +featureGates: +{{ .kubernetes.kubelet.feature_gates | toYaml | indent 2 }} +{{- end }} +cgroupDriver: {{ .cri.cgroup_driver }} +containerLogMaxSize: {{ .kubernetes.kubelet.container_log_max_size }} +containerLogMaxFiles: {{ .kubernetes.kubelet.container_log_max_files }} +{{- if .kubernetes.kubelet.extra_args | empty | not }} +{{ .kubernetes.kubelet.extra_args | toYaml }} +{{- end }} diff --git a/builtin/core/roles/kubernetes/join-kubernetes/tasks/main.yaml b/builtin/core/roles/kubernetes/join-kubernetes/tasks/main.yaml index f7e63ff0..02d0217a 100644 --- a/builtin/core/roles/kubernetes/join-kubernetes/tasks/main.yaml +++ b/builtin/core/roles/kubernetes/join-kubernetes/tasks/main.yaml @@ -2,7 +2,9 @@ - name: Join | Generate kubeadm join configuration file template: src: >- - {{- if .kubernetes.kube_version | semverCompare ">=v1.24.0" -}} + {{- if .kubernetes.kube_version | semverCompare ">=v1.31.0" -}} + kubeadm/kubeadm-join.v1beta4 + {{- else if .kubernetes.kube_version | semverCompare ">=v1.24.0" -}} kubeadm/kubeadm-join.v1beta3 {{- else -}} kubeadm/kubeadm-join.v1beta2 diff --git a/builtin/core/roles/kubernetes/join-kubernetes/templates/kubeadm/kubeadm-join.v1beta4 b/builtin/core/roles/kubernetes/join-kubernetes/templates/kubeadm/kubeadm-join.v1beta4 new file mode 100644 index 00000000..20bc0b54 --- /dev/null +++ b/builtin/core/roles/kubernetes/join-kubernetes/templates/kubeadm/kubeadm-join.v1beta4 @@ -0,0 +1,28 @@ +{{- $internalIPv4 := .internal_ipv4 | default "" -}} +{{- $internalIPv6 := .internal_ipv6 | default "" -}} +--- +apiVersion: kubeadm.k8s.io/v1beta4 +kind: JoinConfiguration +discovery: + bootstrapToken: + apiServerEndpoint: {{ .kubernetes.control_plane_endpoint.host }}:{{ .kubernetes.control_plane_endpoint.port }} + token: "{{ .kubeadm_token }}" + unsafeSkipCAVerification: true +{{- if .groups.kube_control_plane | default list | has .inventory_hostname }} +controlPlane: + localAPIEndpoint: + {{- if $internalIPv4 | empty | not }} + advertiseAddress: {{ $internalIPv4 }} + {{- else if $internalIPv6 | empty | not }} + advertiseAddress: {{ $internalIPv6 }} + {{- end }} + bindPort: {{ .kubernetes.apiserver.port }} + certificateKey: {{ .kubeadm_cert }} +{{- end }} +nodeRegistration: + criSocket: {{ .cri.cri_socket }} + kubeletExtraArgs: + - name: cgroup-driver + value: {{ .cri.cgroup_driver }} + - name: pod-infra-container-image + value: "{{ .cri.sandbox_image.registry }}/{{ .cri.sandbox_image.repository }}:{{ .cri.sandbox_image.tag }}" diff --git a/pkg/converter/tmpl/functions.go b/pkg/converter/tmpl/functions.go index ee61d6e6..c0149be2 100644 --- a/pkg/converter/tmpl/functions.go +++ b/pkg/converter/tmpl/functions.go @@ -1,9 +1,11 @@ package tmpl import ( + "fmt" "math" "net" "os" + "reflect" "strconv" "strings" "text/template" @@ -31,6 +33,7 @@ func funcMap() template.FuncMap { f["fileExist"] = fileExist f["unquote"] = unquote f["getStringSlice"] = getStringSlice + f["mapToNamedStringArgs"] = mapToNamedStringArgs return f } @@ -139,3 +142,22 @@ func getStringSlice(d map[string][]string, key string) []string { } return nil } + +// mapToNamedStringArgs make kubeadm extra args of v1/beta4 +// for string/string extra argument maps, convert to structured extra arguments +func mapToNamedStringArgs(input any) []map[string]string { + v := reflect.ValueOf(input) + if v.Kind() != reflect.Map { + return []map[string]string{} + } + + result := make([]map[string]string, 0, v.Len()) + iter := v.MapRange() + for iter.Next() { + key := iter.Key().String() + value := iter.Value() + valueStr := fmt.Sprintf("%v", value.Interface()) + result = append(result, map[string]string{"name": key, "value": valueStr}) + } + return result +} diff --git a/pkg/converter/tmpl/template_test.go b/pkg/converter/tmpl/template_test.go index ea6f5d10..1eb3b253 100644 --- a/pkg/converter/tmpl/template_test.go +++ b/pkg/converter/tmpl/template_test.go @@ -262,6 +262,16 @@ func TestParseValue(t *testing.T) { }, excepted: []byte("[bar1 bar2]"), }, + { + name: "make kubeadm extra args from map", + input: "{{ mapToNamedStringArgs .foo | toYaml }}", + variable: map[string]any{ + "foo": map[string]any{ + "foo": "bar1", + }, + }, + excepted: []byte("- name: foo\n value: bar1"), + }, { name: "multi level 2", input: "{{ index .foo \"foo\" }}",