From 38b8d046174e9480f24d0b095935aa1d7ed4c416 Mon Sep 17 00:00:00 2001 From: liujian Date: Wed, 28 May 2025 18:06:14 +0800 Subject: [PATCH] fix: Kube_vip may fail when used with Kubernetes versions above v1.29. (#2597) Signed-off-by: joyceliu --- .../infrastructure/v1beta1/kkcluster_types.go | 2 +- .../install/cloud-config/tasks/main.yaml | 2 +- builtin/core/playbooks/delete_cluster.yaml | 2 +- builtin/core/playbooks/delete_nodes.yaml | 2 +- .../tasks/init_kubernetes.yaml | 47 ++++++++++------ .../init-kubernetes/tasks/main.yaml | 14 +++-- .../tasks/join_kubernetes.yaml | 35 ++++-------- .../join-kubernetes/tasks/main.yaml | 49 ++-------------- .../tasks/high-availability/kube_vip.yaml | 10 ++-- .../tasks/high-availability/main.yaml | 56 +++++++++++++++++++ config/capkk/release/cluster-template.yaml | 2 +- 11 files changed, 118 insertions(+), 103 deletions(-) diff --git a/api/capkk/infrastructure/v1beta1/kkcluster_types.go b/api/capkk/infrastructure/v1beta1/kkcluster_types.go index f4a8cdf9..964634e0 100644 --- a/api/capkk/infrastructure/v1beta1/kkcluster_types.go +++ b/api/capkk/infrastructure/v1beta1/kkcluster_types.go @@ -80,7 +80,7 @@ const ( // - ARP Mode: Requires the management cluster and worker cluster nodes to be in the same network segment. // - BGP Mode: Requires a network environment that supports BGP, with proper configuration in both // the management and worker clusters. - ControlPlaneEndpointTypeVIP ControlPlaneEndpointType = "vip" + ControlPlaneEndpointTypeVIP ControlPlaneEndpointType = "kube_vip" ) type InventoryHostConnector struct { diff --git a/builtin/capkk/roles/install/cloud-config/tasks/main.yaml b/builtin/capkk/roles/install/cloud-config/tasks/main.yaml index 8de4c398..2461069d 100644 --- a/builtin/capkk/roles/install/cloud-config/tasks/main.yaml +++ b/builtin/capkk/roles/install/cloud-config/tasks/main.yaml @@ -1,7 +1,7 @@ --- - name: Install kube-vip when: - - eq .kubernetes.control_plane_endpoint.type "vip" + - eq .kubernetes.control_plane_endpoint.type "kube_vip" - or (.kubernetes.roles | has "master") (.kubernetes.roles | has "control-plane") template: src: kube-vip.yaml diff --git a/builtin/core/playbooks/delete_cluster.yaml b/builtin/core/playbooks/delete_cluster.yaml index 28155216..a8f4a05b 100644 --- a/builtin/core/playbooks/delete_cluster.yaml +++ b/builtin/core/playbooks/delete_cluster.yaml @@ -18,7 +18,7 @@ ignore_errors: true command: | sed -i ':a;$!{N;ba};s@# kubekey hosts BEGIN.*# kubekey hosts END@@' {{ .item }} - sed -i ':a;$!{N;ba};s@# kubekey custom BEGIN.*# kubekey custom END@@' {{ .item }} + sed -i ':a;$!{N;ba};s@# kubekey control_plane BEGIN.*# kubekey control_plane END@@' {{ .item }} when: .deleteDNS loop: "{{ .localDNS | toJson }}" diff --git a/builtin/core/playbooks/delete_nodes.yaml b/builtin/core/playbooks/delete_nodes.yaml index f373e667..aed2c955 100644 --- a/builtin/core/playbooks/delete_nodes.yaml +++ b/builtin/core/playbooks/delete_nodes.yaml @@ -61,7 +61,7 @@ ignore_errors: true command: | sed -i ':a;$!{N;ba};s@# kubekey hosts BEGIN.*# kubekey hosts END@@' {{ .item }} - sed -i ':a;$!{N;ba};s@# kubekey custom BEGIN.*# kubekey custom END@@' {{ .item }} + sed -i ':a;$!{N;ba};s@# kubekey control_plane BEGIN.*# kubekey control_plane END@@' {{ .item }} when: - .deleteDNS - .delete_nodes | default list | has .inventory_hostname diff --git a/builtin/core/roles/kubernetes/init-kubernetes/tasks/init_kubernetes.yaml b/builtin/core/roles/kubernetes/init-kubernetes/tasks/init_kubernetes.yaml index 1e8a7c18..346b6850 100644 --- a/builtin/core/roles/kubernetes/init-kubernetes/tasks/init_kubernetes.yaml +++ b/builtin/core/roles/kubernetes/init-kubernetes/tasks/init_kubernetes.yaml @@ -33,26 +33,37 @@ {{- end }} dest: /etc/kubernetes/kubeadm-config.yaml -# HAProxy is deployed as a static Pod, which starts only after Kubelet is running. -# Therefore, the control plane must be reachable before HAProxy starts (e.g., by configuring localDNS file). -- name: Write tmp dns to localDNS file - command: | - cat >> {{ .item }} << EOF - # haproxy dns for kubekey begin - {{- if ne .internal_ipv4 "" }} - {{ .internal_ipv4 }} {{ .kubernetes.control_plane_endpoint.host }} - {{- else if ne .internal_ipv6 "" }} - {{ .internal_ipv6 }} {{ .kubernetes.control_plane_endpoint.host }} - {{- end }} - # haproxy dns for kubekey end - EOF - when: - - .kubernetes.control_plane_endpoint.type | eq "haproxy" - loop: "{{ .localDNS | toJson }}" - - name: Init kubernetes cluster + block: + - name: pre init + when: + - .kube_version | semverCompare ">=v1.29.0" + - eq .kubernetes.control_plane_endpoint.type "kube_vip" + command: | + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + - name: init + command: | + /usr/local/bin/kubeadm init --config=/etc/kubernetes/kubeadm-config.yaml --ignore-preflight-errors=FileExisting-crictl,ImagePull {{ if not .kubernetes.kube_proxy.enabled }}--skip-phases=addon/kube-proxy{{ end }} + - name: post init + when: + - .kube_version | semverCompare ">=v1.29.0" + - eq .kubernetes.control_plane_endpoint.type "kube_vip" + command: | + sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + +# reset localDNS 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}. +# if not the control_plane_endpoint will valid after kube_vip pod running. the task which will execute kubectl apply in current node may be failed. +- name: reset control_plane localDNS command: | - /usr/local/bin/kubeadm init --config=/etc/kubernetes/kubeadm-config.yaml --ignore-preflight-errors=FileExisting-crictl,ImagePull {{ if not .kubernetes.kube_proxy.enabled }}--skip-phases=addon/kube-proxy{{ end }} + sed -i ':a;$!{N;ba};s@# kubekey control_plane BEGIN.*# kubekey control_plane END@@' {{ .item }} + cat >> {{ .item }} << EOF + # kubekey control_plane BEGIN. + 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }} + # kubekey control_plane END + EOF + loop: "{{ .localDNS | toJson }}" - name: Copy kubeconfig to default dir command: | diff --git a/builtin/core/roles/kubernetes/init-kubernetes/tasks/main.yaml b/builtin/core/roles/kubernetes/init-kubernetes/tasks/main.yaml index 81be6527..ea0eecf8 100644 --- a/builtin/core/roles/kubernetes/init-kubernetes/tasks/main.yaml +++ b/builtin/core/roles/kubernetes/init-kubernetes/tasks/main.yaml @@ -22,14 +22,16 @@ {{ index $notInitNodes (randInt 0 ((sub ($notInitNodes | len) 1) | int)) }} {{- end -}} -- name: Set custom hosts to localDNS file - when: eq .kubernetes.control_plane_endpoint.type "local" +- name: Set control_plane hosts to localDNS file + when: + - .kubernetes.control_plane_endpoint.type | eq "local" + - .inventory_hostname | eq .init_kubernetes_node | not command: | - sed -i ':a;$!{N;ba};s@# kubekey custom BEGIN.*# kubekey custom END@@' {{ .item }} + sed -i ':a;$!{N;ba};s@# kubekey control_plane BEGIN.*# kubekey control_plane END@@' {{ .item }} cat >> {{ .item }} <> {{ .item }} <> {{ .item }} <> {{ .item }} << EOF + # kubekey control_plane BEGIN. 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }} - # kubekey custom END - EOF + # kubekey control_plane END + EOF loop: "{{ .localDNS | toJson }}" \ No newline at end of file diff --git a/builtin/core/roles/kubernetes/join-kubernetes/tasks/main.yaml b/builtin/core/roles/kubernetes/join-kubernetes/tasks/main.yaml index 0b984010..f82b2b49 100644 --- a/builtin/core/roles/kubernetes/join-kubernetes/tasks/main.yaml +++ b/builtin/core/roles/kubernetes/join-kubernetes/tasks/main.yaml @@ -1,45 +1,4 @@ -- name: Join kubernetes - when: ne .inventory_hostname .init_kubernetes_node - block: - # HAProxy is deployed as a static Pod, which starts only after Kubelet is running. - # Therefore, the control plane must be reachable before HAProxy starts (e.g., by configuring localDNS file). - - name: Write tmp dns to localDNS file - command: | - cat >> {{ .item }} << EOF - # haproxy dns for kubekey begin - {{- $internalIPv4 := index .hostvars (.groups.kube_control_plane | default list | first) "internal_ipv4" | default "" }} - {{- $internalIPv6 := index .hostvars (.groups.kube_control_plane | default list | first) "internal_ipv6" | default "" }} - {{- if ne $internalIPv4 "" }} - {{ $internalIPv4 }} {{ .kubernetes.control_plane_endpoint.host }} - {{- else if ne $internalIPv6 "" }} - {{ $internalIPv6 }} {{ .kubernetes.control_plane_endpoint.host }} - {{- end }} - # haproxy dns for kubekey end - EOF - when: .kubernetes.control_plane_endpoint.type | eq "haproxy" - loop: "{{ .localDNS | toJson }}" - - include_tasks: join_kubernetes.yaml - when: .kubernetes_install_service.stdout | eq "inactive" - - name: Replace haproxy dns to localDNS file - when: .kubernetes.control_plane_endpoint.type | eq "haproxy" - block: - - name: Replace control_plane by local hosts - when: .groups.kube_control_plane | default list | has .inventory_hostname - command: | - new_dns="# haproxy dns for kubekey begin - {{- if ne .internal_ipv4 "" }} - {{ .internal_ipv4 }} {{ .kubernetes.control_plane_endpoint.host }} - {{- else if ne .internal_ipv6 "" }} - {{ .internal_ipv6 }} {{ .kubernetes.control_plane_endpoint.host }} - {{- end }} - # haproxy dns for kubekey end" - sed -i '/# haproxy dns for kubekey begin/,/# haproxy dns for kubekey end/c\'"$new_dns" {{ .item }} - loop: "{{ .localDNS | toJson }}" - - name: Replace worker by haproxy hosts - when: .groups.worker | default list | has .inventory_hostname - command: | - new_dns="# haproxy dns for kubekey begin - {{ .kubernetes.control_plane_endpoint.haproxy.address }} {{ .kubernetes.control_plane_endpoint.host }} - # haproxy dns for kubekey end" - sed -i '/# haproxy dns for kubekey begin/,/# haproxy dns for kubekey end/c\'"$new_dns" {{ .item }} - loop: "{{ .localDNS | toJson }}" +- include_tasks: join_kubernetes.yaml + when: + - ne .inventory_hostname .init_kubernetes_node + - .kubernetes_install_service.stdout | eq "inactive" diff --git a/builtin/core/roles/kubernetes/pre-kubernetes/tasks/high-availability/kube_vip.yaml b/builtin/core/roles/kubernetes/pre-kubernetes/tasks/high-availability/kube_vip.yaml index d0c2dcc5..ac13a84e 100644 --- a/builtin/core/roles/kubernetes/pre-kubernetes/tasks/high-availability/kube_vip.yaml +++ b/builtin/core/roles/kubernetes/pre-kubernetes/tasks/high-availability/kube_vip.yaml @@ -2,9 +2,9 @@ - name: Get network interface for kube_vip command: | {{- if .kubernetes.control_plane_endpoint.kube_vip.address | ipFamily | eq "IPv4" }} - ip route | grep ' {{ .internal_ipv4 }} ' | grep 'proto kernel scope link src' | sed -e \"s/^.*dev.//\" -e \"s/.proto.*//\"| uniq + ip route | grep '{{ .internal_ipv4 }}' | grep 'proto kernel scope link src' | awk '{print $3}' {{- else if .kubernetes.control_plane_endpoint.host | ipFamily | eq "IPv6" }} - ip route | grep ' {{ .internal_ipv6 }} ' | grep 'proto kernel scope link src' | sed -e \"s/^.*dev.//\" -e \"s/.proto.*//\"| uniq + ip route | grep '{{ .internal_ipv6 }}' | grep 'proto kernel scope link src' | awk '{print $3}' {{- else }} echo "kubernetes.control_plane_endpoint.kube_vip.address" should be ipv4 or ipv6 exit 1 @@ -14,12 +14,12 @@ - name: Check if network is exist assert: that: - - .interface.stderr == "" - - .interface.stdout != "" + - .interface.stderr | eq "" + - .interface.stdout | ne "" fail_msg: "cannot find network interface to match kube_vip" - name: Generate kube_vip manifest template: src: | kubevip/kubevip.{{ .kubernetes.control_plane_endpoint.kube_vip.mode }} - dest: /etc/kubernetes/manifests/kubevip.yaml + dest: /etc/kubernetes/manifests/kube-vip.yaml diff --git a/builtin/core/roles/kubernetes/pre-kubernetes/tasks/high-availability/main.yaml b/builtin/core/roles/kubernetes/pre-kubernetes/tasks/high-availability/main.yaml index 86a85b35..d69e4edd 100644 --- a/builtin/core/roles/kubernetes/pre-kubernetes/tasks/high-availability/main.yaml +++ b/builtin/core/roles/kubernetes/pre-kubernetes/tasks/high-availability/main.yaml @@ -1,3 +1,59 @@ +# set localDNS for each .kubernetes.control_plane_endpoint.type +# local: +# before init cluster +# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }} +# - worker: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }} +# after init cluster +# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }} +# - worker: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }} +# before join cluster +# - control_plane: {{ .init_kubernetes_node }} {{ .kubernetes.control_plane_endpoint.host }} +# - worker: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }} +# after join cluster +# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }} +# - worker: {{ .init_kubernetes_node }} {{ .kubernetes.control_plane_endpoint.host }} +# +# kube_vip: +# before init cluster +# - control_plane: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }} +# - worker: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }} +# after init cluster +# - control_plane: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }} +# - worker: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }} +# before join cluster +# - control_plane: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }} +# - worker: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }} +# after join cluster +# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }} +# - worker: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }} +# +# haproxy: +# before init cluster +# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }} +# - worker: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }} +# after init cluster +# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }} +# - worker: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }} +# before join cluster +# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }} +# - worker: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }} +# after join cluster +# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }} +# - worker: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }} +- name: Set Control Plane to localDNS file + command: | + sed -i ':a;$!{N;ba};s@# kubekey control_plane BEGIN.*# kubekey control_plane END@@' {{ .item }} + cat >> {{ .item }} << EOF + # kubekey control_plane BEGIN. + {{- if .kubernetes.control_plane_endpoint.type | eq "kube_vip" }} + {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }} + {{- else }} + 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }} + {{- end }} + # kubekey control_plane END + EOF + loop: "{{ .localDNS | toJson }}" + # install with static pod: https://kube-vip.io/docs/installation/static/ - include_tasks: high-availability/kube_vip.yaml when: diff --git a/config/capkk/release/cluster-template.yaml b/config/capkk/release/cluster-template.yaml index 34aae3c8..38e17a05 100644 --- a/config/capkk/release/cluster-template.yaml +++ b/config/capkk/release/cluster-template.yaml @@ -27,7 +27,7 @@ kind: KKCluster metadata: name: '${CLUSTER_NAME}' spec: - controlPlaneEndpointType: ${CONTROL_PLANE_ENDPOINT_TYPE="vip"} + controlPlaneEndpointType: ${CONTROL_PLANE_ENDPOINT_TYPE="kube_vip"} inventory: ${INVENTORY_HOSTS} --- kind: KubeadmControlPlane