fix: Kube_vip may fail when used with Kubernetes versions above v1.29. (#2597)

Signed-off-by: joyceliu <joyceliu@yunify.com>

api/capkk/infrastructure/v1beta1/kkcluster_types.go

@@ -80,7 +80,7 @@ const (
 	// - ARP Mode: Requires the management cluster and worker cluster nodes to be in the same network segment.
 	// - BGP Mode: Requires a network environment that supports BGP, with proper configuration in both
 	//   the management and worker clusters.
-	ControlPlaneEndpointTypeVIP ControlPlaneEndpointType = "vip"
+	ControlPlaneEndpointTypeVIP ControlPlaneEndpointType = "kube_vip"
 )
 
 type InventoryHostConnector struct {

builtin/capkk/roles/install/cloud-config/tasks/main.yaml

@@ -1,7 +1,7 @@
 ---   
 - name: Install kube-vip
   when: 
-    - eq .kubernetes.control_plane_endpoint.type "vip"
+    - eq .kubernetes.control_plane_endpoint.type "kube_vip"
     - or (.kubernetes.roles | has "master") (.kubernetes.roles | has "control-plane")
   template:
     src: kube-vip.yaml

builtin/core/playbooks/delete_cluster.yaml

@@ -18,7 +18,7 @@
       ignore_errors: true
       command: |
         sed -i ':a;$!{N;ba};s@# kubekey hosts BEGIN.*# kubekey hosts END@@' {{ .item }}
-        sed -i ':a;$!{N;ba};s@# kubekey custom BEGIN.*# kubekey custom END@@' {{ .item }}
+        sed -i ':a;$!{N;ba};s@# kubekey control_plane BEGIN.*# kubekey control_plane END@@' {{ .item }}
       when: .deleteDNS
       loop: "{{ .localDNS | toJson }}"
 

builtin/core/playbooks/delete_nodes.yaml

@@ -61,7 +61,7 @@
       ignore_errors: true
       command: |
         sed -i ':a;$!{N;ba};s@# kubekey hosts BEGIN.*# kubekey hosts END@@' {{ .item }}
-        sed -i ':a;$!{N;ba};s@# kubekey custom BEGIN.*# kubekey custom END@@' {{ .item }}
+        sed -i ':a;$!{N;ba};s@# kubekey control_plane BEGIN.*# kubekey control_plane END@@' {{ .item }}
       when: 
         - .deleteDNS
         - .delete_nodes | default list | has .inventory_hostname

builtin/core/roles/kubernetes/init-kubernetes/tasks/init_kubernetes.yaml

@@ -33,26 +33,37 @@
       {{- end }}
     dest: /etc/kubernetes/kubeadm-config.yaml
 
-# HAProxy is deployed as a static Pod, which starts only after Kubelet is running.
-# Therefore, the control plane must be reachable before HAProxy starts (e.g., by configuring localDNS file).
-- name: Write tmp dns to localDNS file
-  command: |
-    cat >> {{ .item }} << EOF
-    # haproxy dns for kubekey begin
-    {{- if ne .internal_ipv4 "" }}
-    {{ .internal_ipv4 }} {{ .kubernetes.control_plane_endpoint.host }}
-    {{- else if ne .internal_ipv6 "" }}
-    {{ .internal_ipv6  }} {{ .kubernetes.control_plane_endpoint.host }}
-    {{- end }}
-    # haproxy dns for kubekey end
-    EOF
-  when:
-    - .kubernetes.control_plane_endpoint.type | eq "haproxy"
-  loop: "{{ .localDNS | toJson }}"
-
 - name: Init kubernetes cluster
+  block:
+    - name: pre init
+      when: 
+        - .kube_version | semverCompare ">=v1.29.0"
+        - eq .kubernetes.control_plane_endpoint.type "kube_vip"
+      command: |
+        sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \
+          /etc/kubernetes/manifests/kube-vip.yaml
+    - name: init
+      command: |
+        /usr/local/bin/kubeadm init --config=/etc/kubernetes/kubeadm-config.yaml --ignore-preflight-errors=FileExisting-crictl,ImagePull {{ if not .kubernetes.kube_proxy.enabled }}--skip-phases=addon/kube-proxy{{ end }}
+    - name: post init
+      when: 
+        - .kube_version | semverCompare ">=v1.29.0"
+        - eq .kubernetes.control_plane_endpoint.type "kube_vip"
+      command: |
+        sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \
+          /etc/kubernetes/manifests/kube-vip.yaml
+
+# reset localDNS 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}. 
+# if not the control_plane_endpoint will valid after kube_vip pod running. the task which will execute kubectl apply in current node may be failed.
+- name: reset control_plane localDNS
   command: |
-    /usr/local/bin/kubeadm init --config=/etc/kubernetes/kubeadm-config.yaml --ignore-preflight-errors=FileExisting-crictl,ImagePull {{ if not .kubernetes.kube_proxy.enabled }}--skip-phases=addon/kube-proxy{{ end }}
+    sed -i ':a;$!{N;ba};s@# kubekey control_plane BEGIN.*# kubekey control_plane END@@' {{ .item }}
+    cat >> {{ .item }} << EOF
+    # kubekey control_plane BEGIN.
+    127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
+    # kubekey control_plane END
+    EOF 
+  loop: "{{ .localDNS | toJson }}"
     
 - name: Copy kubeconfig to default dir
   command: |

builtin/core/roles/kubernetes/init-kubernetes/tasks/main.yaml

@@ -22,14 +22,16 @@
       {{ index $notInitNodes (randInt 0 ((sub ($notInitNodes | len) 1) | int)) }}
       {{- end -}}
 
-- name: Set custom hosts to localDNS file
-  when: eq .kubernetes.control_plane_endpoint.type "local"
+- name: Set control_plane hosts to localDNS file
+  when: 
+    - .kubernetes.control_plane_endpoint.type | eq  "local"
+    - .inventory_hostname | eq .init_kubernetes_node | not
   command: |
-    sed -i ':a;$!{N;ba};s@# kubekey custom BEGIN.*# kubekey custom END@@' {{ .item }}
+    sed -i ':a;$!{N;ba};s@# kubekey control_plane BEGIN.*# kubekey control_plane END@@' {{ .item }}
     cat >> {{ .item }} <<EOF
-    # kubekey custom BEGIN
-    127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
-    # kubekey custom END
+    # kubekey control_plane BEGIN.
+    {{ .init_kubernetes_node }} {{ .kubernetes.control_plane_endpoint.host }}
+    # kubekey control_plane END
     EOF
   loop: "{{ .localDNS | toJson }}"
 

builtin/core/roles/kubernetes/join-kubernetes/tasks/join_kubernetes.yaml

@@ -1,20 +1,4 @@
 ---
-- name: Set init_kubernetes_node hosts to localDNS file
-  when: eq .kubernetes.control_plane_endpoint.type "local"
-  command: |
-    sed -i ':a;$!{N;ba};s@# kubekey custom BEGIN.*# kubekey custom END@@' {{ .item }}
-    cat >> {{ .item }} <<EOF
-    # kubekey custom BEGIN
-    {{- if and (index .hostvars .init_kubernetes_node "internal_ipv4") (ne (index .hostvars .init_kubernetes_node "internal_ipv4") "") }}
-    {{ index .hostvars .init_kubernetes_node "internal_ipv4" }} {{ .kubernetes.control_plane_endpoint.host }}
-    {{- end }}
-    {{- if and (index .hostvars .init_kubernetes_node "internal_ipv6") (ne (index .hostvars .init_kubernetes_node "internal_ipv6") "") }}
-    {{ index .hostvars .init_kubernetes_node "internal_ipv6" }} {{ .kubernetes.control_plane_endpoint.host }}
-    {{- end }}
-    # kubekey custom END
-    EOF
-  loop: "{{ .localDNS | toJson }}"
-
 - name: Generate kubeadm join config
   template:
     src: |
@@ -52,15 +36,18 @@
   command: |
     kubectl annotate {{ .hostname }} {{- range $k,$v := .annotations }}{{ printf "%s=%s" $k $v}} {{- end }}
 
-- name: Set change custom hosts to localDNS file
-  when: 
-    - eq .kubernetes.control_plane_endpoint.type "local"
+# reset localDNS 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}. 
+# if not the control_plane_endpoint will valid after kube_vip pod running. the task which will execute kubectl apply in current node may be failed.
+# the haproxy has be 127.0.0.1
+- name: reset control_plane localDNS
+  when:
     - .groups.kube_control_plane | default list | has .inventory_hostname
+    - .kubernetes.control_plane_endpoint.type | ne "haproxy"
   command: |
-    sed -i ':a;$!{N;ba};s@# kubekey custom BEGIN.*# kubekey custom END@@' {{ .item }}
-    cat >> {{ .item }} <<EOF
-    # kubekey custom BEGIN
+    sed -i ':a;$!{N;ba};s@# kubekey control_plane BEGIN.*# kubekey control_plane END@@' {{ .item }}
+    cat >> {{ .item }} << EOF
+    # kubekey control_plane BEGIN.
     127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
-    # kubekey custom END
-    EOF
+    # kubekey control_plane END
+    EOF 
   loop: "{{ .localDNS | toJson }}"

builtin/core/roles/kubernetes/join-kubernetes/tasks/main.yaml

@@ -1,45 +1,4 @@
-- name: Join kubernetes
-  when: ne .inventory_hostname .init_kubernetes_node
-  block:
-    # HAProxy is deployed as a static Pod, which starts only after Kubelet is running.
-    # Therefore, the control plane must be reachable before HAProxy starts (e.g., by configuring localDNS file).
-    - name: Write tmp dns to localDNS file
-      command: |
-        cat >> {{ .item }} << EOF
-        # haproxy dns for kubekey begin
-        {{- $internalIPv4 := index .hostvars (.groups.kube_control_plane | default list | first) "internal_ipv4" | default "" }}
-        {{- $internalIPv6 := index .hostvars (.groups.kube_control_plane | default list | first) "internal_ipv6" | default "" }}
-        {{- if ne $internalIPv4 "" }}
-        {{ $internalIPv4 }} {{ .kubernetes.control_plane_endpoint.host }}
-        {{- else if ne $internalIPv6 "" }}
-        {{ $internalIPv6 }} {{ .kubernetes.control_plane_endpoint.host }}
-        {{- end }}
-        # haproxy dns for kubekey end
-        EOF
-      when: .kubernetes.control_plane_endpoint.type | eq "haproxy"
-      loop: "{{ .localDNS | toJson }}"
-    - include_tasks: join_kubernetes.yaml
-      when: .kubernetes_install_service.stdout | eq "inactive"
-    - name: Replace haproxy dns to localDNS file
-      when: .kubernetes.control_plane_endpoint.type | eq "haproxy"
-      block:
-        - name: Replace control_plane by local hosts
-          when: .groups.kube_control_plane | default list | has .inventory_hostname
-          command: |
-            new_dns="# haproxy dns for kubekey begin
-            {{- if ne .internal_ipv4 "" }}
-            {{ .internal_ipv4 }} {{ .kubernetes.control_plane_endpoint.host }}
-            {{- else if ne .internal_ipv6 "" }}
-            {{ .internal_ipv6  }} {{ .kubernetes.control_plane_endpoint.host }}
-            {{- end }}
-            # haproxy dns for kubekey end"
-            sed -i '/# haproxy dns for kubekey begin/,/# haproxy dns for kubekey end/c\'"$new_dns" {{ .item }}
-          loop: "{{ .localDNS | toJson }}"
-        - name: Replace worker by haproxy hosts
-          when: .groups.worker | default list | has .inventory_hostname
-          command: |
-            new_dns="# haproxy dns for kubekey begin
-            {{ .kubernetes.control_plane_endpoint.haproxy.address }} {{ .kubernetes.control_plane_endpoint.host }}
-            # haproxy dns for kubekey end"
-            sed -i '/# haproxy dns for kubekey begin/,/# haproxy dns for kubekey end/c\'"$new_dns" {{ .item }}
-          loop: "{{ .localDNS | toJson }}"
+- include_tasks: join_kubernetes.yaml
+  when: 
+    - ne .inventory_hostname .init_kubernetes_node
+    - .kubernetes_install_service.stdout | eq "inactive"

builtin/core/roles/kubernetes/pre-kubernetes/tasks/high-availability/kube_vip.yaml

@@ -2,9 +2,9 @@
 - name: Get network interface for kube_vip
   command: |
     {{- if .kubernetes.control_plane_endpoint.kube_vip.address | ipFamily | eq "IPv4" }}
-    ip route | grep ' {{ .internal_ipv4 }} ' | grep 'proto kernel scope link src' | sed -e \"s/^.*dev.//\" -e \"s/.proto.*//\"| uniq
+    ip route | grep '{{ .internal_ipv4 }}' | grep 'proto kernel scope link src' | awk '{print $3}'
     {{- else if .kubernetes.control_plane_endpoint.host | ipFamily | eq "IPv6" }}
-    ip route | grep ' {{ .internal_ipv6 }} ' | grep 'proto kernel scope link src' | sed -e \"s/^.*dev.//\" -e \"s/.proto.*//\"| uniq
+    ip route | grep '{{ .internal_ipv6 }}' | grep 'proto kernel scope link src' | awk '{print $3}'
     {{- else }}
     echo "kubernetes.control_plane_endpoint.kube_vip.address" should be ipv4 or ipv6
     exit 1
@@ -14,12 +14,12 @@
 - name: Check if network is exist
   assert:
     that:
-      - .interface.stderr == ""
-      - .interface.stdout != ""
+      - .interface.stderr | eq ""
+      - .interface.stdout | ne ""
     fail_msg: "cannot find network interface to match kube_vip"
 
 - name: Generate kube_vip manifest
   template:
     src: |
       kubevip/kubevip.{{ .kubernetes.control_plane_endpoint.kube_vip.mode }}
-    dest: /etc/kubernetes/manifests/kubevip.yaml
+    dest: /etc/kubernetes/manifests/kube-vip.yaml

builtin/core/roles/kubernetes/pre-kubernetes/tasks/high-availability/main.yaml

@@ -1,3 +1,59 @@
+# set localDNS for each .kubernetes.control_plane_endpoint.type
+# local:
+# before init cluster
+# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
+# - worker: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
+# after init cluster
+# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
+# - worker: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
+# before join cluster
+# - control_plane: {{ .init_kubernetes_node }} {{ .kubernetes.control_plane_endpoint.host }}
+# - worker: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
+# after join cluster
+# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
+# - worker: {{ .init_kubernetes_node }} {{ .kubernetes.control_plane_endpoint.host }}
+#
+# kube_vip:
+# before init cluster
+# - control_plane: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }}
+# - worker: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }}
+# after init cluster
+# - control_plane: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }}
+# - worker: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }}
+# before join cluster
+# - control_plane: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }}
+# - worker: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }}
+# after join cluster
+# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
+# - worker: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }}
+#
+# haproxy:
+# before init cluster
+# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
+# - worker: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
+# after init cluster
+# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
+# - worker: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
+# before join cluster
+# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
+# - worker: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
+# after join cluster
+# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
+# - worker: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
+- name: Set Control Plane to localDNS file
+  command: |
+    sed -i ':a;$!{N;ba};s@# kubekey control_plane BEGIN.*# kubekey control_plane END@@' {{ .item }}
+    cat >> {{ .item }} << EOF
+    # kubekey control_plane BEGIN.
+    {{- if .kubernetes.control_plane_endpoint.type | eq "kube_vip" }}
+    {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }}
+    {{- else }}
+    127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
+    {{- end }}
+    # kubekey control_plane END
+    EOF
+  loop: "{{ .localDNS | toJson }}"
+
 # install with static pod: https://kube-vip.io/docs/installation/static/
 - include_tasks: high-availability/kube_vip.yaml
   when:

config/capkk/release/cluster-template.yaml

@@ -27,7 +27,7 @@ kind: KKCluster
 metadata:
   name: '${CLUSTER_NAME}'
 spec:
-  controlPlaneEndpointType: ${CONTROL_PLANE_ENDPOINT_TYPE="vip"}
+  controlPlaneEndpointType: ${CONTROL_PLANE_ENDPOINT_TYPE="kube_vip"}
   inventory: ${INVENTORY_HOSTS}
 ---
 kind: KubeadmControlPlane