From e109fbb7cf3c6755db7e8369fa70c310b58a9684 Mon Sep 17 00:00:00 2001
From: liujian <joyceliu@yunify.com>
Date: Wed, 11 Jun 2025 18:56:26 +0800
Subject: [PATCH] fix: Offline install (#2611)

Signed-off-by: joyceliu <joyceliu@yunify.com>
---
 builtin/capkk/playbooks/vars/main.yaml        | 17 +++----
 .../init-artifacts/tasks/download_binary.yaml | 22 ++++-----
 .../init-artifacts/tasks/download_helm.yaml   |  6 +--
 .../init/init-os/tasks/init_ntpserver.yaml    | 14 +++---
 .../roles/init/init-os/templates/init-os.sh   | 28 +++++------
 .../install/cloud-config/defaults/main.yaml   |  7 ++-
 .../cloud-config/templates/kube-vip.yaml      |  2 +-
 .../capkk/roles/install/cni/tasks/calico.yaml |  2 +-
 .../capkk/roles/install/cni/tasks/cilium.yaml |  2 +-
 .../roles/install/cri/defaults/main.yaml      |  6 ++-
 .../install/cri/tasks/install_containerd.yaml |  4 +-
 .../install/cri/tasks/install_crictl.yaml     |  2 +-
 .../install/cri/tasks/install_cridockerd.yaml |  2 +-
 .../install/cri/tasks/install_docker.yaml     |  2 +-
 .../install/cri/templates/containerd.config   |  6 +--
 .../install/cri/templates/cri-dockerd.service |  2 +-
 .../roles/install/kubernetes/tasks/main.yaml  | 10 ++--
 .../kubernetes/templates/kubeadm/kubelet.env  |  4 +-
 .../install/storageclass/defaults/main.yaml   | 14 ++++--
 .../storageclass/templates/local-volume.yaml  |  4 +-
 .../roles/precheck/env_check/tasks/cri.yaml   |  4 +-
 .../precheck/env_check/tasks/kubernetes.yaml  |  4 +-
 builtin/core/defaults/config/v1.23.15.yaml    |  3 +-
 builtin/core/defaults/config/v1.31.2.yaml     | 34 +++++++++++++-
 builtin/core/playbooks/delete_nodes.yaml      |  1 +
 .../vars/create_cluster_kubernetes.yaml       | 33 +++++++++----
 .../init/init-artifact/defaults/main.yaml     |  4 +-
 .../core/roles/install/cni/defaults/main.yaml | 10 +++-
 .../roles/install/cni/templates/multus.yaml   |  2 +-
 .../core/roles/install/cri/defaults/main.yaml | 13 +++--
 .../install/cri/templates/containerd.config   | 10 ++--
 .../install/cri/templates/cri-dockerd.service |  2 +-
 .../install/image-registry/defaults/main.yaml | 11 +++--
 .../image-registry/tasks/load_images.yaml     | 47 +++++++++----------
 .../image-registry/templates/harbor.config    |  2 +-
 .../install/storageclass/defaults/main.yaml   | 14 ++++--
 .../storageclass/templates/local-volume.yaml  |  4 +-
 .../templates/dns/nodelocaldns.yaml           | 10 ++--
 .../templates/kubeadm/kubeadm-init.v1beta2    | 42 ++++++++---------
 .../templates/kubeadm/kubeadm-init.v1beta3    | 44 ++++++++---------
 .../templates/kubeadm/kubeadm-join.v1beta2    |  6 +--
 .../templates/kubeadm/kubeadm-join.v1beta3    |  6 +--
 .../templates/haproxy/haproxy.yaml            |  2 +-
 .../templates/kubevip/kubevip.ARP             |  2 +-
 .../templates/kubevip/kubevip.BGP             |  2 +-
 45 files changed, 267 insertions(+), 201 deletions(-)

diff --git a/builtin/capkk/playbooks/vars/main.yaml b/builtin/capkk/playbooks/vars/main.yaml
index 057f7bf0..960c1b49 100644
--- a/builtin/capkk/playbooks/vars/main.yaml
+++ b/builtin/capkk/playbooks/vars/main.yaml
@@ -7,32 +7,29 @@ cloud_config_dir: /capkk/cloud
 # tmp_dir for kubekey in remote node. it will store file like binary package, iso file etc.
 tmp_dir: /tmp/kubekey
 
-# image registry
+# global_registry for all image
 global_registry: ""
+# dockerio_registry for docker.io image.
 dockerio_registry: >-
-  {{- if ne .global_registry "" -}}
+  {{- if .global_registry | empty | not -}}
   {{ .global_registry }}
   {{- else -}}
   docker.io
   {{- end -}}
+# quayio_registry for quay.io image.
 quayio_registry: >-
-  {{- if ne .global_registry "" -}}
+  {{- if .global_registry | empty | not -}}
   {{ .global_registry }}
   {{- else -}}
   quay.io
   {{- end -}}
+# ghcrio_registry for ghcr.io image.
 ghcrio_registry: >-
-  {{- if ne .global_registry "" -}}
+  {{- if .global_registry | empty | not -}}
   {{ .global_registry }}
   {{- else -}}
   ghcr.io
   {{- end -}}
-k8s_registry: >-
-  {{- if ne .global_registry "" -}}
-  {{ .global_registry }}
-  {{- else -}}
-  registry.k8s.io
-  {{- end -}}
 
 cri:
   # support: containerd,docker
diff --git a/builtin/capkk/roles/init/init-artifacts/tasks/download_binary.yaml b/builtin/capkk/roles/init/init-artifacts/tasks/download_binary.yaml
index c1254adb..bec3e14d 100644
--- a/builtin/capkk/roles/init/init-artifacts/tasks/download_binary.yaml
+++ b/builtin/capkk/roles/init/init-artifacts/tasks/download_binary.yaml
@@ -33,7 +33,7 @@
         curl -L -o $kube_path/kubectl {{ get .artifact.artifact_url.kubectl .item }}
     fi
   loop: "{{ .artifact.arch | toJson }}"
-  when: and .kube_version (ne .kube_version "")
+  when: .kube_version | empty | not
 
 - name: Check binaries for cni_plugins
   command: |
@@ -50,7 +50,7 @@
       curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.cni_plugins .item }}
     fi
   loop: "{{ .artifact.arch | toJson }}"
-  when: and .cni_plugins_version (ne .cni_plugins_version "")
+  when: .cni_plugins_version  | empty | not
 
 - name: Check binaries for ciliumcli
   command: |
@@ -68,8 +68,8 @@
     fi
   loop: "{{ .artifact.arch | toJson }}"
   when: 
-    - and .cilium_version (ne .cilium_version "")
-    - and .ciliumcli_version (ne .ciliumcli_version "")
+    - .cilium_version | empty | not
+    - .ciliumcli_version  | empty | not
 
 - name: Check binaries for helm
   command: |
@@ -86,7 +86,7 @@
       curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.helm .item }}
     fi
   loop: "{{ .artifact.arch | toJson }}"
-  when: and .helm_version (ne .helm_version "")
+  when: .helm_version  | empty | not
 
 - name: Check binaries for crictl
   command: |
@@ -103,7 +103,7 @@
       curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.crictl .item }}
     fi
   loop: "{{ .artifact.arch | toJson }}"
-  when: and .crictl_version (ne .crictl_version "")
+  when: .crictl_version  | empty | not
 
 - name: Check binaries for docker
   command: |
@@ -120,7 +120,7 @@
       curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.docker .item }}
     fi
   loop: "{{ .artifact.arch | toJson }}"
-  when: and .docker_version (ne .docker_version "")
+  when: .docker_version  | empty | not
 
 - name: Check binaries for cridockerd
   command: |
@@ -137,7 +137,7 @@
       curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.cridockerd .item }}
     fi
   loop: "{{ .artifact.arch | toJson }}"
-  when: and .cridockerd_version (ne .docker_version "")
+  when: .cridockerd_version | empty | not
 
 - name: Check binaries for containerd
   command: |
@@ -154,7 +154,7 @@
       curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.containerd .item }}
     fi
   loop: "{{ .artifact.arch | toJson }}"
-  when: and .containerd_version (ne .containerd_version "")
+  when: .containerd_version | empty | not
 
 - name: Check binaries for runc
   command: |
@@ -171,7 +171,7 @@
       curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.runc .item }}
     fi
   loop: "{{ .artifact.arch | toJson }}"
-  when: and .runc_version (ne .runc_version "")
+  when: .runc_version | empty | not
 
 - name: Check binaries for calicoctl
   command: |
@@ -188,4 +188,4 @@
       curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.calicoctl .item }}
     fi
   loop: "{{ .artifact.arch | toJson }}"
-  when: and .calico_version (ne .calico_version "")
\ No newline at end of file
+  when: .calico_version | empty | not
\ No newline at end of file
diff --git a/builtin/capkk/roles/init/init-artifacts/tasks/download_helm.yaml b/builtin/capkk/roles/init/init-artifacts/tasks/download_helm.yaml
index 6d6d3460..3e68c9a1 100644
--- a/builtin/capkk/roles/init/init-artifacts/tasks/download_helm.yaml
+++ b/builtin/capkk/roles/init/init-artifacts/tasks/download_helm.yaml
@@ -8,7 +8,7 @@
       # download online
       curl -Lo $artifact_path/$artifact_name {{ .artifact.artifact_url.calico }}
     fi
-  when: and .calico_version (ne .calico_version "")
+  when: .calico_version | empty | not
 
 - name: Check binaries for cilium
   command: |
@@ -19,7 +19,7 @@
       # download online
       curl -Lo $artifact_path/$artifact_name {{ .artifact.artifact_url.cilium }}
     fi
-  when: and .cilium_version (ne .cilium_version "")
+  when: .cilium_version | empty | not
 
 - name: Check binaries for flannel
   command: |
@@ -30,4 +30,4 @@
       # download online
       curl -Lo $artifact_path/$artifact_name {{ .artifact.artifact_url.flannel }}
     fi
-  when: and .flannel_version (ne .flannel_version "")
\ No newline at end of file
+  when: .flannel_version | empty | not
\ No newline at end of file
diff --git a/builtin/capkk/roles/init/init-os/tasks/init_ntpserver.yaml b/builtin/capkk/roles/init/init-os/tasks/init_ntpserver.yaml
index 5b972c4d..32316882 100644
--- a/builtin/capkk/roles/init/init-os/tasks/init_ntpserver.yaml
+++ b/builtin/capkk/roles/init/init-os/tasks/init_ntpserver.yaml
@@ -17,8 +17,8 @@
       {{- $internalIPv6 := "" }}
       {{- range $.hostvars }}
         {{- if eq .hostname $server }}
-          {{- $internalIPv4 = .internal_ipv4 | default "" }}
-          {{- $internalIPv6 = .internal_ipv6 | default "" }}
+          {{- $internalIPv4 = .internal_ipv4 }}
+          {{- $internalIPv6 = .internal_ipv6 }}
         {{- end }}
       {{- end }}
     # add ntp server: {{ $server }}
@@ -28,19 +28,19 @@
       {{- if $internalIPv6 }}
     grep -q '^server {{ $internalIPv6 }} iburst' $chronyConfigFile || sed '1a server [{{ $internalIPv6 }}] iburst' -i $chronyConfigFile
       {{- end }}
-      {{- if and (eq $internalIPv4 "") (eq $internalIPv6 "") }}
+      {{- if and ($internalIPv4 | empty) ($internalIPv6 | empty) }}
     grep -q '^server {{ $server }} iburst' $chronyConfigFile || sed '1a server {{ $server }} iburst' -i $chronyConfigFile
       {{- end }}      
     {{- end }}
   when:
     - .ntp.enabled
-    - .ntp.servers | fromJson | len | lt 0
+    - .ntp.servers | fromJson | empty | not
 
 - name: Set timezone
   command: |
     timedatectl set-timezone {{ .timezone }}
-    timedatectl set-ntp {{ and .ntp.enabled (.ntp.servers | fromJson | len | lt 0) }}
-  when: or (and .ntp.enabled (.ntp.servers | fromJson | len | lt 0)) (.timezone | ne "")
+    timedatectl set-ntp {{ and .ntp.enabled (.ntp.servers | fromJson empty | not) }}
+  when: or (and .ntp.enabled (.ntp.servers | fromJson | empty | not)) (.timezone | empty | not)
 
 - name: Restart ntp server
   command: |
@@ -48,4 +48,4 @@
     systemctl restart chrony.service
     {{- end }}
     systemctl restart chronyd.service
-  when: or (and .ntp.enabled (.ntp.servers | fromJson | len | lt 0)) (.timezone | ne "")
+  when: or (and .ntp.enabled (.ntp.servers | fromJson | empty | not)) (.timezone | empty | not)
diff --git a/builtin/capkk/roles/init/init-os/templates/init-os.sh b/builtin/capkk/roles/init/init-os/templates/init-os.sh
index 2e0d8745..52b79b72 100644
--- a/builtin/capkk/roles/init/init-os/templates/init-os.sh
+++ b/builtin/capkk/roles/init/init-os/templates/init-os.sh
@@ -47,7 +47,7 @@ echo 'fs.aio-max-nr = 262144' >> /etc/sysctl.conf
 echo 'kernel.pid_max = 65535' >> /etc/sysctl.conf
 echo 'kernel.watchdog_thresh = 5' >> /etc/sysctl.conf
 echo 'kernel.hung_task_timeout_secs = 5' >> /etc/sysctl.conf
-{{- if and .internal_ipv4 (.internal_ipv4 | ne "") }}
+{{- if .internal_ipv4 | empty | not }}
 # add for ipv4
 echo 'net.ipv4.ip_forward = 1' >> /etc/sysctl.conf
 echo 'net.bridge.bridge-nf-call-ip6tables = 1' >> /etc/sysctl.conf
@@ -68,7 +68,7 @@ echo 'net.ipv4.conf.default.arp_accept = 1' >> /etc/sysctl.conf
 echo 'net.ipv4.conf.all.arp_ignore = 1' >> /etc/sysctl.conf
 echo 'net.ipv4.conf.default.arp_ignore = 1' >> /etc/sysctl.conf
 {{- end }}
-{{- if and .internal_ipv6 (.internal_ipv6 | ne "") }}
+{{- if .internal_ipv6 | empty | not }}
 # add for ipv6
 echo 'net.bridge.bridge-nf-call-iptables = 1' >> /etc/sysctl.conf
 echo 'net.ipv6.conf.all.disable_ipv6 = 0' >> /etc/sysctl.conf
@@ -98,7 +98,7 @@ sed -r -i  "s@#{0,}?net.core.somaxconn ?= ?([0-9]{1,})@net.core.somaxconn = 3276
 sed -r -i  "s@#{0,}?fs.aio-max-nr ?= ?([0-9]{1,})@fs.aio-max-nr = 262144@g" /etc/sysctl.conf
 sed -r -i  "s@#{0,}?kernel.watchdog_thresh ?= ?([0-9]{1,})@kernel.watchdog_thresh = 5@g" /etc/sysctl.conf
 sed -r -i  "s@#{0,}?kernel.hung_task_timeout_secs ?= ?([0-9]{1,})@kernel.hung_task_timeout_secs = 5@g" /etc/sysctl.conf
-{{- if and .internal_ipv4 (.internal_ipv4 | ne "") }}
+{{- if .internal_ipv4 | empty | not }}
 sed -r -i "s@#{0,}?net.ipv4.tcp_tw_recycle ?= ?(0|1|2)@net.ipv4.tcp_tw_recycle = 0@g" /etc/sysctl.conf
 sed -r -i "s@#{0,}?net.ipv4.tcp_tw_reuse ?= ?(0|1)@net.ipv4.tcp_tw_reuse = 0@g" /etc/sysctl.conf
 sed -r -i "s@#{0,}?net.ipv4.conf.all.rp_filter ?= ?(0|1|2)@net.ipv4.conf.all.rp_filter = 1@g" /etc/sysctl.conf
@@ -119,7 +119,7 @@ sed -r -i  "s@#{0,}?net.ipv4.udp_wmem_min ?= ?([0-9]{1,})@net.ipv4.udp_wmem_min
 sed -r -i  "s@#{0,}?net.ipv4.conf.all.arp_ignore ?= ??(0|1|2)@net.ipv4.conf.all.arp_ignore = 1@g" /etc/sysctl.conf
 sed -r -i  "s@#{0,}?net.ipv4.conf.default.arp_ignore ?= ??(0|1|2)@net.ipv4.conf.default.arp_ignore = 1@g" /etc/sysctl.conf
 {{- end }}
-{{- if and .internal_ipv6 (.internal_ipv6 | ne "") }}
+{{- if .internal_ipv6 | empty | not }}
 #add for ipv6
 sed -r -i  "s@#{0,}?net.bridge.bridge-nf-call-ip6tables ?= ?(0|1)@net.bridge.bridge-nf-call-ip6tables = 1@g" /etc/sysctl.conf
 sed -r -i "s@#{0,}?net.ipv6.conf.all.disable_ipv6 ?= ?([0-9]{1,})@net.ipv6.conf.all.disable_ipv6 = 0@g" /etc/sysctl.conf
@@ -217,37 +217,37 @@ cat >>/etc/hosts<<EOF
   {{- $hostname := index $.hostvars . "hostname" -}}
   {{- $clusterName := $.kubernetes.cluster_name | default "kubekey" -}}
   {{- $dnsDomain := $.kubernetes.networking.dns_domain | default "cluster.local" -}}
-  {{- if and (index $.hostvars . "internal_ipv4") (ne (index $.hostvars . "internal_ipv4") "") }}
+  {{- if (index $.hostvars . "internal_ipv4") | empty | not }}
 {{ index $.hostvars . "internal_ipv4" }} {{ $hostname }} {{ printf "%s.%s" $hostname $clusterName }} {{ printf "%s.%s.%s" $hostname $clusterName $dnsDomain }}
   {{- end }}
-  {{- if and (index $.hostvars . "internal_ipv6") (ne (index $.hostvars . "internal_ipv6") "") }}
+  {{- if (index $.hostvars . "internal_ipv6") | empty | not }}
 {{ index $.hostvars . "internal_ipv6" }} {{ $hostname }} {{ printf "%s.%s" $hostname $clusterName }} {{ printf "%s.%s.%s" $hostname $clusterName $dnsDomain }}
   {{- end }}
 {{- end }}
 # etcd hosts
 {{- range .groups.etcd | default list }}
-  {{- if and (index $.hostvars . "internal_ipv4") (ne (index $.hostvars . "internal_ipv4") "") }}
+  {{- if (index $.hostvars . "internal_ipv4") | empty | not }}
 {{ index $.hostvars . "internal_ipv4" }} {{ index $.hostvars . "hostname" }}
   {{- end }}
-  {{- if and (index $.hostvars . "internal_ipv6") (ne (index $.hostvars . "internal_ipv6") "") }}
+  {{- if (index $.hostvars . "internal_ipv6") | empty | not }}
 {{ index $.hostvars . "internal_ipv6" }} {{ index $.hostvars . "hostname" }}
   {{- end }}
 {{- end }}
 # image registry hosts
 {{- range .groups.image_registry | default list }}
-  {{- if and (index $.hostvars . "internal_ipv4") (ne (index $.hostvars . "internal_ipv4") "") }}
+  {{- if (index $.hostvars . "internal_ipv4") | empty | not }}
 {{ index $.hostvars . "internal_ipv4" }} {{ index $.hostvars . "hostname" }}
   {{- end }}
-  {{- if and (index $.hostvars . "internal_ipv6") (ne (index $.hostvars . "internal_ipv6") "") }}
+  {{- if (index $.hostvars . "internal_ipv6") | empty | not }}
 {{ index $.hostvars . "internal_ipv6" }} {{ index $.hostvars . "hostname" }}
   {{- end }}
 {{- end }}
 # nfs hosts
 {{- range .groups.nfs | default list }}
-  {{- if and (index $.hostvars . "internal_ipv4") (ne (index $.hostvars . "internal_ipv4") "") }}
+  {{- if (index $.hostvars . "internal_ipv4") | empty | not }}
 {{ index $.hostvars . "internal_ipv4" }} {{ index $.hostvars . "hostname" }}
   {{- end }}
-  {{- if and (index $.hostvars . "internal_ipv6") (ne (index $.hostvars . "internal_ipv6") "") }}
+  {{- if (index $.hostvars . "internal_ipv6") | empty | not }}
 {{ index $.hostvars . "internal_ipv4" }} {{ index $.hostvars . "hostname" }}
   {{- end }}
 {{- end }}
@@ -258,10 +258,10 @@ sync
 # echo 3 > /proc/sys/vm/drop_caches
 
 # Make sure the iptables utility doesn't use the nftables backend.
-{{- if and .internal_ipv4 (.internal_ipv4 | ne "") }}
+{{- if .internal_ipv4 | empty | not }}
 update-alternatives --set iptables /usr/sbin/iptables-legacy >/dev/null 2>&1 || true
 {{- end }}
-{{- if and .internal_ipv6 (.internal_ipv6 | ne "") }}
+{{- if .internal_ipv6 | empty | not }}
 update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy >/dev/null 2>&1 || true
 {{- end }}
 update-alternatives --set arptables /usr/sbin/arptables-legacy >/dev/null 2>&1 || true
diff --git a/builtin/capkk/roles/install/cloud-config/defaults/main.yaml b/builtin/capkk/roles/install/cloud-config/defaults/main.yaml
index 3b405039..14d62515 100644
--- a/builtin/capkk/roles/install/cloud-config/defaults/main.yaml
+++ b/builtin/capkk/roles/install/cloud-config/defaults/main.yaml
@@ -1,5 +1,8 @@
 kubernetes:
   control_plane_endpoint:
     kube_vip:
-      image: >-
-        {{ .dockerio_registry }}/plndr/kube-vip:v0.7.2
\ No newline at end of file
+      image: 
+        registry: >-
+          {{ .dockerio_registry }}
+        repository: plndr/kube-vip
+        tag: v0.7.2
\ No newline at end of file
diff --git a/builtin/capkk/roles/install/cloud-config/templates/kube-vip.yaml b/builtin/capkk/roles/install/cloud-config/templates/kube-vip.yaml
index ed438880..45752e67 100644
--- a/builtin/capkk/roles/install/cloud-config/templates/kube-vip.yaml
+++ b/builtin/capkk/roles/install/cloud-config/templates/kube-vip.yaml
@@ -38,7 +38,7 @@ spec:
       value: "true"
     - name: lb_port
       value: "6443"
-    image: {{ .kubernetes.control_plane_endpoint.kube_vip.image }}
+    image: {{ .kubernetes.control_plane_endpoint.kube_vip.image.registry }}/{{ .kubernetes.control_plane_endpoint.kube_vip.image.repository }}:{{ .kubernetes.control_plane_endpoint.kube_vip.image.tag }}
     imagePullPolicy: IfNotPresent
     name: kube-vip
     resources: {}
diff --git a/builtin/capkk/roles/install/cni/tasks/calico.yaml b/builtin/capkk/roles/install/cni/tasks/calico.yaml
index b5a77b20..8c25beea 100644
--- a/builtin/capkk/roles/install/cni/tasks/calico.yaml
+++ b/builtin/capkk/roles/install/cni/tasks/calico.yaml
@@ -5,7 +5,7 @@
   register: calicoctl_install_version
   register_type: yaml
 - name: Install calicoctl
-  when: .calicoctl_install_version.stderr | ne ""
+  when: .calicoctl_install_version.stderr | empty | not
   block:
     - name: Sync calicoctl to remote
       copy:
diff --git a/builtin/capkk/roles/install/cni/tasks/cilium.yaml b/builtin/capkk/roles/install/cni/tasks/cilium.yaml
index 7336b742..74667b11 100644
--- a/builtin/capkk/roles/install/cni/tasks/cilium.yaml
+++ b/builtin/capkk/roles/install/cni/tasks/cilium.yaml
@@ -1,6 +1,6 @@
 ---
 - name: Sync cilium cli package
-  when: and .ciliumcli_version (ne .ciliumcli_version "")
+  when: .ciliumcli_version | empty | not
   copy:
     src: >-
       {{ .binary_dir }}/cni/cilium/ciliumcli-{{ .ciliumcli_version }}/{{ .item }}
diff --git a/builtin/capkk/roles/install/cri/defaults/main.yaml b/builtin/capkk/roles/install/cri/defaults/main.yaml
index 39b332a9..a948a2ca 100644
--- a/builtin/capkk/roles/install/cri/defaults/main.yaml
+++ b/builtin/capkk/roles/install/cri/defaults/main.yaml
@@ -1,7 +1,11 @@
 cri:
   # support: systemd, cgroupfs
   cgroup_driver: systemd
-  sandbox_image_tag: 3.5
+  sandbox_image: 
+    registry: >-
+      {{ .dockerio_registry }}
+    repository: kubesphere/pause
+    tag: 3.5
   # support: containerd,docker,crio
   # the endpoint of containerd
   cri_socket: >-
diff --git a/builtin/capkk/roles/install/cri/tasks/install_containerd.yaml b/builtin/capkk/roles/install/cri/tasks/install_containerd.yaml
index 592bd4b2..6be160e5 100644
--- a/builtin/capkk/roles/install/cri/tasks/install_containerd.yaml
+++ b/builtin/capkk/roles/install/cri/tasks/install_containerd.yaml
@@ -4,7 +4,7 @@
   command: runc --version
   register: runc_install_version
 - name: Sync runc binary to remote
-  when: or (.runc_install_version.stderr | ne "") (.runc_install_version.stdout | contains (printf "runc version %s\n" (.runc_version | default "" | trimPrefix "v" )) | not)
+  when: or (.runc_install_version.stderr | empty | not) (.runc_install_version.stdout | contains (printf "runc version %s\n" (.runc_version | default "" | trimPrefix "v" )) | not)
   copy:
     src: >-
       {{ .binary_dir }}/runc/{{ .runc_version }}/{{ .binary_type.stdout }}/runc.{{ .binary_type.stdout }}
@@ -16,7 +16,7 @@
   command: containerd --version
   register: containerd_install_version
 - name: Install containerd
-  when: or (.containerd_install_version.stderr | ne "") (.containerd_install_version.stdout | contains (printf " %s " .containerd_version) | not)
+  when: or (.containerd_install_version.stderr | empty | not) (.containerd_install_version.stdout | contains (printf " %s " .containerd_version) | not)
   block:
     - name: Sync containerd binary to remote
       copy:
diff --git a/builtin/capkk/roles/install/cri/tasks/install_crictl.yaml b/builtin/capkk/roles/install/cri/tasks/install_crictl.yaml
index 2cd184d6..792eb0be 100644
--- a/builtin/capkk/roles/install/cri/tasks/install_crictl.yaml
+++ b/builtin/capkk/roles/install/cri/tasks/install_crictl.yaml
@@ -5,7 +5,7 @@
   register: crictl_install_version
 
 - name: Install crictl
-  when: or (.crictl_install_version.stderr | ne "") (.crictl_install_version.stdout | ne (printf "crictl version %s" .crictl_version))
+  when: or (.crictl_install_version.stderr | empty | not) (.crictl_install_version.stdout | ne (printf "crictl version %s" .crictl_version))
   block:
     - name: Sync crictl binary to remote
       copy:
diff --git a/builtin/capkk/roles/install/cri/tasks/install_cridockerd.yaml b/builtin/capkk/roles/install/cri/tasks/install_cridockerd.yaml
index af89159d..08d5c9de 100644
--- a/builtin/capkk/roles/install/cri/tasks/install_cridockerd.yaml
+++ b/builtin/capkk/roles/install/cri/tasks/install_cridockerd.yaml
@@ -5,7 +5,7 @@
   register: cridockerd_install_version
 
 - name: Install cri-dockerd
-  when: or (.cridockerd_install_version.stderr | ne "") (.cridockerd_install_version.stdout | hasPrefix (printf "cri-dockerd %s " .cridockerd_version) | not)
+  when: or (.cridockerd_install_version.stderr | empty | not) (.cridockerd_install_version.stdout | hasPrefix (printf "cri-dockerd %s " .cridockerd_version) | not)
   block:
     - name: Sync cri-dockerd Binary to remote
       copy:
diff --git a/builtin/capkk/roles/install/cri/tasks/install_docker.yaml b/builtin/capkk/roles/install/cri/tasks/install_docker.yaml
index 9d196601..df1c737b 100644
--- a/builtin/capkk/roles/install/cri/tasks/install_docker.yaml
+++ b/builtin/capkk/roles/install/cri/tasks/install_docker.yaml
@@ -5,7 +5,7 @@
   register: docker_install_version
 
 - name: Install docker
-  when: or (.docker_install_version.stderr | ne "") (.docker_install_version.stdout | hasPrefix (printf "Docker version %s," .docker_version) | not)
+  when: or (.docker_install_version.stderr | empty | not) (.docker_install_version.stdout | hasPrefix (printf "Docker version %s," .docker_version) | not)
   block:
     - name: Sync docker binary to remote
       copy:
diff --git a/builtin/capkk/roles/install/cri/templates/containerd.config b/builtin/capkk/roles/install/cri/templates/containerd.config
index f24f34ac..60da9ad1 100644
--- a/builtin/capkk/roles/install/cri/templates/containerd.config
+++ b/builtin/capkk/roles/install/cri/templates/containerd.config
@@ -36,7 +36,7 @@ state = "/run/containerd"
 
 [plugins]
   [plugins."io.containerd.grpc.v1.cri"]
-    sandbox_image = "{{ .k8s_registry }}/pause:{{ .cri.sandbox_image_tag }}"
+    sandbox_image = "{{ .cri.sandbox_image.registry }}/{{ .cri.sandbox_image.repository }}:{{ .cri.sandbox_image.tag }}"
     [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
       runtime_type = "io.containerd.runc.v2"
       [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
@@ -48,7 +48,7 @@ state = "/run/containerd"
       conf_template = ""
     [plugins."io.containerd.grpc.v1.cri".registry]
       [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
-{{- if .cri.registry.mirrors | len | lt 0 }}
+{{- if .cri.registry.mirrors | empty | not }}
         [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
           endpoint = {{ .cri.registry.mirrors | toJson }}
 {{- end }}
@@ -56,7 +56,7 @@ state = "/run/containerd"
         [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ . }}"]
           endpoint = ["http://{{ . }}"]
 {{- end }}
-{{- if .cri.registry.auths | len | lt 0 }}
+{{- if .cri.registry.auths | empty | not }}
         [plugins."io.containerd.grpc.v1.cri".registry.configs]
   {{- range .cri.registry.auths }}
           [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .repo }}".auth]
diff --git a/builtin/capkk/roles/install/cri/templates/cri-dockerd.service b/builtin/capkk/roles/install/cri/templates/cri-dockerd.service
index 0a131b7d..244ef9a6 100644
--- a/builtin/capkk/roles/install/cri/templates/cri-dockerd.service
+++ b/builtin/capkk/roles/install/cri/templates/cri-dockerd.service
@@ -4,7 +4,7 @@ Documentation=https://docs.mirantis.com
 
 [Service]
 Type=notify
-ExecStart=/usr/local/bin/cri-dockerd --pod-infra-container-image "{{ .k8s_registry }}/pause:{{ .cri.sandbox_image_tag }}"
+ExecStart=/usr/local/bin/cri-dockerd --pod-infra-container-image "{{ .cri.sandbox_image.registry }}/{{ .cri.sandbox_image.repository }}:{{ .cri.sandbox_image.tag }}"
 ExecReload=/bin/kill -s HUP $MAINPID
 TimeoutSec=0
 RestartSec=2
diff --git a/builtin/capkk/roles/install/kubernetes/tasks/main.yaml b/builtin/capkk/roles/install/kubernetes/tasks/main.yaml
index c4184bba..ec6d11c8 100644
--- a/builtin/capkk/roles/install/kubernetes/tasks/main.yaml
+++ b/builtin/capkk/roles/install/kubernetes/tasks/main.yaml
@@ -4,7 +4,7 @@
   command: helm version
   register: helm_install_version
 - name: Install helm
-  when: or (.helm_install_version.stderr | ne "") (.helm_install_version.stdout | contains (printf "Version:\"%s\"" .helm_version) | not)
+  when: or (.helm_install_version.stderr | empty | not) (.helm_install_version.stdout | contains (printf "Version:\"%s\"" .helm_version) | not)
   block:
     - name: Sync helm to remote
       copy:
@@ -21,7 +21,7 @@
   command: kubeadm version -o short
   register: kubeadm_install_version
 - name: Install kubeadm
-  when: or (.kubeadm_install_version.stderr | ne "") (.kubeadm_install_version.stdout | ne .kube_version)
+  when: or (.kubeadm_install_version.stderr | empty | not) (.kubeadm_install_version.stdout | ne .kube_version)
   copy:
     src: >-
       {{ .binary_dir }}/kube/{{ .kube_version }}/{{ .binary_type.stdout }}/kubeadm
@@ -35,7 +35,7 @@
   register_type: yaml
 - name: Sync kubectl to remote
   when: |
-    or (.kubectl_install_version.stderr | ne "") ((get .kubectl_install_version.stdout "Server Version") | ne  .kube_version)
+    or (.kubectl_install_version.stderr | empty | not) ((get .kubectl_install_version.stdout "Server Version") | ne  .kube_version)
   copy:
     src: >-
       {{ .binary_dir }}/kube/{{ .kube_version }}/{{ .binary_type.stdout }}/kubectl
@@ -47,7 +47,7 @@
   command: kubelet --version
   register: kubelet_install_version
 - name: Install kubelet
-  when: or (.kubelet_install_version.stderr | ne "") (.kubelet_install_version.stdout | ne (printf "Kubernetes %s" .kube_version))
+  when: or (.kubelet_install_version.stderr | empty | not) (.kubelet_install_version.stdout | ne (printf "Kubernetes %s" .kube_version))
   block:
     - name: Sync kubelet to remote
       copy:
@@ -67,7 +67,7 @@
       command: systemctl daemon-reload && systemctl enable kubelet.service
 
 - name: Install cni plugins
-  when: and .cni_plugins_version (ne .cni_plugins_version "")
+  when: .cni_plugins_version | empty | not
   block:
     - name: Sync cni-plugin to remote
       copy:
diff --git a/builtin/capkk/roles/install/kubernetes/templates/kubeadm/kubelet.env b/builtin/capkk/roles/install/kubernetes/templates/kubeadm/kubelet.env
index 02f7d7e2..e8527cbf 100644
--- a/builtin/capkk/roles/install/kubernetes/templates/kubeadm/kubelet.env
+++ b/builtin/capkk/roles/install/kubernetes/templates/kubeadm/kubelet.env
@@ -9,9 +9,9 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
 EnvironmentFile=-/etc/default/kubelet
 {{- $internalIPv4 := .internal_ipv4 | default "" }}
 {{- $internalIPv6 := .internal_ipv6 | default "" }}
-{{- if ne $internalIPv4 "" }}
+{{- if $internalIPv4 | empty | not }}
 Environment="KUBELET_EXTRA_ARGS=--node-ip={{ $internalIPv4 }} --hostname-override={{ .hostname }} {{ range $k,$v := .kubernetes.kubelet.extra_args }}--{{ $k }} {{ $v }} {{ end }}"
-{{- else if ne $internalIPv6 "" }}
+{{- else if $internalIPv6 | empty | not }}
 Environment="KUBELET_EXTRA_ARGS=--node-ip={{ $internalIPv6 }} --hostname-override={{ .hostname }} {{ range $k,$v := .kubernetes.kubelet.extra_args }}--{{ $k }} {{ $v }} {{ end }}"
 {{- end }}
 ExecStart=
diff --git a/builtin/capkk/roles/install/storageclass/defaults/main.yaml b/builtin/capkk/roles/install/storageclass/defaults/main.yaml
index dbe90133..914fff1f 100644
--- a/builtin/capkk/roles/install/storageclass/defaults/main.yaml
+++ b/builtin/capkk/roles/install/storageclass/defaults/main.yaml
@@ -2,10 +2,16 @@ sc:
   local:
     enabled: true
     default: true
-    provisioner_image: >-
-      {{ .dockerio_registry }}/openebs/provisioner-localpv:3.3.0
-    linux_utils_image: >-
-      {{ .dockerio_registry }}/openebs/linux-utils:3.3.0
+    provisioner_image:
+      registry: >-
+        {{ .dockerio_registry }}
+      repository: openebs/provisioner-localpv
+      tag: 3.3.0
+    linux_utils_image:
+      registry: >-
+        {{ .dockerio_registry }}
+      repository: openebs/linux-utils
+      tag: 3.3.0
     path: /var/openebs/local
   nfs: # each k8s_cluster node should install nfs-utils
     enabled: false
diff --git a/builtin/capkk/roles/install/storageclass/templates/local-volume.yaml b/builtin/capkk/roles/install/storageclass/templates/local-volume.yaml
index 2b12dcb0..0505ed6d 100644
--- a/builtin/capkk/roles/install/storageclass/templates/local-volume.yaml
+++ b/builtin/capkk/roles/install/storageclass/templates/local-volume.yaml
@@ -100,7 +100,7 @@ spec:
       containers:
         - name: openebs-provisioner-hostpath
           imagePullPolicy: IfNotPresent
-          image: {{ .sc.local.provisioner_image }}
+          image: {{ .sc.local.provisioner_image.registry }}/{{ .sc.local.provisioner_image.repository }}:{{ .sc.local.provisioner_image.tag }}
           env:
             # OPENEBS_IO_K8S_MASTER enables openebs provisioner to connect to K8s
             # based on this address. This is ignored if empty.
@@ -131,7 +131,7 @@ spec:
             - name: OPENEBS_IO_INSTALLER_TYPE
               value: "openebs-operator-lite"
             - name: OPENEBS_IO_HELPER_IMAGE
-              value: "{{ .sc.local.linux_utils_image }}"
+              value: "{{ .sc.local.linux_utils_image.registry }}/{{ .sc.local.linux_utils_image.repository }}:{{ .sc.local.linux_utils_image.tag }}"
           # LEADER_ELECTION_ENABLED is used to enable/disable leader election. By default
           # leader election is enabled.
           #- name: LEADER_ELECTION_ENABLED
diff --git a/builtin/capkk/roles/precheck/env_check/tasks/cri.yaml b/builtin/capkk/roles/precheck/env_check/tasks/cri.yaml
index 2f273719..dcd7ce86 100644
--- a/builtin/capkk/roles/precheck/env_check/tasks/cri.yaml
+++ b/builtin/capkk/roles/precheck/env_check/tasks/cri.yaml
@@ -5,7 +5,7 @@
     fail_msg: >-
       the container manager:{{ .cri.container_manager }}, must be {{ .cluster_require.require_container_manager | toJson }}
   run_once: true
-  when: and .cri.container_manager (ne .cri.container_manager "")
+  when: cri.container_manager | empty | not
 
 - name: Ensure minimum containerd version
   assert:
@@ -14,5 +14,5 @@
       containerd_version is too low. Minimum version {{ .cluster_require.containerd_min_version_required }}
   run_once: true
   when:
-    - and .containerd_version (ne .containerd_version "")
+    - .containerd_version | empty | not
     - .cri.container_manager | eq "containerd"
diff --git a/builtin/capkk/roles/precheck/env_check/tasks/kubernetes.yaml b/builtin/capkk/roles/precheck/env_check/tasks/kubernetes.yaml
index d9632511..7b1fe5ef 100644
--- a/builtin/capkk/roles/precheck/env_check/tasks/kubernetes.yaml
+++ b/builtin/capkk/roles/precheck/env_check/tasks/kubernetes.yaml
@@ -1,6 +1,6 @@
 - name: Should defined internal_ipv4 or internal_ipv6
   assert:
-    that: or (and .internal_ipv4 (ne .internal_ipv4 "")) (and .internal_ipv6 (ne .internal_ipv6 ""))
+    that: or (.internal_ipv4 | empty | not) (.internal_ipv6 | empty | not))
     fail_msg: >-
       "internal_ipv4" and "internal_ipv6" cannot both be empty
 
@@ -28,7 +28,7 @@
     that: .kube_version | semverCompare (printf ">=%s" .cluster_require.kube_version_min_required)
     fail_msg: >-
       the current release of KubeKey only support newer version of Kubernetes than {{ .cluster_require.kube_version_min_required }} - You are trying to apply {{ .kube_version }}
-  when: and .kube_version (ne .kube_version "")
+  when: .kube_version | empty | not
 
 - name: Check if kubernetes installed
   when: .groups.k8s_cluster | default list | has .inventory_hostname
diff --git a/builtin/core/defaults/config/v1.23.15.yaml b/builtin/core/defaults/config/v1.23.15.yaml
index 9a60501b..672083f1 100644
--- a/builtin/core/defaults/config/v1.23.15.yaml
+++ b/builtin/core/defaults/config/v1.23.15.yaml
@@ -60,4 +60,5 @@ spec:
       extra_args:
         cluster-signing-duration: 87600h
   cri:
-    sandbox_image_tag: 3.5
\ No newline at end of file
+    sandbox_image:
+      tag: 3.5
\ No newline at end of file
diff --git a/builtin/core/defaults/config/v1.31.2.yaml b/builtin/core/defaults/config/v1.31.2.yaml
index c6b522dc..81341ead 100644
--- a/builtin/core/defaults/config/v1.31.2.yaml
+++ b/builtin/core/defaults/config/v1.31.2.yaml
@@ -60,6 +60,36 @@ spec:
       extra_args:
         cluster-signing-duration: 87600h
   cri:
-    sandbox_image_tag: 3.10
+    sandbox_image:
+      tag: 3.10
     # support: containerd,docker
-    container_manager: containerd
\ No newline at end of file
+    container_manager: containerd
+  
+  # image_manifests:
+  #  - docker.io/calico/apiserver:v3.29.2
+  #  - docker.io/calico/cni:v3.29.2
+  #  - docker.io/calico/csi:v3.29.2
+  #  - docker.io/calico/kube-controllers:v3.29.2
+  #  - docker.io/calico/node-driver-registrar:v3.29.2
+  #  - docker.io/calico/node:v3.29.2
+  #  - docker.io/calico/pod2daemon-flexvol:v3.29.2
+  #  - docker.io/kubesphere/k8s-dns-node-cache:1.22.20
+  #  - docker.io/openebs/provisioner-localpv:3.3.0
+  #  - docker.io/coredns/coredns:1.8.6
+  #  - docker.io/kubesphere/kube-apiserver:v1.31.2
+  #  - docker.io/kubesphere/kube-controller-manager:v1.31.2
+  #  - docker.io/kubesphere/kube-proxy:v1.31.2
+  #  - docker.io/kubesphere/kube-scheduler:v1.31.2
+  #  - docker.io/kubesphere/pause:3.10
+  #  - quay.io/tigera/operator:v1.36.5
+  #  - docker.io/kubesphere/pause:3.1
+  #  - docker.io/calico/ctl:v3.29.2
+  #  - docker.io/calico/typha:v3.29.2
+  #  - docker.io/calico/apiserver:v3.29.2
+  #  - docker.io/calico/kube-controllers:v3.29.2
+  #  - docker.io/calico/node:v3.29.2
+  #  - docker.io/calico/pod2daemon-flexvol:v3.29.2
+  #  - docker.io/calico/cni:v3.29.2
+  #  - docker.io/calico/node-driver-registrar:v3.29.2
+  #  - docker.io/calico/csi:v3.29.2
+
diff --git a/builtin/core/playbooks/delete_nodes.yaml b/builtin/core/playbooks/delete_nodes.yaml
index 73ea29ed..f49ddbc5 100644
--- a/builtin/core/playbooks/delete_nodes.yaml
+++ b/builtin/core/playbooks/delete_nodes.yaml
@@ -55,6 +55,7 @@
     - role: uninstall/cri
       when: 
         - .deleteCRI
+        - .groups.image_registry | default list | has .inventory_hostname | not
         - .delete_nodes | default list | has .inventory_hostname
   post_tasks:
     - name: delete localDNS file
diff --git a/builtin/core/playbooks/vars/create_cluster_kubernetes.yaml b/builtin/core/playbooks/vars/create_cluster_kubernetes.yaml
index 793082db..32cfe109 100644
--- a/builtin/core/playbooks/vars/create_cluster_kubernetes.yaml
+++ b/builtin/core/playbooks/vars/create_cluster_kubernetes.yaml
@@ -4,7 +4,7 @@ kubernetes:
   kube_network_plugin: calico
   # the image repository of kubernetes.
   image_repository: >-
-    {{ .k8s_registry }}
+    {{ .dockerio_registry }}/kubesphere
   # memory size for each kube_worker node.(unit kB)
   # should be greater than or equal to minimal_node_memory_mb.
   minimal_node_memory_mb: 10
@@ -22,11 +22,16 @@ kubernetes:
     service_cidr: 10.233.0.0/18
     dns_domain: cluster.local
     dns_image: 
+      registry: >-
+        {{ .dockerio_registry }}
       repository: >-
-        {{ .k8s_registry }}/coredns
-      tag: v1.8.6
-    dns_cache_image: >-
-      {{ .dockerio_registry }}/kubesphere/k8s-dns-node-cache:1.22.20
+        coredns
+      tag: 1.8.6
+    dns_cache_image: 
+      registry: >-
+        {{ .dockerio_registry }}
+      repository: kubesphere/k8s-dns-node-cache
+      tag: 1.22.20
     dns_service_ip: >-
       {{ .kubernetes.networking.service_cidr | ipInCIDR 2 }}
   apiserver:
@@ -171,21 +176,29 @@ kubernetes:
       # address: 
       # support ARP or BGP
       mode: ARP
-      image: >-
-        {{ .dockerio_registry }}/plndr/kube-vip:v0.7.2
+      image: 
+        registry: >-
+          {{ .dockerio_registry }}
+        repository: plndr/kube-vip
+        tag: v0.7.2
     haproxy:
       # the ip address in node network interface: "lo" 
       address: 127.0.0.1
       health_port: 8081
-      image: >-
-        {{ .dockerio_registry }}/library/haproxy:2.9.6-alpine
+      image: 
+        registry: >-
+          {{ .dockerio_registry }}
+        repository: library/haproxy
+        tag: 2.9.6-alpine
   etcd:
     # It is possible to deploy etcd with three methods.
     # external: Deploy etcd cluster with external etcd cluster.
     # internal: Deploy etcd cluster by static pod.
     deployment_type: external
     image: 
-      repository: "{{ .k8s_registry }}"
+      registry: >-
+        {{ .dockerio_registry }}
+      repository: kubesphere/etcd
       tag: 3.5.0
   custom_label: {}
   # if auto renew kubernetes certs
diff --git a/builtin/core/roles/init/init-artifact/defaults/main.yaml b/builtin/core/roles/init/init-artifact/defaults/main.yaml
index 8c957d85..0572a60e 100644
--- a/builtin/core/roles/init/init-artifact/defaults/main.yaml
+++ b/builtin/core/roles/init/init-artifact/defaults/main.yaml
@@ -220,6 +220,4 @@ artifact:
     kubeovn: https://kubeovn.github.io/kube-ovn/kube-ovn-{{ .kubeovn_version }}.tgz
     hybridnet: https://github.com/alibaba/hybridnet/releases/download/helm-chart-{{ .hybridnet_version }}/hybridnet-{{ .hybridnet_version }}.tgz
     nfs_provisioner: https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/releases/download/nfs-subdir-external-provisioner-4.0.18/nfs-subdir-external-provisioner-{{ .nfs_provisioner_version }}.tgz
-  images:
-    auth: []
-    list: []
+  download_image: true
diff --git a/builtin/core/roles/install/cni/defaults/main.yaml b/builtin/core/roles/install/cni/defaults/main.yaml
index c2558d39..231539bb 100644
--- a/builtin/core/roles/install/cni/defaults/main.yaml
+++ b/builtin/core/roles/install/cni/defaults/main.yaml
@@ -5,7 +5,11 @@ cni:
   multus: 
     # if install multus thick plugins. 
     enabled: false
-    image: "{{ .dockerio_registry }}/kubesphere/multus-cni:v3.8"
+    image: 
+      registry:  >-
+        {{ .dockerio_registry }}
+      repository: kubesphere/multus-cni
+      tag: v3.8
   # In Kubernetes, the Pod CIDR supports both IPv4 and IPv6 configurations. It can be specified as follows:
   # "Single-stack IPv4": the pod_cidr value format "ipv4"
   # "Single-stack IPv6": the pod_cidr value format "ipv6"
@@ -35,6 +39,10 @@ cni:
   calico:
     values: |
       # calico helm values
+      tigeraOperator:
+        registry: {{ .quayio_registry }}
+      calicoctl:
+        image: {{ .dockerio_registry }}/calico/ctl
       installation:
         registry: {{ .dockerio_registry }}
         calicoNetwork:
diff --git a/builtin/core/roles/install/cni/templates/multus.yaml b/builtin/core/roles/install/cni/templates/multus.yaml
index 5f666e50..d657c9a0 100644
--- a/builtin/core/roles/install/cni/templates/multus.yaml
+++ b/builtin/core/roles/install/cni/templates/multus.yaml
@@ -169,7 +169,7 @@ spec:
       serviceAccountName: multus
       containers:
         - name: kube-multus
-          image: {{ .cni.multus.image }}
+          image: {{ .cni.multus.image.registry }}/{{ .cni.multus.image.repository }}:{{ .cni.multus.image.tag }}
           command: ["/entrypoint.sh"]
           args:
             - "--multus-conf-file=auto"
diff --git a/builtin/core/roles/install/cri/defaults/main.yaml b/builtin/core/roles/install/cri/defaults/main.yaml
index d4f29ee2..bd4380a7 100644
--- a/builtin/core/roles/install/cri/defaults/main.yaml
+++ b/builtin/core/roles/install/cri/defaults/main.yaml
@@ -1,8 +1,11 @@
 cri:
   # support: systemd, cgroupfs
   cgroup_driver: systemd
-  sandbox_image: >-
-    {{ .k8s_registry }}/pause:3.5
+  sandbox_image: 
+    registry: >-
+      {{ .dockerio_registry }}
+    repository: kubesphere/pause
+    tag: 3.5
   # support: containerd,docker
   # container_manager: docker
   # the endpoint of containerd
@@ -25,14 +28,14 @@ image_registry:
   #  ha_vip: 192.168.122.59
   auth:
     registry: >-
-      {{- if and .image_registry.ha_vip (ne .image_registry.ha_vip "") -}}
+      {{- if .image_registry.ha_vip | empty | not -}}
         {{ .image_registry.ha_vip }}
       {{- else if .groups.image_registry | default list | len | lt 0 -}}
         {{- $internalIPv4 := index .hostvars (.groups.image_registry | default list | first) "internal_ipv4" | default "" -}}
         {{- $internalIPv6 := index .hostvars (.groups.image_registry | default list | first) "internal_ipv6" | default "" -}}
-        {{- if ne $internalIPv4 "" -}}
+        {{- if  $internalIPv4 | empty | not -}}
           {{ $internalIPv4 }}
-        {{- else if ne $internalIPv6 "" -}}
+        {{- else if $internalIPv6 | empty | not -}}
           {{ $internalIPv6 }}
         {{- end -}}
       {{- end -}}
diff --git a/builtin/core/roles/install/cri/templates/containerd.config b/builtin/core/roles/install/cri/templates/containerd.config
index 78312a82..ec043d0d 100644
--- a/builtin/core/roles/install/cri/templates/containerd.config
+++ b/builtin/core/roles/install/cri/templates/containerd.config
@@ -36,7 +36,7 @@ state = "/run/containerd"
 
 [plugins]
   [plugins."io.containerd.grpc.v1.cri"]
-    sandbox_image = "{{ .k8s_registry }}/pause:{{ .cri.sandbox_image_tag }}"
+    sandbox_image = "{{ .cri.sandbox_image.registry }}/{{ .cri.sandbox_image.repository }}:{{ .cri.sandbox_image.tag }}"
     [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
       runtime_type = "io.containerd.runc.v2"
       [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
@@ -48,7 +48,7 @@ state = "/run/containerd"
       conf_template = ""
     [plugins."io.containerd.grpc.v1.cri".registry]
       [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
-{{- if .cri.registry.mirrors | len | lt 0 }}
+{{- if .cri.registry.mirrors | empty | not }}
         [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
           endpoint = {{ .cri.registry.mirrors | toJson }}
 {{- end }}
@@ -56,10 +56,10 @@ state = "/run/containerd"
         [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ . }}"]
           endpoint = ["http://{{ . }}"]
 {{- end }}
-{{- if or (.cri.registry.auths | len | lt 0) (.groups.image_registry | default list | len | lt 0) }}
+{{- if or (.cri.registry.auths | empty | not) (.groups.image_registry | default list | empty | not) }}
         [plugins."io.containerd.grpc.v1.cri".registry.configs]
 {{- end }}
-{{- if .groups.image_registry | default list | len | lt 0 }}
+{{- if .groups.image_registry | default list | empty | not }}
           [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .image_registry.auth.registry }}".auth]
             username = "{{ .image_registry.auth.username }}"
             password = "{{ .image_registry.auth.password }}"
@@ -68,7 +68,7 @@ state = "/run/containerd"
             cert_file = "/etc/containerd/certs.d/{{ .image_registry.auth.registry }}/server.crt"
             key_file = "/etc/containerd/certs.d/{{ .image_registry.auth.registry }}/server.key"
 {{- end }}
-{{- if .cri.registry.auths | len | lt 0 }}
+{{- if .cri.registry.auths | empty | not }}
   {{- range .cri.registry.auths }}
           [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .repo }}".auth]
             username = "{{ .username }}"
diff --git a/builtin/core/roles/install/cri/templates/cri-dockerd.service b/builtin/core/roles/install/cri/templates/cri-dockerd.service
index 0a131b7d..244ef9a6 100644
--- a/builtin/core/roles/install/cri/templates/cri-dockerd.service
+++ b/builtin/core/roles/install/cri/templates/cri-dockerd.service
@@ -4,7 +4,7 @@ Documentation=https://docs.mirantis.com
 
 [Service]
 Type=notify
-ExecStart=/usr/local/bin/cri-dockerd --pod-infra-container-image "{{ .k8s_registry }}/pause:{{ .cri.sandbox_image_tag }}"
+ExecStart=/usr/local/bin/cri-dockerd --pod-infra-container-image "{{ .cri.sandbox_image.registry }}/{{ .cri.sandbox_image.repository }}:{{ .cri.sandbox_image.tag }}"
 ExecReload=/bin/kill -s HUP $MAINPID
 TimeoutSec=0
 RestartSec=2
diff --git a/builtin/core/roles/install/image-registry/defaults/main.yaml b/builtin/core/roles/install/image-registry/defaults/main.yaml
index 8f8b47b7..3f26e774 100644
--- a/builtin/core/roles/install/image-registry/defaults/main.yaml
+++ b/builtin/core/roles/install/image-registry/defaults/main.yaml
@@ -1,19 +1,18 @@
 image_registry:
   #  ha_vip: 192.168.122.59
-  namespace_override: ""
   # which store images data which will push to registry.
   images_dir: >-
     {{ .tmp_dir }}/images/
   auth:
     registry: >-
-      {{- if and .image_registry.ha_vip (ne .image_registry.ha_vip "") -}}
+      {{- if .image_registry.ha_vip | empty | not -}}
         {{ .image_registry.ha_vip }}
-      {{- else if .groups.image_registry | default list | len | lt 0 -}}
+      {{- else if .groups.image_registry | default list | empty | not -}}
         {{- $internalIPv4 := index .hostvars (.groups.image_registry | default list | first) "internal_ipv4" | default "" -}}
         {{- $internalIPv6 := index .hostvars (.groups.image_registry | default list | first) "internal_ipv6" | default "" -}}
-        {{- if ne $internalIPv4 "" -}}
+        {{- if $internalIPv4 | empty | not -}}
           {{ $internalIPv4 }}
-        {{- else if ne $internalIPv6 "" -}}
+        {{- else if $internalIPv6 | empty | not -}}
           {{ $internalIPv6 }}
         {{- end -}}
       {{- end -}}
@@ -22,6 +21,8 @@ image_registry:
   # registry type. support: harbor, registry
   type: harbor
   # Virtual IP address for repository High Availability. the Virtual IP address should be available.
+  harbor:
+    data_dir: /opt/harbor/data
   registry:
     version: 2
     config:
diff --git a/builtin/core/roles/install/image-registry/tasks/load_images.yaml b/builtin/core/roles/install/image-registry/tasks/load_images.yaml
index 6f9639b5..a07cf4ce 100644
--- a/builtin/core/roles/install/image-registry/tasks/load_images.yaml
+++ b/builtin/core/roles/install/image-registry/tasks/load_images.yaml
@@ -10,35 +10,30 @@
 - name: Create harbor project for each image
   tags: ["only_image"]
   command: |
-    {{- if .image_registry.namespace_override | eq "" }}
-    for dir in {{ .image_registry.images_dir }}*; do
-      if [ ! -d "$dir" ]; then
-        # only deal with directories
+    # Iterate through first-level subdirectories in images_dir (skip blobs)
+    for registry_dir in {{ .image_registry.images_dir }}*; do
+      if [ ! -d "$registry_dir" ] || [ "$(basename "$registry_dir")" = "blobs" ]; then
         continue
       fi
-    
-      project=${dir##*/}
-    
-      if [[ "$project" == "blobs" ]]; then
-        # skip blobs dir
-        continue
-      fi
-  
-      # if project is not exist, create it
-      resp=$(curl -u "{{ .image_registry.auth.username }}:{{ .image_registry.auth.password }}" -k -X GET  "https://{{ .image_registry.auth.registry }}/api/v2.0/projects/${project}")
-      if echo "$resp" | grep -q '"code":"NOT_FOUND"'; then
-          # create project
-          curl -u "{{ .image_registry.auth.username }}:{{ .image_registry.auth.password }}" -k -X POST -H "Content-Type: application/json" "https://{{ .image_registry.auth.registry }}/api/v2.0/projects" -d "{ \"project_name\": \"${project}\", \"public\": true}"
-      fi    
+      
+      # Iterate through second-level subdirectories in registry_dir
+      for project_dir in "$registry_dir"/*; do
+        if [ ! -d "$project_dir" ]; then
+          continue
+        fi
+        
+        project=$(basename "$project_dir")
+        
+        # Check if project exists, create if not
+        resp=$(curl -u "{{ .image_registry.auth.username }}:{{ .image_registry.auth.password }}" -k -X GET "https://{{ .image_registry.auth.registry }}/api/v2.0/projects/${project}")
+        if echo "$resp" | grep -q '"code":"NOT_FOUND"'; then
+          curl -u "{{ .image_registry.auth.username }}:{{ .image_registry.auth.password }}" -k -X POST \
+            -H "Content-Type: application/json" \
+            "https://{{ .image_registry.auth.registry }}/api/v2.0/projects" \
+            -d "{ \"project_name\": \"${project}\", \"public\": true}"
+        fi
+      done
     done
-    {{- else }}
-    # if project is not exist, create it
-    resp=$(curl -u "{{ .image_registry.auth.username }}:{{ .image_registry.auth.password }}" -k -X GET "https://{{ .image_registry.auth.registry }}/api/v2.0/projects/{{ .image_registry.namespace_override }}")
-    if echo "$resp" | grep -q '"code":"NOT_FOUND"'; then
-        # create project
-        curl -u "{{ .image_registry.auth.username }}:{{ .image_registry.auth.password }}" -k -X POST -H "Content-Type: application/json" "https://{{ .image_registry.auth.registry }}/api/v2.0/projects" -d "{ \"project_name\": \"{{ .image_registry.namespace_override }}\", \"public\": true}"
-    fi
-    {{- end }}
   when: .image_registry.type | eq "harbor"
 
 - name: Sync images package to harbor
diff --git a/builtin/core/roles/install/image-registry/templates/harbor.config b/builtin/core/roles/install/image-registry/templates/harbor.config
index 621394b6..2bfa82f4 100644
--- a/builtin/core/roles/install/image-registry/templates/harbor.config
+++ b/builtin/core/roles/install/image-registry/templates/harbor.config
@@ -53,7 +53,7 @@ database:
   conn_max_idle_time: 0
 
 # The default data volume
-data_volume: /data
+data_volume: {{ .image_registry.harbor.data_dir }}
 
 # Harbor Storage settings by default is using /data dir on local filesystem
 # Uncomment storage_service setting If you want to using external storage
diff --git a/builtin/core/roles/install/storageclass/defaults/main.yaml b/builtin/core/roles/install/storageclass/defaults/main.yaml
index dbe90133..08bd279a 100644
--- a/builtin/core/roles/install/storageclass/defaults/main.yaml
+++ b/builtin/core/roles/install/storageclass/defaults/main.yaml
@@ -2,10 +2,16 @@ sc:
   local:
     enabled: true
     default: true
-    provisioner_image: >-
-      {{ .dockerio_registry }}/openebs/provisioner-localpv:3.3.0
-    linux_utils_image: >-
-      {{ .dockerio_registry }}/openebs/linux-utils:3.3.0
+    provisioner_image: 
+      registry: >-
+        {{ .dockerio_registry }}
+      repository: openebs/provisioner-localpv
+      tag: 3.3.0
+    linux_utils_image:
+      registry: >-
+        {{ .dockerio_registry }}
+      repository: openebs/linux-utils
+      tag: 3.3.0
     path: /var/openebs/local
   nfs: # each k8s_cluster node should install nfs-utils
     enabled: false
diff --git a/builtin/core/roles/install/storageclass/templates/local-volume.yaml b/builtin/core/roles/install/storageclass/templates/local-volume.yaml
index 2b12dcb0..0505ed6d 100644
--- a/builtin/core/roles/install/storageclass/templates/local-volume.yaml
+++ b/builtin/core/roles/install/storageclass/templates/local-volume.yaml
@@ -100,7 +100,7 @@ spec:
       containers:
         - name: openebs-provisioner-hostpath
           imagePullPolicy: IfNotPresent
-          image: {{ .sc.local.provisioner_image }}
+          image: {{ .sc.local.provisioner_image.registry }}/{{ .sc.local.provisioner_image.repository }}:{{ .sc.local.provisioner_image.tag }}
           env:
             # OPENEBS_IO_K8S_MASTER enables openebs provisioner to connect to K8s
             # based on this address. This is ignored if empty.
@@ -131,7 +131,7 @@ spec:
             - name: OPENEBS_IO_INSTALLER_TYPE
               value: "openebs-operator-lite"
             - name: OPENEBS_IO_HELPER_IMAGE
-              value: "{{ .sc.local.linux_utils_image }}"
+              value: "{{ .sc.local.linux_utils_image.registry }}/{{ .sc.local.linux_utils_image.repository }}:{{ .sc.local.linux_utils_image.tag }}"
           # LEADER_ELECTION_ENABLED is used to enable/disable leader election. By default
           # leader election is enabled.
           #- name: LEADER_ELECTION_ENABLED
diff --git a/builtin/core/roles/kubernetes/init-kubernetes/templates/dns/nodelocaldns.yaml b/builtin/core/roles/kubernetes/init-kubernetes/templates/dns/nodelocaldns.yaml
index c205438a..92136d55 100644
--- a/builtin/core/roles/kubernetes/init-kubernetes/templates/dns/nodelocaldns.yaml
+++ b/builtin/core/roles/kubernetes/init-kubernetes/templates/dns/nodelocaldns.yaml
@@ -43,7 +43,7 @@ spec:
         operator: "Exists"
       containers:
       - name: node-cache
-        image: {{ .kubernetes.networking.dns_cache_image }}
+        image: {{ .kubernetes.networking.dns_cache_image.registry }}/{{ .kubernetes.networking.dns_cache_image.repository }}:{{ .kubernetes.networking.dns_cache_image.tag }}
         resources:
           limits:
             memory: 200Mi
@@ -132,7 +132,7 @@ data:
 
     {{- range .forward }}
       forward {{ .from }} {{ .to | join " " }} {
-      {{- if .except | len | lt 0 }}
+      {{- if .except | empty | not }}
         except {{ .except | join " " }}
       {{- end }}
       {{- if .force_tcp }}
@@ -161,7 +161,7 @@ data:
       }
     {{- end }}
 
-    {{- if $.kubernetes.coredns.dns_etc_hosts | len | lt 0 }}
+    {{- if $.kubernetes.coredns.dns_etc_hosts | empty | not }}
       hosts /etc/coredns/hosts {
         fallthrough
       }
@@ -214,14 +214,14 @@ data:
       bind 169.254.25.10
       forward . /etc/resolv.conf
       prometheus :9253
-    {{- if .kubernetes.coredns.dns_etc_hosts | len | lt 0 }}
+    {{- if .kubernetes.coredns.dns_etc_hosts | empty | not }}
       hosts /etc/coredns/hosts {
         fallthrough
       }
     {{- end }}
     }
 
-{{- if .kubernetes.coredns.dns_etc_hosts | len | lt 0 }}
+{{- if .kubernetes.coredns.dns_etc_hosts | empty | not }}
   hosts: |
   {{- range .kubernetes.coredns.dns_etc_hosts }}
     {{ . }}
diff --git a/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta2 b/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta2
index 26f2cb90..70c3cae6 100644
--- a/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta2
+++ b/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta2
@@ -6,7 +6,7 @@ kind: ClusterConfiguration
 etcd:
 {{- if .kubernetes.etcd.deployment_type | eq "internal" }}
   local:
-    imageRepository: {{ .kubernetes.etcd.image.repository }}
+    imageRepository: {{ .kubernetes.etcd.image.registry }}
     imageTag: {{ .kubernetes.etcd.image.tag }}
     serverCertSANs:
     {{- range .groups.etcd | default list }}
@@ -24,7 +24,7 @@ etcd:
 {{- end }}
 dns:
   type: CoreDNS
-  imageRepository: {{ .kubernetes.networking.dns_image.repository }}
+  imageRepository: {{ .kubernetes.networking.dns_image.registry }}/{{ .kubernetes.networking.dns_image.repository }}
   imageTag: {{ .kubernetes.networking.dns_image.tag }}
 imageRepository: {{ .kubernetes.image_repository }}
 kubernetesVersion: {{ .kube_version }}
@@ -37,9 +37,9 @@ networking:
   serviceSubnet: {{ .kubernetes.networking.service_cidr }}
 apiServer:
   extraArgs:
-{{- if ne $internalIPv4 "" }}
+{{- if $internalIPv4 | empty | not }}
     bind-address: 0.0.0.0
-{{- else if ne $internalIPv6 "" }}
+{{- else if $internalIPv6 | empty | not }}
     bind-address: ::
 {{- end }}
 {{- if .security_enhancement }}
@@ -58,7 +58,7 @@ apiServer:
     audit-policy-file: /etc/kubernetes/audit/policy.yaml
     audit-webhook-config-file: /etc/kubernetes/audit/webhook.yaml
 {{- end }}
-{{- if .kubernetes.apiserver.extra_args }}
+{{- if .kubernetes.apiserver.extra_args | empty | not }}
 {{ .kubernetes.apiserver.extra_args | toYaml | indent 4 }}
 {{- end }}
   certSANs:
@@ -78,10 +78,10 @@ apiServer:
     - {{ index $.hostvars . "hostname" }}.{{ $.kubernetes.cluster_name }}.{{ $.kubernetes.networking.dns_domain }}
     {{- $internalIPv4 := index $.hostvars . "internal_ipv4" | default "" }}
     {{- $internalIPv6 := index $.hostvars . "internal_ipv6" | default "" }}
-    {{- if ne $internalIPv4 "" }}
+    {{- if $internalIPv4 | empty | not }}
     - {{ $internalIPv4 }}
     {{- end }}
-    {{- if ne $internalIPv6 "" }}
+    {{- if $internalIPv6 | empty | not }}
     - {{ $internalIPv6 }}
     {{- end }}
   {{- end }}
@@ -104,18 +104,18 @@ controllerManager:
     node-cidr-mask-size-ipv6: "{{ .kubernetes.networking.ipv6_mask_size }}"
 {{- end }}
 {{- if .security_enhancement }}
-  {{- if ne $internalIPv4 "" }}
+  {{- if $internalIPv4 | empty | not }}
     bind-address: 127.0.0.1
-  {{- else if ne $internalIPv6 "" }}
+  {{- else if $internalIPv6 | empty | not }}
     bind-address: ::1
   {{- end }}
     profiling: false
     terminated-pod-gc-threshold: 50
     use-service-account-credentials: true
 {{- else }}
-  {{- if ne $internalIPv4 "" }}
+  {{- if $internalIPv4 | empty | not }}
     bind-address: 0.0.0.0
-  {{- else if ne $internalIPv6 "" }}
+  {{- else if $internalIPv6 | empty | not }}
     bind-address: ::
   {{- end }}
 {{- end }}
@@ -130,16 +130,16 @@ controllerManager:
 scheduler:
   extraArgs:
 {{- if .security_enhancement }}
-  {{- if ne $internalIPv4 "" }}
+  {{- if $internalIPv4 | empty | not }}
     bind-address: 127.0.0.1
-  {{- else if ne $internalIPv6 "" }}
+  {{- else if $internalIPv6 | empty | not }}
     bind-address: ::1
   {{- end }}
     profiling: false
 {{- else }}
-  {{- if ne $internalIPv4 "" }}
+  {{- if $internalIPv4 | empty | not }}
     bind-address: 0.0.0.0
-  {{- else if ne $internalIPv6 "" }}
+  {{- else if $internalIPv6 | empty | not }}
     bind-address: ::
   {{- end }}
 {{- end }}
@@ -150,9 +150,9 @@ scheduler:
 apiVersion: kubeadm.k8s.io/v1beta2
 kind: InitConfiguration
 localAPIEndpoint:
-{{- if ne $internalIPv4 "" }}
+{{- if $internalIPv4 | empty | not }}
   advertiseAddress: {{ $internalIPv4 }}
-{{- else if ne $internalIPv6 "" }}
+{{- else if $internalIPv6 | empty | not }}
   advertiseAddress: {{ $internalIPv6 }}
 {{- end }}
   bindPort: {{ .kubernetes.apiserver.port }}
@@ -160,14 +160,14 @@ nodeRegistration:
   criSocket: {{ .cri.cri_socket }}
   kubeletExtraArgs:
     cgroup-driver: {{ .cri.cgroup_driver }}
-    pod-infra-container-image: "{{ .k8s_registry }}/pause:{{ .cri.sandbox_image_tag }}"
+    pod-infra-container-image: "{{ .cri.sandbox_image.registry }}/{{ .cri.sandbox_image.repository }}:{{ .cri.sandbox_image.tag }}"
 
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
 clusterCIDR: {{ .kubernetes.networking.pod_cidr }}
 mode: {{ .kubernetes.kube_proxy.mode }}
-{{- if .kubernetes.kube_proxy.config }}
+{{- if .kubernetes.kube_proxy.config | empty | not }}
 {{ .kubernetes.kube_proxy.config | toYaml }}
 {{- end }}
 ---
@@ -205,13 +205,13 @@ tlsCipherSuites:
   - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
   - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
 {{- end }}
-{{- if .kubernetes.kubelet.feature_gates }} 
+{{- if .kubernetes.kubelet.feature_gates | empty | not }} 
 featureGates:
 {{ .kubernetes.kubelet.feature_gates | toYaml | indent 2 }}
 {{- end }}
 cgroupDriver: {{ .cri.cgroup_driver }}
 containerLogMaxSize: {{ .kubernetes.kubelet.container_log_max_size }}
 containerLogMaxFiles: {{ .kubernetes.kubelet.container_log_max_files }}
-{{- if .kubernetes.kubelet.extra_args }}
+{{- if .kubernetes.kubelet.extra_args | empty | not }}
 {{ .kubernetes.kubelet.extra_args | toYaml }}
 {{- end }}
diff --git a/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta3 b/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta3
index e0797b20..73c144f2 100644
--- a/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta3
+++ b/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta3
@@ -6,7 +6,7 @@ kind: ClusterConfiguration
 etcd:
 {{- if .kubernetes.etcd.deployment_type | eq "internal" }}
   local:
-    imageRepository: {{ .kubernetes.etcd.image.repository }}
+    imageRepository: {{ .kubernetes.etcd.image.registry }}
     imageTag: {{ .kubernetes.etcd.image.tag }}
     serverCertSANs:
     {{- range .groups.etcd | default list }}
@@ -23,7 +23,7 @@ etcd:
     keyFile: /etc/kubernetes/pki/etcd/client.key
 {{- end }}
 dns:
-  imageRepository: {{ .kubernetes.networking.dns_image.repository }}
+  imageRepository: {{ .kubernetes.networking.dns_image.registry }}/{{ .kubernetes.networking.dns_image.repository }}
   imageTag: {{ .kubernetes.networking.dns_image.tag }}
 imageRepository: {{ .kubernetes.image_repository }}
 kubernetesVersion: {{ .kube_version }}
@@ -36,9 +36,9 @@ networking:
   serviceSubnet: {{ .kubernetes.networking.service_cidr }}
 apiServer:
   extraArgs:
-{{- if ne $internalIPv4 "" }}
+{{- if $internalIPv4 | empty | not }}
     bind-address: 0.0.0.0
-{{- else if ne $internalIPv6 "" }}
+{{- else if $internalIPv6 | empty | not }}
     bind-address: ::
 {{- end }}
 {{- if .security_enhancement }}
@@ -57,7 +57,7 @@ apiServer:
     audit-policy-file: /etc/kubernetes/audit/policy.yaml
     audit-webhook-config-file: /etc/kubernetes/audit/webhook.yaml
 {{- end }}
-{{- if .kubernetes.apiserver.extra_args }}
+{{- if .kubernetes.apiserver.extra_args | empty | not }}
 {{ .kubernetes.apiserver.extra_args | toYaml | indent 4 }}
 {{- end }}
   certSANs:
@@ -77,10 +77,10 @@ apiServer:
     - {{ index $.hostvars . "hostname" }}.{{ $.kubernetes.cluster_name }}.{{ $.kubernetes.networking.dns_domain }}
     {{- $internalIPv4 := index $.hostvars . "internal_ipv4" | default "" }}
     {{- $internalIPv6 := index $.hostvars . "internal_ipv6" | default "" }}
-    {{- if ne $internalIPv4 "" }}
+    {{- if $internalIPv4 | empty | not }}
     - {{ $internalIPv4 }}
     {{- end }}
-    {{- if ne $internalIPv6 "" }}
+    {{- if $internalIPv6 | empty | not }}
     - {{ $internalIPv6 }}
     {{- end }}
   {{- end }}
@@ -103,22 +103,22 @@ controllerManager:
     node-cidr-mask-size-ipv6: "{{ .kubernetes.networking.ipv6_mask_size }}"
 {{- end }}
 {{- if .security_enhancement }}
-  {{- if ne $internalIPv4 "" }}
+  {{- if $internalIPv4 | empty | not }}
     bind-address: 127.0.0.1
-  {{- else if ne $internalIPv6 "" }}
+  {{- else if $internalIPv6 | empty | not }}
     bind-address: ::1
   {{- end }}
     profiling: false
     terminated-pod-gc-threshold: 50
     use-service-account-credentials: true
 {{- else }}
-  {{- if ne $internalIPv4 "" }}
+  {{- if $internalIPv4 | empty | not }}
     bind-address: 0.0.0.0
-  {{- else if ne $internalIPv6 "" }}
+  {{- else if $internalIPv6 | empty | not }}
     bind-address: ::
   {{- end }}
 {{- end }}
-{{- if .kubernetes.controller_manager.extra_args }}
+{{- if .kubernetes.controller_manager.extra_args | empty | not }}
 {{ .kubernetes.controller_manager.extra_args | toYaml | indent 4 }}
 {{- end }}
   extraVolumes:
@@ -129,29 +129,29 @@ controllerManager:
 scheduler:
   extraArgs:
 {{- if .security_enhancement }}
-  {{- if ne $internalIPv4 "" }}
+  {{- if $internalIPv4 | empty | not }}
     bind-address: 127.0.0.1
-  {{- else if ne $internalIPv6 "" }}
+  {{- else if $internalIPv6 | empty | not }}
     bind-address: ::1
   {{- end }}
     profiling: false
 {{- else }}
-  {{- if ne $internalIPv4 "" }}
+  {{- if $internalIPv4 | empty | not }}
     bind-address: 0.0.0.0
-  {{- else if ne $internalIPv6 "" }}
+  {{- else if $internalIPv6 | empty | not }}
     bind-address: ::
   {{- end }}
 {{- end }}
-{{- if .kubernetes.scheduler.extra_args }}
+{{- if .kubernetes.scheduler.extra_args | empty | not }}
 {{ .kubernetes.scheduler.extra_args | toYaml | indent 4 }}
 {{- end }}
 ---
 apiVersion: kubeadm.k8s.io/v1beta3
 kind: InitConfiguration
 localAPIEndpoint:
-{{- if ne $internalIPv4 "" }}
+{{- if $internalIPv4 | empty | not }}
   advertiseAddress: {{ $internalIPv4 }}
-{{- else if ne $internalIPv6 "" }}
+{{- else if $internalIPv6 | empty | not }}
   advertiseAddress: {{ $internalIPv6 }}
 {{- end }}
   bindPort: {{ .kubernetes.apiserver.port }}
@@ -159,7 +159,7 @@ nodeRegistration:
   criSocket: {{ .cri.cri_socket }}
   kubeletExtraArgs:
     cgroup-driver: {{ .cri.cgroup_driver }}
-    pod-infra-container-image: "{{ .k8s_registry }}/pause:{{ .cri.sandbox_image_tag }}"
+    pod-infra-container-image: "{{ .cri.sandbox_image.registry }}/{{ .cri.sandbox_image.repository }}:{{ .cri.sandbox_image.tag }}"
 
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
@@ -204,13 +204,13 @@ tlsCipherSuites:
   - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
   - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
 {{- end }}
-{{- if .kubernetes.kubelet.feature_gates }} 
+{{- if .kubernetes.kubelet.feature_gates | empty | not }} 
 featureGates:
 {{ .kubernetes.kubelet.feature_gates | toYaml | indent 2 }}
 {{- end }}
 cgroupDriver: {{ .cri.cgroup_driver }}
 containerLogMaxSize: {{ .kubernetes.kubelet.container_log_max_size }}
 containerLogMaxFiles: {{ .kubernetes.kubelet.container_log_max_files }}
-{{- if .kubernetes.kubelet.extra_args }}
+{{- if .kubernetes.kubelet.extra_args | empty | not }}
 {{ .kubernetes.kubelet.extra_args | toYaml }}
 {{- end }}
diff --git a/builtin/core/roles/kubernetes/join-kubernetes/templates/kubeadm/kubeadm-join.v1beta2 b/builtin/core/roles/kubernetes/join-kubernetes/templates/kubeadm/kubeadm-join.v1beta2
index f8488d59..c1fa0ef3 100644
--- a/builtin/core/roles/kubernetes/join-kubernetes/templates/kubeadm/kubeadm-join.v1beta2
+++ b/builtin/core/roles/kubernetes/join-kubernetes/templates/kubeadm/kubeadm-join.v1beta2
@@ -11,9 +11,9 @@ discovery:
 {{- if .groups.kube_control_plane | default list | has .inventory_hostname }}
 controlPlane:
   localAPIEndpoint:
-  {{- if ne $internalIPv4 "" }}
+  {{- if $internalIPv4 | empty | not }}
     advertiseAddress: {{ $internalIPv4 }}
-  {{- else if ne $internalIPv6 "" }}
+  {{- else if $internalIPv6 | empty | not }}
     advertiseAddress: {{ $internalIPv6 }}
   {{- end }}
     bindPort: {{ .kubernetes.apiserver.port }}
@@ -23,4 +23,4 @@ nodeRegistration:
   criSocket: {{ .cri.cri_socket }}
   kubeletExtraArgs:
     cgroup-driver: {{ .cri.cgroup_driver }}
-    pod-infra-container-image: "{{ .k8s_registry }}/pause:{{ .cri.sandbox_image_tag }}"
+    pod-infra-container-image: "{{ .cri.sandbox_image.registry }}/{{ .cri.sandbox_image.repository }}:{{ .cri.sandbox_image.tag }}"
diff --git a/builtin/core/roles/kubernetes/join-kubernetes/templates/kubeadm/kubeadm-join.v1beta3 b/builtin/core/roles/kubernetes/join-kubernetes/templates/kubeadm/kubeadm-join.v1beta3
index f65f19f3..422d34d7 100644
--- a/builtin/core/roles/kubernetes/join-kubernetes/templates/kubeadm/kubeadm-join.v1beta3
+++ b/builtin/core/roles/kubernetes/join-kubernetes/templates/kubeadm/kubeadm-join.v1beta3
@@ -11,9 +11,9 @@ discovery:
 {{- if .groups.kube_control_plane | default list | has .inventory_hostname }}
 controlPlane:
   localAPIEndpoint:
-  {{- if ne $internalIPv4 "" }}
+  {{- if $internalIPv4 | empty | not }}
     advertiseAddress: {{ $internalIPv4 }}
-  {{- else if ne $internalIPv6 "" }}
+  {{- else if $internalIPv6 | empty | not }}
     advertiseAddress: {{ $internalIPv6 }}
   {{- end }}
     bindPort: {{ .kubernetes.apiserver.port }}
@@ -23,4 +23,4 @@ nodeRegistration:
   criSocket: {{ .cri.cri_socket }}
   kubeletExtraArgs:
     cgroup-driver: {{ .cri.cgroup_driver }}
-    pod-infra-container-image: "{{ .k8s_registry }}/pause:{{ .cri.sandbox_image_tag }}"
+    pod-infra-container-image: "{{ .cri.sandbox_image.registry }}/{{ .cri.sandbox_image.repository }}:{{ .cri.sandbox_image.tag }}"
diff --git a/builtin/core/roles/kubernetes/pre-kubernetes/templates/haproxy/haproxy.yaml b/builtin/core/roles/kubernetes/pre-kubernetes/templates/haproxy/haproxy.yaml
index 76882109..85d463c9 100644
--- a/builtin/core/roles/kubernetes/pre-kubernetes/templates/haproxy/haproxy.yaml
+++ b/builtin/core/roles/kubernetes/pre-kubernetes/templates/haproxy/haproxy.yaml
@@ -17,7 +17,7 @@ spec:
   priorityClassName: system-node-critical
   containers:
     - name: haproxy
-      image: {{ .kubernetes.control_plane_endpoint.haproxy.image }}
+      image: {{ .kubernetes.control_plane_endpoint.haproxy.image.registry }}/{{ .kubernetes.control_plane_endpoint.haproxy.image.repository }}:{{ .kubernetes.control_plane_endpoint.haproxy.image.tag }}
       imagePullPolicy: IfNotPresent
       resources:
         requests:
diff --git a/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubevip/kubevip.ARP b/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubevip/kubevip.ARP
index c1ce9a9c..05979968 100644
--- a/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubevip/kubevip.ARP
+++ b/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubevip/kubevip.ARP
@@ -39,7 +39,7 @@ spec:
       value: "6443"
     - name: address
       value: {{ .kubernetes.control_plane_endpoint.kube_vip.address }}
-    image: {{ .kubernetes.control_plane_endpoint.kube_vip.image }}
+    image: {{ .kubernetes.control_plane_endpoint.kube_vip.image.registry }}/{{ .kubernetes.control_plane_endpoint.kube_vip.image.repository }}:{{ .kubernetes.control_plane_endpoint.kube_vip.image.tag }}
     imagePullPolicy: IfNotPresent
     name: kube-vip
     resources: {}
diff --git a/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubevip/kubevip.BGP b/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubevip/kubevip.BGP
index 5b44633a..afece06d 100644
--- a/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubevip/kubevip.BGP
+++ b/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubevip/kubevip.BGP
@@ -57,7 +57,7 @@ spec:
       value: {{ .kubernetes.control_plane_endpoint.kube_vip.address }}
     - name: prometheus_server
       value: :2112
-    image: {{ .kubernetes.control_plane_endpoint.kube_vip.image }}
+    image: {{ .kubernetes.control_plane_endpoint.kube_vip.image.registry }}/{{ .kubernetes.control_plane_endpoint.kube_vip.image.repository }}:{{ .kubernetes.control_plane_endpoint.kube_vip.image.tag }}
     imagePullPolicy: IfNotPresent
     name: kube-vip
     resources: {}