feat: add and update playbooks (#2418)

* feat: update capkk playbooks and roles * feat: bug fix and add tags for playbook
2025-12-26 01:22:51 +00:00 · 2024-10-29 18:01:05 +08:00 · 2024-10-29 18:01:05 +08:00 · bb6f0c6ca7
parent a083447d15
commit bb6f0c6ca7
56 changed files with 1057 additions and 17 deletions
--- a/builtin/playbooks/artifact_export.yaml
+++ b/builtin/playbooks/artifact_export.yaml
@ -2,6 +2,7 @@
    - localhost
  roles:
    - init/init-artifact
+    - init/init-cert
  tasks:
    - name: Export artifact
      command: |
--- a/builtin/playbooks/artifact_images.yaml
+++ b/builtin/playbooks/artifact_images.yaml
@ -3,4 +3,5 @@
  tags: ["always"]
  roles:
    - init/init-artifact
+    - init/init-cert
    - install/image-registry
--- a/builtin/playbooks/capkk_binary_install.yaml
+++ b/builtin/playbooks/capkk_binary_install.yaml
@ -0,0 +1,14 @@
+---
+- name: NFS Install
+  hosts:
+    - nfs
+  gather_facts: true
+  roles:
+    - install/nfs
+
+#- name: Image Registry Install
+#  hosts:
+#    - image_registry
+#  gather_facts: true
+#  roles:
+#    - install/image-registry
--- a/builtin/playbooks/capkk_bootstrap_ready.yaml
+++ b/builtin/playbooks/capkk_bootstrap_ready.yaml
@ -0,0 +1,46 @@
+---
+- name: CNI Binary Install Pre-Check
+  hosts:
+    - k8s_cluster
+  gather_facts: true
+  vars_files:
+    - vars/cluster_requirements.yaml
+  tasks:
+    - name: Stop if arch supported
+      assert:
+        that: or (.cluster_require.supported_architectures.amd64 | has .os.architecture) (.cluster_require.supported_architectures.arm64 | has .os.architecture)
+        success_msg: |
+          {{- if .cluster_require.supported_architectures.amd64 | has .os.architecture }}
+          amd64
+          {{- else }}
+          arm64
+          {{- end }}
+        fail_msg: "{{ .os.architecture }} is not a known arch"
+      register: binary_type
+
+- name: Kubernetes Install
+  hosts:
+    - k8s_cluster
+  vars_files:
+    - vars/create_cluster_kubernetes.yaml
+  roles:
+    - install/cri
+    - capkk/install/kubernetes
+
+- name: Certs Distribution
+  hosts:
+    - kube_control_plane
+  roles:
+    - role: install/certs
+      when: .renew_certs.enabled
+
+- name: Install CNI
+  hosts:
+    - kube_control_plane|random
+  roles:
+    - addons/cni
+    - addons/kata
+    - addons/nfd
+    - addons/sc
+
+- import_playbook: hook/post_install.yaml
--- a/builtin/playbooks/capkk_check_connect.yaml
+++ b/builtin/playbooks/capkk_check_connect.yaml
@ -0,0 +1,11 @@
+---
+- name: Check Connect
+  hosts: all
+  ignore_errors: true
+  tasks:
+    - name: Check Connect for Hosts
+      when: .inventory_name | ne "localhost"
+      command: echo 1
+#  post_tasks:
+#  - name: Compute Respond Time for Hosts
+#    command: ping -c 1 -W 1 k8s-master-node | grep time= | awk '{print $8}' | cut -d'=' -f2
--- a/builtin/playbooks/capkk_delete_cluster.yaml
+++ b/builtin/playbooks/capkk_delete_cluster.yaml
@ -0,0 +1,18 @@
+---
+- name: Delete Kubernetes
+  hosts:
+    - k8s_cluster
+  roles:
+    - capkk/delete/kubernetes
+
+- name: Delete Network
+  hosts:
+    - k8s_cluster
+  roles:
+    - capkk/delete/network
+
+- name: Delete ETCD
+  hosts:
+    - etcd
+  roles:
+    - capkk/delete/etcd
--- a/builtin/playbooks/capkk_etcd_binary_install.yaml
+++ b/builtin/playbooks/capkk_etcd_binary_install.yaml
@ -0,0 +1,27 @@
+---
+- name: ETCD Binary Install Pre-Check
+  when: .kubernetes.etcd.deployment_type | eq "external"
+  hosts:
+    - etcd
+  gather_facts: true
+  vars_files:
+    - vars/cluster_requirements.yaml
+  tasks:
+    - name: Stop if arch supported
+      assert:
+        that: or (.cluster_require.supported_architectures.amd64 | has .os.architecture) (.cluster_require.supported_architectures.arm64 | has .os.architecture)
+        success_msg: |
+          {{- if .cluster_require.supported_architectures.amd64 | has .os.architecture }}
+          amd64
+          {{- else }}
+          arm64
+          {{- end }}
+        fail_msg: "{{ .os.architecture }} is not a known arch"
+      register: binary_type
+
+- name: ETCD Binary Install
+  when: .kubernetes.etcd.deployment_type | eq "external"
+  hosts:
+    - etcd
+  roles:
+    - install/etcd
--- a/builtin/playbooks/capkk_preparation.yaml
+++ b/builtin/playbooks/capkk_preparation.yaml
@ -0,0 +1,34 @@
+---
+- import_playbook: hook/pre_install.yaml
+
+- name: Environment PreCheck
+  gather_facts: true
+  hosts:
+    - k8s_cluster
+    - etcd
+    - image_registry
+    - nfs
+  tags: [ "always" ]
+  roles:
+    - precheck/env_check
+
+- name: Initialize OS
+  gather_facts: true
+  hosts:
+    - k8s_cluster
+    - etcd
+    - image_registry
+  roles:
+    - init/init-os
+
+- name: Download Artifacts
+  hosts:
+    - localhost
+  roles:
+    - init/init-artifact
+
+- name: Generate Certs
+  hosts:
+    - localhost
+  roles:
+    - init/init-cert
--- a/builtin/playbooks/certs_renew.yaml
+++ b/builtin/playbooks/certs_renew.yaml
@ -4,7 +4,7 @@
  vars_files:
    - vars/certs_renew.yaml
  roles:
-    - init/init-artifact
+    - init/init-cert

 - hosts:
    - etcd
--- a/builtin/playbooks/create_cluster.yaml
+++ b/builtin/playbooks/create_cluster.yaml
@ -18,11 +18,11 @@
  roles:
    - install/etcd

- hosts:
-    - image_registry
-  gather_facts: true
-  roles:
-    - install/image-registry
+#- hosts:
+#    - image_registry
+#  gather_facts: true
+#  roles:
+#    - install/image-registry

 - hosts:
    - k8s_cluster
--- a/builtin/playbooks/hook/post_install.yaml
+++ b/builtin/playbooks/hook/post_install.yaml
@ -4,14 +4,17 @@
    - all
  tasks:
    - name: Copy post install scripts to remote
-      ignore_errors: yes
+      ignore_errors: true
      copy:
        src: |
          {{ .work_dir }}/scripts/post_install_{{ .inventory_name }}.sh
        dest: |
          /etc/kubekey/scripts/post_install_{{ .inventory_name }}.sh
        mode: 0755
+      register: execute_result
+
    - name: Execute post install scripts
+      when: .execute_result.stderr | eq ""
      command: |
        for file in /etc/kubekey/scripts/post_install_*.sh; do
          if [ -f $file ]; then
--- a/builtin/playbooks/hook/pre_install.yaml
+++ b/builtin/playbooks/hook/pre_install.yaml
@ -4,14 +4,17 @@
    - all
  tasks:
    - name: Copy pre install scripts to remote
-      ignore_errors: yes
+      ignore_errors: true
      copy:
        src: |
          {{ .work_dir }}/scripts/pre_install_{{ .inventory_name }}.sh
        dest: |
          /etc/kubekey/scripts/pre_install_{{ .inventory_name }}.sh
        mode: 0755
+      register: execute_result
+
    - name: Execute pre install scripts
+      when: .execute_result.stderr | eq ""
      command: |
        for file in /etc/kubekey/scripts/pre_install_*.sh; do
          if [ -f $file ]; then
--- a/builtin/playbooks/init_os.yaml
+++ b/builtin/playbooks/init_os.yaml
@ -3,6 +3,7 @@
    - localhost
  roles:
    - init/init-artifact
+    - init/init-cert

 - hosts:
    - etcd
--- a/builtin/playbooks/init_registry.yaml
+++ b/builtin/playbooks/init_registry.yaml
@ -5,6 +5,7 @@
   - localhost
  roles:
   - init/init-artifact
+   - init/init-cert

 - hosts:
    - image_registry
--- a/builtin/playbooks/vars/cluster_requirements.yaml
+++ b/builtin/playbooks/vars/cluster_requirements.yaml
@ -0,0 +1,8 @@
+cluster_require:
+  supported_architectures:
+    amd64:
+      - amd64
+      - x86_64
+    arm64:
+      - arm64
+      - aarch64
--- a/builtin/playbooks/vars/init_artifact.yaml
+++ b/builtin/playbooks/vars/init_artifact.yaml
@ -0,0 +1,223 @@
+work_dir: /kubekey
+artifact:
+  arch: [ "amd64" ]
+  # offline artifact package for kk.
+  artifact_file: ""
+  # the md5_file of artifact_file.
+  artifact_md5: ""
+  artifact_url:
+    etcd:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/etcd/release/download/{{ .etcd_version }}/etcd-{{ .etcd_version }}-linux-amd64.tar.gz
+        {{- else }}
+        https://github.com/etcd-io/etcd/releases/download/{{ .etcd_version }}/etcd-{{ .etcd_version }}-linux-amd64.tar.gz
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/etcd/release/download/{{ .etcd_version }}/etcd-{{ .etcd_version }}-linux-arm64.tar.gz
+        {{- else }}
+        https://github.com/etcd-io/etcd/releases/download/{{ .etcd_version }}/etcd-{{ .etcd_version }}-linux-arm64.tar.gz
+        {{- end }}
+    kubeadm:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/release/{{ .kube_version }}/bin/linux/amd64/kubeadm
+        {{- else }}
+        https://storage.googleapis.com/kubernetes-release/release/{{ .kube_version }}/bin/linux/amd64/kubeadm
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/release/{{ .kube_version }}/bin/linux/arm64/kubeadm
+        {{- else }}
+        https://storage.googleapis.com/kubernetes-release/release/{{ .kube_version }}/bin/linux/arm64/kubeadm
+        {{- end }}
+    kubelet:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/release/{{ .kube_version }}/bin/linux/amd64/kubelet
+        {{- else }}
+        https://storage.googleapis.com/kubernetes-release/release/{{ .kube_version }}/bin/linux/amd64/kubelet
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/release/{{ .kube_version }}/bin/linux/arm64/kubelet
+        {{- else }}
+        https://storage.googleapis.com/kubernetes-release/release/{{ .kube_version }}/bin/linux/arm64/kubelet
+        {{- end }}
+    kubectl:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/release/{{ .kube_version }}/bin/linux/amd64/kubectl
+        {{- else }}
+        https://storage.googleapis.com/kubernetes-release/release/{{ .kube_version }}/bin/linux/amd64/kubectl
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/release/{{ .kube_version }}/bin/linux/arm64/kubectl
+        {{- else }}
+        https://storage.googleapis.com/kubernetes-release/release/{{ .kube_version }}/bin/linux/arm64/kubectl
+        {{- end }}
+    cni:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://github.com/containernetworking/plugins/releases/download/{{ .cni_version }}/cni-plugins-linux-amd64-{{ .cni_version }}.tgz
+        {{- else }}
+        https://containernetworking.pek3b.qingstor.com/plugins/releases/download/{{ .cni_version }}/cni-plugins-linux-amd64-{{ .cni_version }}.tgz
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://github.com/containernetworking/plugins/releases/download/{{ .cni_version }}/cni-plugins-linux-arm64-{{ .cni_version }}.tgz
+        {{- else }}
+        https://containernetworking.pek3b.qingstor.com/plugins/releases/download/{{ .cni_version }}/cni-plugins-linux-arm64-{{ .cni_version }}.tgz
+        {{- end }}
+    helm:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-helm.pek3b.qingstor.com/helm-{{ .helm_version }}-linux-amd64.tar.gz
+        {{- else }}
+        https://get.helm.sh/helm-{{ .helm_version }}-linux-amd64.tar.gz
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-helm.pek3b.qingstor.com/helm-{{ .helm_version }}-linux-arm64.tar.gz
+        {{- else }}
+        https://get.helm.sh/helm-{{ .helm_version }}-linux-arm64.tar.gz
+        {{- end }}
+    crictl:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/cri-tools/releases/download/{{ .crictl_version }}/crictl-{{ .crictl_version }}-linux-amd64.tar.gz
+        {{- else }}
+        https://github.com/kubernetes-sigs/cri-tools/releases/download/{{ .crictl_version }}/crictl-{{ .crictl_version }}-linux-amd64.tar.gz
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/cri-tools/releases/download/{{ .crictl_version }}/crictl-{{ .crictl_version }}-linux-arm64.tar.gz
+        {{- else }}
+        https://github.com/kubernetes-sigs/cri-tools/releases/download/{{ .crictl_version }}/crictl-{{ .crictl_version }}-linux-arm64.tar.gz
+        {{- end }}
+    docker:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://mirrors.aliyun.com/docker-ce/linux/static/stable/x86_64/docker-{{ .docker_version }}.tgz
+        {{- else }}
+        https://download.docker.com/linux/static/stable/x86_64/docker-{{ .docker_version }}.tgz
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://mirrors.aliyun.com/docker-ce/linux/static/stable/aarch64/docker-{{ .docker_version }}.tgz
+        {{- else }}
+        https://download.docker.com/linux/static/stable/aarch64/docker-{{ .docker_version }}.tgz
+        {{- end }}
+    cridockerd:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/releases/download/{{ .cridockerd_version }}/cri-dockerd-{{ .cridockerd_version | default "" | trimPrefix "v" }}.amd64.tgz
+        {{- else }}
+        https://github.com/Mirantis/cri-dockerd/releases/download/{{ .cridockerd_version }}/cri-dockerd-{{ .cridockerd_version | default "" | trimPrefix "v" }}.amd64.tgz
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/releases/download/{{ .cridockerd_version }}/cri-dockerd-{{ .cridockerd_version | default "" | trimPrefix "v" }}.arm64.tgz
+        {{- else }}
+        https://github.com/Mirantis/cri-dockerd/releases/download/{{ .cridockerd_version }}/cri-dockerd-{{ .cridockerd_version | default "" | trimPrefix "v" }}.arm64.tgz
+        {{- end }}
+    containerd:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/containerd/containerd/releases/download/{{ .containerd_version }}/containerd-{{ .containerd_version | default "" | trimPrefix "v" }}-linux-amd64.tar.gz
+        {{- else }}
+        https://github.com/containerd/containerd/releases/download/{{ .containerd_version }}/containerd-{{ .containerd_version | default "" | trimPrefix "v" }}-linux-amd64.tar.gz
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/containerd/containerd/releases/download/{{ .containerd_version }}/containerd-{{ .containerd_version | default "" | trimPrefix "v" }}-linux-arm64.tar.gz
+        {{- else }}
+        https://github.com/containerd/containerd/releases/download/{{ .containerd_version }}/containerd-{{ .containerd_version | default "" | trimPrefix "v"  }}-linux-arm64.tar.gz
+        {{- end }}
+    runc:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/opencontainers/runc/releases/download/{{ .runc_version }}/runc.amd64
+        {{- else }}
+        https://github.com/opencontainers/runc/releases/download/{{ .runc_version }}/runc.amd64
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/opencontainers/runc/releases/download/{{ .runc_version }}/runc.arm64
+        {{- else }}
+        https://github.com/opencontainers/runc/releases/download/{{ .runc_version }}/runc.arm64
+        {{- end }}
+    calicoctl:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/projectcalico/calico/releases/download/{{ .calico_version }}/calicoctl-linux-amd64
+        {{- else }}
+        https://github.com/projectcalico/calico/releases/download/{{ .calico_version }}/calicoctl-linux-amd64
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/projectcalico/calico/releases/download/{{ .calico_version }}/calicoctl-linux-arm64
+        {{- else }}
+        https://github.com/projectcalico/calico/releases/download/{{ .calico_version }}/calicoctl-linux-arm64
+        {{- end }}
+    dockercompose:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/docker/compose/releases/download/{{ .dockercompose_version }}/docker-compose-linux-x86_64
+        {{- else }}
+        https://github.com/docker/compose/releases/download/{{ .dockercompose_version }}/docker-compose-linux-x86_64
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/docker/compose/releases/download/{{ .dockercompose_version }}/docker-compose-linux-aarch64
+        {{- else }}
+        https://github.com/docker/compose/releases/download/{{ .dockercompose_version }}/docker-compose-linux-aarch64
+        {{- end }}
+    #    registry:
+    #      amd64: |
+    #        {{- if .kkzone | eq "cn" }}
+    #        https://kubernetes-release.pek3b.qingstor.com/registry/{{ .registry_version }}/registry-{{ .registry_version }}-linux-amd64.tgz
+    #        {{- else }}
+    #        https://github.com/kubesphere/kubekey/releases/download/{{ .registry_version }}/registry-{{ .registry_version }}-linux-amd64.tgz
+    #        {{- end }}
+    #      arm64: |
+    #        {{- if .kkzone | eq "cn" }}
+    #        https://kubernetes-release.pek3b.qingstor.com/registry/{{ .registry_version }}/registry-{{ .registry_version }}-linux-arm64.tgz
+    #        {{- else }}
+    #        https://github.com/kubesphere/kubekey/releases/download/{{ .registry_version }}/registry-{{ .registry_version }}-linux-arm64.tgz
+    #        {{- end }}
+    harbor:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://github.com/goharbor/harbor/releases/download/{{ .harbor_version }}/harbor-offline-installer-{{ .harbor_version }}.tgz
+        {{- else }}
+        https://github.com/goharbor/harbor/releases/download/{{ .harbor_version }}/harbor-offline-installer-{{ .harbor_version }}.tgz
+        {{- end }}
+    #      arm64: |
+    #        {{- if .kkzone | eq "cn" }}
+    #        https://github.com/goharbor/harbor/releases/download/{{ .harbor_version }}/harbor-{{ .harbor_version }}-linux-arm64.tgz
+    #        {{- else }}
+    #        https://github.com/goharbor/harbor/releases/download/{{ .harbor_version }}/harbor-{{ .harbor_version }}-linux-arm64.tgz
+    #        {{- end }}
+    #    keepalived:
+    #      amd64: |
+    #        {{- if .kkzone | eq "cn" }}
+    #        https://kubernetes-release.pek3b.qingstor.com/osixia/keepalived/releases/download/{{ .keepalived_version }}/keepalived-{{ .keepalived_version }}-linux-amd64.tgz
+    #        {{- else }}
+    #        https://github.com/osixia/keepalived/releases/download/{{ .keepalived_version }}/keepalived-{{ .keepalived_version }}-linux-amd64.tgz
+    #        {{- end }}
+    #      arm64: |
+    #        {{- if .kkzone | eq "cn" }}
+    #        https://kubernetes-release.pek3b.qingstor.com/osixia/keepalived/releases/download/{{ .keepalived_version }}/keepalived-{{ .keepalived_version }}-linux-arm64.tgz
+    #        {{- else }}
+    #        https://github.com/osixia/keepalived/releases/download/{{ .keepalived_version }}/keepalived-{{ .keepalived_version }}-linux-arm64.tgz
+    #        {{- end }}
+    cilium: https://helm.cilium.io/cilium-{{ .cilium_version }}.tgz
+    kubeovn: https://kubeovn.github.io/kube-ovn/kube-ovn-{{ .kubeovn_version }}.tgz
+    hybridnet: https://github.com/alibaba/hybridnet/releases/download/helm-chart-{{ .hybridnet_version }}/hybridnet-{{ .hybridnet_version }}.tgz
+    nfs_provisioner: https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/releases/download/nfs-subdir-external-provisioner-4.0.18/nfs-subdir-external-provisioner-{{ .nfs_provisioner_version }}.tgz
+  images:
+    auth: []
+    list: []
--- a/builtin/roles/capkk/delete/docker/tasks/main.yaml
+++ b/builtin/roles/capkk/delete/docker/tasks/main.yaml
@ -0,0 +1,15 @@
+---
+- name: Stop Docker Service
+  command: systemctl stop docker && systemctl disable docker
+
+- name: Delete Binary ETCD
+  command: rm /usr/local/bin/etcd && rm /usr/local/bin/etcdctl
+
+- name: Delete ETCD Certs
+  command: rm -r /etc/ssl/etcd/
+
+- name: Delete ETCD service file and env file
+  command: rm /etc/systemd/system/etcd.service && rm /etc/etcd.env
+
+- name: Delete Directories
+  command: rm -rf /var/lib/etcd/
--- a/builtin/roles/capkk/delete/etcd/tasks/main.yaml
+++ b/builtin/roles/capkk/delete/etcd/tasks/main.yaml
@ -0,0 +1,15 @@
+---
+- name: Stop ETCD Service
+  command: systemctl stop etcd && systemctl disable etcd
+
+- name: Delete Binary ETCD
+  command: rm /usr/local/bin/etcd && rm /usr/local/bin/etcdctl
+
+- name: Delete ETCD Certs
+  command: rm -r /etc/ssl/etcd/
+
+- name: Delete ETCD service file and env file
+  command: rm /etc/systemd/system/etcd.service && rm /etc/etcd.env
+
+- name: Delete Directories
+  command: rm -rf /var/lib/etcd/
--- a/builtin/roles/capkk/delete/kubernetes/tasks/main.yaml
+++ b/builtin/roles/capkk/delete/kubernetes/tasks/main.yaml
@ -0,0 +1,21 @@
+---
+- name: Cordon Node
+  command: kubectl cordon {{ .inventory_name }}
+
+- name: Drain Node
+  command: kubectl drain {{ .inventory_name }} --ignore-daemonsets --delete-emptydir-data
+
+- name: Delete Node
+  command: kubectl delete node {{ .inventory_name }}
+
+- name: Reset by Kubeadm
+  command: kubeadm reset -f
+
+- name: Remove Kube Services
+  command: rm -rf /etc/systemd/system/kube*
+
+- name: Delete Directories
+  command: |
+    rm -rf /etc/kubernetes/
+    rm -rf /var/lib/kubelet/
+    rm -rf {{ .work_dir }}
--- a/builtin/roles/capkk/delete/network/tasks/main.yaml
+++ b/builtin/roles/capkk/delete/network/tasks/main.yaml
@ -0,0 +1,13 @@
+---
+- name: Reset iptables
+  ignore_errors: true
+  command: iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X
+
+- name: Reset IPVS
+  ignore_errors: true
+  command: ipvsadm -C
+
+- name: Delete net.d
+  command: |
+    rm -rf /etc/cni/net.d/
+    rm -rf /var/lib/cni/
--- a/builtin/roles/capkk/download/curl/calicoctl/tasks/main.yaml
+++ b/builtin/roles/capkk/download/curl/calicoctl/tasks/main.yaml
@ -0,0 +1,18 @@
+---
+- name: Check binaries for calicoctl
+  tags: ["calicoctl"]
+  command: |
+    artifact_name=calicoctl
+    artifact_path={{ .work_dir }}/kubekey/cni/{{ .calico_version }}/{{ .item }}
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.calicoctl .item }})
+      if [ $http_code != 200 ]; then
+        echo "http code is $http_code"
+        exit 1
+      fi
+      curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.calicoctl .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .calico_version (ne .calico_version "")
--- a/builtin/roles/capkk/download/curl/cni/tasks/main.yaml
+++ b/builtin/roles/capkk/download/curl/cni/tasks/main.yaml
@ -0,0 +1,18 @@
+---
+- name: Check binaries for cni
+  tags: ["cni"]
+  command: |
+    artifact_name={{ get .artifact.artifact_url.cni .item | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/cni/{{ .cni_version }}/{{ .item }}
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+        http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.cni .item }})
+        if [ $http_code != 200 ]; then
+          echo "http code is $http_code"
+          exit 1
+        fi
+      curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.cni .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .cni_version (ne .cni_version "")
--- a/builtin/roles/capkk/download/curl/containerd/tasks/main.yaml
+++ b/builtin/roles/capkk/download/curl/containerd/tasks/main.yaml
@ -0,0 +1,18 @@
+---
+- name: Check binaries for containerd
+  tags: ["containerd"]
+  command: |
+    artifact_name={{ get .artifact.artifact_url.containerd .item | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/containerd/{{ .containerd_version }}/{{ .item }}
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.containerd .item }})
+      if [ $http_code != 200 ]; then
+        echo "http code is $http_code"
+        exit 1
+      fi
+      curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.containerd .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .containerd_version (ne .containerd_version "")
--- a/builtin/roles/capkk/download/curl/crictl/tasks/main.yaml
+++ b/builtin/roles/capkk/download/curl/crictl/tasks/main.yaml
@ -0,0 +1,18 @@
+---
+- name: Check binaries for crictl
+  tags: ["crictl"]
+  command: |
+    artifact_name={{ get .artifact.artifact_url.crictl .item | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/crictl/{{ .crictl_version }}/{{ .item }}
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.crictl .item }})
+      if [ $http_code != 200 ]; then
+        echo "http code is $http_code"
+        exit 1
+      fi
+      curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.crictl .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .crictl_version (ne .crictl_version "")
--- a/builtin/roles/capkk/download/curl/cridockerd/tasks/main.yaml
+++ b/builtin/roles/capkk/download/curl/cridockerd/tasks/main.yaml
@ -0,0 +1,17 @@
+---
+- name: Check binaries for containerd
+  command: |
+    artifact_name={{ get .artifact.artifact_url.containerd .item | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/containerd/{{ .containerd_version }}/{{ .item }}
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.containerd .item }})
+      if [ $http_code != 200 ]; then
+        echo "http code is $http_code"
+        exit 1
+      fi
+      curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.containerd .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .containerd_version (ne .containerd_version "")
--- a/builtin/roles/capkk/download/curl/docker-compose/tasks/main.yaml
+++ b/builtin/roles/capkk/download/curl/docker-compose/tasks/main.yaml
@ -0,0 +1,13 @@
+---
+- name: Check binaries for docker-compose
+  tags: ["docker-compose"]
+  command: |
+    compose_name=docker-compose
+    compose_path={{ .work_dir }}/kubekey/image-registry/docker-compose/{{ .dockercompose_version }}/{{ .item }}
+    if [ ! -f $compose_path/$compose_name ]; then
+      mkdir -p $compose_path
+      # download online
+      curl -L -o $compose_path/$compose_name {{ get .artifact.artifact_url.dockercompose .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .dockercompose_version (ne .dockercompose_version "")
--- a/builtin/roles/capkk/download/curl/docker/tasks/main.yaml
+++ b/builtin/roles/capkk/download/curl/docker/tasks/main.yaml
@ -0,0 +1,18 @@
+---
+- name: Check binaries for docker
+  tags: ["docker"]
+  command: |
+    artifact_name={{ get .artifact.artifact_url.docker .item | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/docker/{{ .docker_version }}/{{ .item }}
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.docker .item }})
+      if [ $http_code != 200 ]; then
+        echo "http code is $http_code"
+        exit 1
+      fi
+      curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.docker .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .docker_version (ne .docker_version "")
--- a/builtin/roles/capkk/download/curl/etcd/tasks/main.yaml
+++ b/builtin/roles/capkk/download/curl/etcd/tasks/main.yaml
@ -0,0 +1,18 @@
+---
+- name: Check binaries for etcd
+  tags: ["etcd"]
+  command: |
+    artifact_name={{ get .artifact.artifact_url.etcd .item | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/etcd/{{ .etcd_version }}/{{ .item }}
+    if [ ! -f $artifact_path/$artifact_name ]; then
+        mkdir -p $artifact_path
+        # download online
+        http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.etcd .item }})
+        if [ $http_code != 200 ]; then
+          echo "http code is $http_code"
+          exit 1
+        fi
+        curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.etcd .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .etcd_version (ne .etcd_version "")
--- a/builtin/roles/capkk/download/curl/harbor/tasks/main.yaml
+++ b/builtin/roles/capkk/download/curl/harbor/tasks/main.yaml
@ -0,0 +1,18 @@
+---
+- name: Check binaries for harbor
+  tags: ["harbor"]
+  command: |
+    harbor_name={{ get .artifact.artifact_url.harbor .item | splitList "/" | last }}
+    harbor_path={{ .work_dir }}/kubekey/image-registry/harbor/{{ .harbor_version }}/{{ .item }}
+    if [ ! -f $harbor_path/$harbor_name ]; then
+      mkdir -p $harbor_path
+      # download online
+      http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.harbor .item }})
+      if [ $http_code != 200 ]; then
+        echo "http code is $http_code"
+        exit 1
+      fi
+      curl -L -o $harbor_path/$harbor_name {{ get .artifact.artifact_url.harbor .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .harbor_version (ne .harbor_version "")
--- a/builtin/roles/capkk/download/curl/helm/tasks/main.yaml
+++ b/builtin/roles/capkk/download/curl/helm/tasks/main.yaml
@ -0,0 +1,18 @@
+---
+- name: Check binaries for helm
+  tags: ["helm"]
+  command: |
+    artifact_name={{ get .artifact.artifact_url.helm .item | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/helm/{{ .helm_version }}/{{ .item }}
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.helm .item }})
+      if [ $http_code != 200 ]; then
+        echo "http code is $http_code"
+        exit 1
+      fi
+      curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.helm .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .helm_version (ne .helm_version "")
--- a/builtin/roles/capkk/download/curl/keepalived/tasks/main.yaml
+++ b/builtin/roles/capkk/download/curl/keepalived/tasks/main.yaml
@ -0,0 +1,18 @@
+---
+- name: Check binaries for keepalived
+  tags: ["keepalived"]
+  command: |
+    artifact_name={{ get .artifact.artifact_url.keepalived .item | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/image-registry/keepalived/{{ .keepalived_version }}/{{ .item }}
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.keepalived .item }})
+      if [ $http_code != 200 ]; then
+        echo "http code is $http_code"
+        exit 1
+      fi
+      curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.keepalived .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .keepalived_version (ne .keepalived_version "")
--- a/builtin/roles/capkk/download/curl/kube/tasks/main.yaml
+++ b/builtin/roles/capkk/download/curl/kube/tasks/main.yaml
@ -0,0 +1,37 @@
+---
+- name: Check binaries for kube
+  tags: ["kube"]
+  command: |
+    kube_path={{ .work_dir }}/kubekey/kube/{{ .kube_version }}/{{ .item }}
+    if [ ! -f $kube_path/kubelet ]; then
+        mkdir -p $kube_path
+        # download online
+        http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.kubelet .item }})
+        if [ $http_code != 200 ]; then
+          echo "http code is $http_code"
+          exit 1
+        fi
+        curl -L -o $kube_path/kubelet {{ get .artifact.artifact_url.kubelet .item }}
+    fi
+    if [ ! -f $kube_path/kubeadm ]; then
+        mkdir -p $kube_path
+        # download online
+        http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.kubeadm .item }})
+        if [ $http_code != 200 ]; then
+          echo "http code is $http_code"
+          exit 1
+        fi
+        curl -L -o $kube_path/kubeadm {{ get .artifact.artifact_url.kubeadm .item }}
+    fi
+    if [ ! -f $kube_path/kubectl ]; then
+        mkdir -p $kube_path
+        # download online
+        http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.kubectl .item }})
+        if [ $http_code != 200 ]; then
+          echo "http code is $http_code"
+          exit 1
+        fi
+        curl -L -o $kube_path/kubectl {{ get .artifact.artifact_url.kubectl .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .kube_version (ne .kube_version "")
--- a/builtin/roles/capkk/download/curl/registry/tasks/main.yaml
+++ b/builtin/roles/capkk/download/curl/registry/tasks/main.yaml
@ -0,0 +1,18 @@
+---
+- name: Check binaries for registry
+  tags: ["registry"]
+  command: |
+    artifact_name={{ get .artifact.artifact_url.registry .item | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/image-registry/registry/{{ .registry_version }}/{{ .item }}
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.registry .item }})
+      if [ $http_code != 200 ]; then
+        echo "http code is $http_code"
+        exit 1
+      fi
+      curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.registry .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .registry_version (ne .registry_version "")
--- a/builtin/roles/capkk/download/curl/runc/tasks/main.yaml
+++ b/builtin/roles/capkk/download/curl/runc/tasks/main.yaml
@ -0,0 +1,18 @@
+---
+- name: Check binaries for runc
+  tags: ["runc"]
+  command: |
+    artifact_name={{ get .artifact.artifact_url.runc .item | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/runc/{{ .runc_version }}/{{ .item }}
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.runc .item }})
+      if [ $http_code != 200 ]; then
+        echo "http code is $http_code"
+        exit 1
+      fi
+      curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.runc .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .runc_version (ne .runc_version "")
--- a/builtin/roles/capkk/download/helm/cilium/tasks/main.yaml
+++ b/builtin/roles/capkk/download/helm/cilium/tasks/main.yaml
@ -0,0 +1,12 @@
+---
+- name: Check binaries for cilium
+  tags: ["cilium"]
+  command: |
+    artifact_name={{ .artifact.artifact_url.cilium | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/cni
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      cd $artifact_path && helm pull {{ .artifact.artifact_url.cilium }}
+    fi
+  when: and .cilium_version (ne .cilium_version "")
--- a/builtin/roles/capkk/download/helm/hybridnet/tasks/main.yaml
+++ b/builtin/roles/capkk/download/helm/hybridnet/tasks/main.yaml
@ -0,0 +1,12 @@
+---
+- name: Check binaries for hybridnet
+  tags: ["hybridnet"]
+  command: |
+    artifact_name={{ .artifact.artifact_url.hybridnet | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/cni
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      cd $artifact_path && helm pull {{ .artifact.artifact_url.hybridnet }}
+    fi
+  when: and .hybridnet_version (ne .hybridnet_version "")
--- a/builtin/roles/capkk/download/helm/kubeovn/tasks/main.yaml
+++ b/builtin/roles/capkk/download/helm/kubeovn/tasks/main.yaml
@ -0,0 +1,12 @@
+---
+- name: Check binaries for kubeovn
+  tags: ["kubeovn"]
+  command: |
+    artifact_name={{ .artifact.artifact_url.kubeovn | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/cni
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      cd $artifact_path && helm pull {{ .artifact.artifact_url.kubeovn }}
+    fi
+  when: and .kubeovn_version (ne .kubeovn_version "")
--- a/builtin/roles/capkk/download/helm/nfs-provisioner/tasks/main.yaml
+++ b/builtin/roles/capkk/download/helm/nfs-provisioner/tasks/main.yaml
@ -0,0 +1,12 @@
+---
+- name: Check binaries for nfs-provisioner
+  tags: ["nfs-provisioner"]
+  command: |
+    artifact_name={{ .artifact.artifact_url.nfs_provisioner |splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/sc
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      cd $artifact_path && helm pull {{ .artifact.artifact_url.nfs_provisioner }}
+    fi
+  when: and .nfs_provisioner_version (ne .nfs_provisioner_version "")
--- a/builtin/roles/capkk/install/etcd/defaults/main.yaml
+++ b/builtin/roles/capkk/install/etcd/defaults/main.yaml
@ -0,0 +1,27 @@
+etcd:
+  #    endpoints: ["https://172.1.1.1:2379"]
+  # etcd binary
+  state: new
+#  env config
+  env:
+    election_timeout: 5000
+    heartbeat_interval: 250
+    compaction_retention: 8
+    snapshot_count: 10000
+    data_dir: /var/lib/etcd
+    token: k8s_etcd
+#    metrics: basic
+#    quota_backend_bytes: 100
+#    max_request_bytes: 100
+#    max_snapshots: 100
+#    max_wals: 5
+#    log_level: info
+#    unsupported_arch: arm64
+#  backup config
+  backup:
+    backup_dir: /var/lib/etcd-backup
+    keep_backup_number: 5
+    etcd_backup_script: "backup.sh"
+    on_calendar: "*-*-* *:00/30:00"
+  performance: false
+  traffic_priority: false
--- a/builtin/roles/capkk/install/etcd/files/backup.service
+++ b/builtin/roles/capkk/install/etcd/files/backup.service
@ -0,0 +1,5 @@
+[Unit]
+Description=Backup ETCD
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/kube-scripts/backup_etcd.sh
--- a/builtin/roles/capkk/install/etcd/files/etcd.service
+++ b/builtin/roles/capkk/install/etcd/files/etcd.service
@ -0,0 +1,18 @@
+[Unit]
+Description=etcd
+After=network.target
+
+[Service]
+User=root
+Type=notify
+Nice=-20
+OOMScoreAdjust=-1000
+EnvironmentFile=/etc/etcd.env
+ExecStart=/usr/local/bin/etcd
+NotifyAccess=all
+RestartSec=10s
+LimitNOFILE=40000
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
--- a/builtin/roles/capkk/install/etcd/tasks/backup_etcd.yaml
+++ b/builtin/roles/capkk/install/etcd/tasks/backup_etcd.yaml
@ -0,0 +1,21 @@
+---
+- name: Sync custom backup etcd script
+  template:
+    src: |
+      {{ .etcd.backup.etcd_backup_script }}
+    dest: /usr/local/bin/kube-scripts/backup-etcd.sh
+    mode: 777
+
+- name: Generate backup etcd service
+  copy:
+    src: backup.service
+    dest: /etc/systemd/system/backup-etcd.service
+
+- name: Generate backup etcd timer
+  template:
+    src: backup.timer
+    dest: /etc/systemd/system/backup-etcd.timer
+
+- name: Enable etcd timer
+  command: |
+    systemctl daemon-reload && systemctl enable --now backup-etcd.timer
--- a/builtin/roles/capkk/install/etcd/tasks/install_etcd.yaml
+++ b/builtin/roles/capkk/install/etcd/tasks/install_etcd.yaml
@ -0,0 +1,55 @@
+---
+- name: Sync etcd binary to node
+  copy:
+    src: |
+      {{ .work_dir }}/kubekey/etcd/{{ .etcd_version }}/{{ .binary_type.stdout }}/etcd-{{ .etcd_version }}-linux-{{ .binary_type.stdout }}.tar.gz
+    dest: |
+      /tmp/kubekey/etcd-{{ .etcd_version }}-linux-{{ .binary_type.stdout }}.tar.gz
+
+- name: Extract etcd binary
+  command: |
+    tar --strip-components=1 -C /usr/local/bin/ -xvf /tmp/kubekey/etcd-{{ .etcd_version }}-linux-{{ .binary_type.stdout }}.tar.gz \
+      --wildcards etcd-{{ .etcd_version }}-linux-{{ .binary_type.stdout }}/etcd*
+
+- name: Sync ca file to remote
+  copy:
+    src: /etc/kubernetes/pki/ca/tls.crt
+    dest: /etc/ssl/etcd/ssl/ca.crt
+
+- name: Sync etcd cert file to remote
+  copy:
+    src: /etc/kubernetes/pki/etcd/tls.crt
+    dest: /etc/ssl/etcd/ssl/server.crt
+
+- name: Sync etcd key file to remote
+  copy:
+    src: /etc/kubernetes/pki/etcd/tls.key
+    dest: /etc/ssl/etcd/ssl/server.key
+
+- name: Generate etcd env file
+  template:
+    src: etcd.env
+    dest: /etc/etcd.env
+
+- name: Generate etcd systemd service file
+  copy:
+    src: etcd.service
+    dest: /etc/systemd/system/etcd.service
+
+# refer: https://etcd.io/docs/v3.5/tuning/
+- name: Set cpu to performance
+  command: |
+    echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
+  when: .etcd.performance
+
+- name: Set Traffic Priority
+  command: |
+    tc qdisc add dev eth0 root handle 1: prio bands 3
+    tc filter add dev eth0 parent 1: protocol ip prio 1 u32 match ip sport 2380 0xffff flowid 1:1
+    tc filter add dev eth0 parent 1: protocol ip prio 1 u32 match ip dport 2380 0xffff flowid 1:1
+    tc filter add dev eth0 parent 1: protocol ip prio 2 u32 match ip sport 2379 0xffff flowid 1:1
+    tc filter add dev eth0 parent 1: protocol ip prio 2 u32 match ip dport 2379 0xffff flowid 1:1
+  when: .etcd.traffic_priority
+
+- name: Start etcd service
+  command: systemctl daemon-reload && systemctl start etcd && systemctl enable etcd
--- a/builtin/roles/capkk/install/etcd/tasks/main.yaml
+++ b/builtin/roles/capkk/install/etcd/tasks/main.yaml
@ -0,0 +1,27 @@
+---
+- name: Check if etcd is installed
+  ignore_errors: true
+  command: etcd --version
+  run_once: true
+  register: etcd_install_version
+
+- name: Install etcd
+  when: |
+    or (.etcd_install_version.stderr | ne "") (.etcd_install_version.stdout | hasPrefix (printf "etcd Version: %s\n" (.etcd_version | default "" | trimPrefix "v")) | not)
+  block:
+    - name: Init etcd
+      block:
+        - name: Add etcd user
+          command: |
+            useradd -M -c 'Etcd user' -s /sbin/nologin -r etcd || :
+        - name: Create etcd directories
+          command: |
+            if [ ! -d "{{ .item }}" ]; then
+              mkdir -p {{ .item }} && chown -R etcd {{ .item }}
+            fi
+          loop:
+           - "/var/lib/etcd"
+
+    - include_tasks: install_etcd.yaml
+
+    - include_tasks: backup_etcd.yaml
--- a/builtin/roles/capkk/install/etcd/templates/backup.sh
+++ b/builtin/roles/capkk/install/etcd/templates/backup.sh
@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+ETCDCTL_PATH='/usr/local/bin/etcdctl'
+ENDPOINTS='https://{{ .internal_ipv4 }}:2379'
+ETCD_DATA_DIR="{{ .etcd.env.data_dir }}"
+BACKUP_DIR="{{ .etcd.backup.backup_dir }}/etcd-$(date +%Y-%m-%d-%H-%M-%S)"
+KEEPBACKUPNUMBER='{{ .etcd.backup.keep_backup_number }}'
+((KEEPBACKNUMBER++))
+
+ETCDCTL_CERT="/etc/ssl/etcd/ssl/server.crt"
+ETCDCTL_KEY="/etc/ssl/etcd/ssl/server.key"
+ETCDCTL_CA_FILE="/etc/ssl/etcd/ssl/ca.crt"
+
+[ ! -d $BACKUP_DIR ] && mkdir -p $BACKUP_DIR
+
+export ETCDCTL_API=2;$ETCDCTL_PATH backup --data-dir $ETCD_DATA_DIR --backup-dir $BACKUP_DIR
+
+sleep 3
+
+{
+export ETCDCTL_API=3;$ETCDCTL_PATH --endpoints="$ENDPOINTS" snapshot save $BACKUP_DIR/snapshot.db \
+                                   --cacert="$ETCDCTL_CA_FILE" \
+                                   --cert="$ETCDCTL_CERT" \
+                                   --key="$ETCDCTL_KEY"
+} > /dev/null
+
+sleep 3
+
+cd $BACKUP_DIR/../ && ls -lt |awk '{if(NR > '$KEEPBACKUPNUMBER'){print "rm -rf "$9}}'|sh
--- a/builtin/roles/capkk/install/etcd/templates/backup.timer
+++ b/builtin/roles/capkk/install/etcd/templates/backup.timer
@ -0,0 +1,7 @@
+[Unit]
+Description=Timer to backup ETCD
+[Timer]
+OnCalendar={{ .etcd.backup.on_calendar }}
+Unit=backup-etcd.service
+[Install]
+WantedBy=multi-user.target
--- a/builtin/roles/capkk/install/etcd/templates/etcd.env
+++ b/builtin/roles/capkk/install/etcd/templates/etcd.env
@ -0,0 +1,57 @@
+ETCD_DATA_DIR={{ .etcd.env.data_dir }}
+ETCD_ADVERTISE_CLIENT_URLS={{ printf "https://%s:2379" .internal_ipv4 }}
+ETCD_INITIAL_ADVERTISE_PEER_URLS={{ printf "https://%s:2380" .internal_ipv4 }}
+ETCD_INITIAL_CLUSTER_STATE={{ .etcd.state }}
+ETCD_LISTEN_CLIENT_URLS={{ printf "https://%s:2379" .internal_ipv4 }},https://127.0.0.1:2379
+ETCD_INITIAL_CLUSTER_TOKEN={{ .etcd.env.token }}
+ETCD_LISTEN_PEER_URLS={{ printf "https://%s:2380" .internal_ipv4 }}
+ETCD_NAME={{ .hostname }}
+ETCD_PROXY=off
+ETCD_ENABLE_V2=true
+{{- $ips := list }}
+{{- range .groups.etcd | default list }}
+    {{- $ips = append $ips (printf "%s=https://%s:2380" (index $.inventory_hosts . "hostname") (index $.inventory_hosts . "internal_ipv4")) }}
+{{- end }}
+ETCD_INITIAL_CLUSTER={{ $ips | join "," }}
+ETCD_ELECTION_TIMEOUT={{ .etcd.env.election_timeout }}
+ETCD_HEARTBEAT_INTERVAL={{ .etcd.env.heartbeat_interval }}
+ETCD_AUTO_COMPACTION_RETENTION={{ .etcd.env.compaction_retention }}
+ETCD_SNAPSHOT_COUNT={{ .etcd.env.snapshot_count }}
+{{- if .etcd.metrics }}
+ETCD_METRICS={{ .etcd.env.metrics }}
+{{- end }}
+{{- if .etcd.env.quota_backend_bytes }}
+ETCD_QUOTA_BACKEND_BYTES={{ .etcd.env.quota_backend_bytes }}
+{{- end }}
+{{- if .etcd.env.max_request_bytes }}
+ETCD_MAX_REQUEST_BYTES={{ .etcd.env.max_request_bytes }}
+{{- end }}
+{{- if .etcd.env.max_snapshots }}
+ETCD_MAX_SNAPSHOTS={{ .etcd.env.max_snapshots }}
+{{- end }}
+{{- if .etcd.env.max_wals }}
+ETCD_MAX_WALS={{ .etcd.env.max_wals }}
+{{- end }}
+{{- if .etcd.env.log_level }}
+ETCD_LOG_LEVEL={{ .etcd.env.log_level }}
+{{- end }}
+{{- if .etcd.env.unsupported_arch }}
+ETCD_UNSUPPORTED_ARCH={{ .etcd.env.unsupported_arch }}
+{{- end }}
+
+# TLS settings
+ETCD_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.crt
+ETCD_CERT_FILE=/etc/ssl/etcd/ssl/server.crt
+ETCD_KEY_FILE=/etc/ssl/etcd/ssl/server.key
+ETCD_CLIENT_CERT_AUTH=true
+
+ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.crt
+ETCD_PEER_CERT_FILE=/etc/ssl/etcd/ssl/server.crt
+ETCD_PEER_KEY_FILE=/etc/ssl/etcd/ssl/server.key
+ETCD_PEER_CLIENT_CERT_AUTH=true
+
+# CLI settings
+ETCDCTL_ENDPOINTS=https://127.0.0.1:2379
+ETCDCTL_CACERT=/etc/ssl/etcd/ssl/ca.crt
+ETCDCTL_CERT=/etc/ssl/etcd/ssl/server.crt
+ETCDCTL_KEY=/etc/ssl/etcd/ssl/server.key
--- a/builtin/roles/init/init-artifact/defaults/main.yaml
+++ b/builtin/roles/init/init-artifact/defaults/main.yaml
@ -5,8 +5,6 @@ artifact:
  artifact_file: ""
  # the md5_file of artifact_file.
  artifact_md5: ""
-  # how to generate cert file.support: IfNotPresent, Always
-  gen_cert_policy: IfNotPresent
  artifact_url:
    etcd:
      amd64: |
--- a/builtin/roles/init/init-artifact/tasks/download_by_curl.yaml
+++ b/builtin/roles/init/init-artifact/tasks/download_by_curl.yaml
@ -1,5 +1,6 @@
 ---
 - name: Check binaries for etcd
+  tags: ["etcd"]
  command: |
    artifact_name={{ get .artifact.artifact_url.etcd .item | splitList "/" | last }}
    artifact_path={{ .work_dir }}/kubekey/etcd/{{ .etcd_version }}/{{ .item }}
@ -17,6 +18,7 @@
  when: and .etcd_version (ne .etcd_version "")

 - name: Check binaries for kube
+  tags: ["kube"]
  command: |
    kube_path={{ .work_dir }}/kubekey/kube/{{ .kube_version }}/{{ .item }}
    if [ ! -f $kube_path/kubelet ]; then
@ -53,6 +55,7 @@
  when: and .kube_version (ne .kube_version "")

 - name: Check binaries for cni
+  tags: ["cni"]
  command: |
    artifact_name={{ get .artifact.artifact_url.cni .item | splitList "/" | last }}
    artifact_path={{ .work_dir }}/kubekey/cni/{{ .cni_version }}/{{ .item }}
@ -70,6 +73,7 @@
  when: and .cni_version (ne .cni_version "")

 - name: Check binaries for helm
+  tags: ["helm"]
  command: |
    artifact_name={{ get .artifact.artifact_url.helm .item | splitList "/" | last }}
    artifact_path={{ .work_dir }}/kubekey/helm/{{ .helm_version }}/{{ .item }}
@ -87,6 +91,7 @@
  when: and .helm_version (ne .helm_version "")

 - name: Check binaries for crictl
+  tags: ["crictl"]
  command: |
    artifact_name={{ get .artifact.artifact_url.crictl .item | splitList "/" | last }}
    artifact_path={{ .work_dir }}/kubekey/crictl/{{ .crictl_version }}/{{ .item }}
@ -104,6 +109,7 @@
  when: and .crictl_version (ne .crictl_version "")

 - name: Check binaries for docker
+  tags: ["docker"]
  command: |
    artifact_name={{ get .artifact.artifact_url.docker .item | splitList "/" | last }}
    artifact_path={{ .work_dir }}/kubekey/docker/{{ .docker_version }}/{{ .item }}
@ -121,6 +127,7 @@
  when: and .docker_version (ne .docker_version "")

 - name: Check binaries for cridockerd
+  tags: ["cridockerd"]
  command: |
    artifact_name={{ get .artifact.artifact_url.cridockerd .item | splitList "/" | last }}
    artifact_path={{ .work_dir }}/kubekey/cri-dockerd/{{ .cridockerd_version }}/{{ .item }}
@ -138,6 +145,7 @@
  when: and .cridockerd_version (ne .docker_version "")

 - name: Check binaries for containerd
+  tags: ["containerd"]
  command: |
    artifact_name={{ get .artifact.artifact_url.containerd .item | splitList "/" | last }}
    artifact_path={{ .work_dir }}/kubekey/containerd/{{ .containerd_version }}/{{ .item }}
@ -155,6 +163,7 @@
  when: and .containerd_version (ne .containerd_version "")

 - name: Check binaries for runc
+  tags: ["runc"]
  command: |
    artifact_name={{ get .artifact.artifact_url.runc .item | splitList "/" | last }}
    artifact_path={{ .work_dir }}/kubekey/runc/{{ .runc_version }}/{{ .item }}
@ -172,6 +181,7 @@
  when: and .runc_version (ne .runc_version "")

 - name: Check binaries for calicoctl
+  tags: ["calicoctl"]
  command: |
    artifact_name=calicoctl
    artifact_path={{ .work_dir }}/kubekey/cni/{{ .calico_version }}/{{ .item }}
@ -189,6 +199,7 @@
  when: and .calico_version (ne .calico_version "")

 - name: Check binaries for registry
+  tags: ["registry"]
  command: |
    artifact_name={{ get .artifact.artifact_url.registry .item | splitList "/" | last }}
    artifact_path={{ .work_dir }}/kubekey/image-registry/registry/{{ .registry_version }}/{{ .item }}
@ -206,6 +217,7 @@
  when: and .registry_version (ne .registry_version "")

 - name: Check binaries for docker-compose
+  tags: ["docker-compose"]
  command: |
    compose_name=docker-compose
    compose_path={{ .work_dir }}/kubekey/image-registry/docker-compose/{{ .dockercompose_version }}/{{ .item }}
@ -218,6 +230,7 @@
  when: and .dockercompose_version (ne .dockercompose_version "")

 - name: Check binaries for harbor
+  tags: ["harbor"]
  command: |
    harbor_name={{ get .artifact.artifact_url.harbor .item | splitList "/" | last }}
    harbor_path={{ .work_dir }}/kubekey/image-registry/harbor/{{ .harbor_version }}/{{ .item }}
@ -235,6 +248,7 @@
  when: and .harbor_version (ne .harbor_version "")

 - name: Check binaries for keepalived
+  tags: ["keepalived"]
  command: |
    artifact_name={{ get .artifact.artifact_url.keepalived .item | splitList "/" | last }}
    artifact_path={{ .work_dir }}/kubekey/image-registry/keepalived/{{ .keepalived_version }}/{{ .item }}
--- a/builtin/roles/init/init-artifact/tasks/download_by_helm.yaml
+++ b/builtin/roles/init/init-artifact/tasks/download_by_helm.yaml
@ -1,5 +1,6 @@
 ---
 - name: Check binaries for cilium
+  tags: ["cilium"]
  command: |
    artifact_name={{ .artifact.artifact_url.cilium | splitList "/" | last }}
    artifact_path={{ .work_dir }}/kubekey/cni
@ -11,6 +12,7 @@
  when: and .cilium_version (ne .cilium_version "")

 - name: Check binaries for kubeovn
+  tags: ["kubeovn"]
  command: |
    artifact_name={{ .artifact.artifact_url.kubeovn | splitList "/" | last }}
    artifact_path={{ .work_dir }}/kubekey/cni
@ -22,6 +24,7 @@
  when: and .kubeovn_version (ne .kubeovn_version "")

 - name: Check binaries for hybridnet
+  tags: ["hybridnet"]
  command: |
    artifact_name={{ .artifact.artifact_url.hybridnet | splitList "/" | last }}
    artifact_path={{ .work_dir }}/kubekey/cni
@ -33,6 +36,7 @@
  when: and .hybridnet_version (ne .hybridnet_version "")

 - name: Check binaries for nfs_provisioner
+  tags: ["nfs_provisioner"]
  command: |
    artifact_name={{ .artifact.artifact_url.nfs_provisioner |splitList "/" | last }}
    artifact_path={{ .work_dir }}/kubekey/sc
--- a/builtin/roles/init/init-artifact/tasks/main.yaml
+++ b/builtin/roles/init/init-artifact/tasks/main.yaml
@ -28,11 +28,8 @@
          manifests: "{{ .image_manifests | toJson }}"
      when: .image_manifests | default list | len | lt 0

- include_tasks: pki.yaml
-  tags: ["certs"]
-
 - name: Chown work_dir to sudo
  tags: ["always"]
-  ignore_errors: yes
+  ignore_errors: true
  command: |
    chown -R ${SUDO_UID}:${SUDO_GID} {{ .work_dir }}
--- a/builtin/roles/init/init-cert/defaults/main.yaml
+++ b/builtin/roles/init/init-cert/defaults/main.yaml
@ -0,0 +1,4 @@
+work_dir: /kubekey
+artifact:
+  # how to generate cert file.support: IfNotPresent, Always
+  gen_cert_policy: IfNotPresent
--- a/builtin/roles/init/init-artifact/tasks/pki.yaml
+++ b/builtin/roles/init/init-artifact/tasks/pki.yaml
@ -50,3 +50,8 @@
    out_cert: |
      {{ .work_dir }}/kubekey/pki/image_registry.crt
  when: and .groups.image_registry (.groups.image_registry | default list | len | lt 0)
+
+- name: Chown pki to sudo
+  ignore_errors: yes
+  command: |
+    chown -R ${SUDO_UID}:${SUDO_GID} {{ .work_dir }}/kubekey/pki
--- a/builtin/roles/install/kubernetes/defaults/main.yaml
+++ b/builtin/roles/install/kubernetes/defaults/main.yaml
@ -18,7 +18,7 @@ kubernetes:
    pod_cidr: 10.233.64.0/18
    service_cidr: 10.233.0.0/18
    dns_image: |
-      {{ .k8s_registry }}/coredns/coredns:1.8.6
+      {{ .k8s_registry }}/coredns/coredns:v1.8.6
    dns_cache_image: |
      {{ .dockerio_registry }}/kubesphere/k8s-dns-node-cache:1.22.20
    dns_service_ip: |
--- a/builtin/roles/install/kubernetes/templates/kubeadm/kubeadm-init.v1beta2
+++ b/builtin/roles/install/kubernetes/templates/kubeadm/kubeadm-init.v1beta2
@ -8,7 +8,7 @@ etcd:
    imageTag: {{ .kubernetes.etcd.image | splitList ":" | last }}
    serverCertSANs:
    {{- range .groups.etcd | default list }}
-    - https://{{ index $.inventory_hosts . "internal_ipv4" }}:2379
+    - {{ index $.inventory_hosts . "internal_ipv4" }}
    {{- end }}
 {{- else }}
  external:
--- a/builtin/roles/install/kubernetes/templates/kubeadm/kubeadm-init.v1beta3
+++ b/builtin/roles/install/kubernetes/templates/kubeadm/kubeadm-init.v1beta3
@ -8,7 +8,7 @@ etcd:
    imageTag: {{ .kubernetes.etcd.image | splitList ":" | last }}
    serverCertSANs:
    {{- range .groups.etcd | default list }}
-    - https://{{ index $.inventory_hosts . "internal_ipv4" }}:2379
+    - {{ index $.inventory_hosts . "internal_ipv4" }}
    {{- end }}
 {{- else }}
  external: