From 104f014b1febc25474f7641a1a5a3641dc089975 Mon Sep 17 00:00:00 2001
From: DingYongliang <1521323439@qq.com>
Date: Thu, 10 Oct 2024 14:21:24 +0800
Subject: [PATCH 1/2] feat: update capkk playbooks and roles

---
 builtin/playbooks/artifact_export.yaml        |   1 +
 builtin/playbooks/artifact_images.yaml        |   1 +
 builtin/playbooks/capkk_binary_install.yaml   |  14 ++
 builtin/playbooks/capkk_bootstrap_ready.yaml  |  46 ++++
 builtin/playbooks/capkk_check_connect.yaml    |  11 +
 builtin/playbooks/capkk_delete_cluster.yaml   |  18 ++
 .../playbooks/capkk_etcd_binary_install.yaml  |  42 ++++
 builtin/playbooks/capkk_preparation.yaml      |  34 +++
 builtin/playbooks/certs_renew.yaml            |   2 +-
 builtin/playbooks/create_cluster.yaml         |  10 +-
 builtin/playbooks/hook/post_install.yaml      |   5 +-
 builtin/playbooks/hook/pre_install.yaml       |   5 +-
 builtin/playbooks/init_os.yaml                |   1 +
 builtin/playbooks/init_registry.yaml          |   1 +
 .../playbooks/vars/cluster_requirements.yaml  |   8 +
 builtin/playbooks/vars/init_artifact.yaml     | 223 ++++++++++++++++++
 .../roles/capkk/delete/docker/tasks/main.yaml |  15 ++
 .../roles/capkk/delete/etcd/tasks/main.yaml   |  15 ++
 .../capkk/delete/kubernetes/tasks/main.yaml   |  21 ++
 .../capkk/delete/network/tasks/main.yaml      |  13 +
 .../download/curl/calicoctl/tasks/main.yaml   |  18 ++
 .../capkk/download/curl/cni/tasks/main.yaml   |  18 ++
 .../download/curl/containerd/tasks/main.yaml  |  18 ++
 .../download/curl/crictl/tasks/main.yaml      |  18 ++
 .../download/curl/cridockerd/tasks/main.yaml  |  17 ++
 .../curl/docker-compose/tasks/main.yaml       |  13 +
 .../download/curl/docker/tasks/main.yaml      |  18 ++
 .../capkk/download/curl/etcd/tasks/main.yaml  |  18 ++
 .../download/curl/harbor/tasks/main.yaml      |  18 ++
 .../capkk/download/curl/helm/tasks/main.yaml  |  18 ++
 .../download/curl/keepalived/tasks/main.yaml  |  18 ++
 .../capkk/download/curl/kube/tasks/main.yaml  |  37 +++
 .../download/curl/registry/tasks/main.yaml    |  18 ++
 .../capkk/download/curl/runc/tasks/main.yaml  |  18 ++
 .../download/helm/cilium/tasks/main.yaml      |  12 +
 .../download/helm/hybridnet/tasks/main.yaml   |  12 +
 .../download/helm/kubeovn/tasks/main.yaml     |  12 +
 .../helm/nfs-provisioner/tasks/main.yaml      |  12 +
 .../capkk/install/etcd/defaults/main.yaml     |  27 +++
 .../capkk/install/etcd/files/backup.service   |   5 +
 .../capkk/install/etcd/files/etcd.service     |  18 ++
 .../capkk/install/etcd/tasks/backup_etcd.yaml |  21 ++
 .../install/etcd/tasks/install_etcd.yaml      |  55 +++++
 .../roles/capkk/install/etcd/tasks/main.yaml  |  27 +++
 .../capkk/install/etcd/templates/backup.sh    |  33 +++
 .../capkk/install/etcd/templates/backup.timer |   7 +
 .../capkk/install/etcd/templates/etcd.env     |  57 +++++
 .../init/init-artifact/defaults/main.yaml     |   2 -
 .../roles/init/init-artifact/tasks/main.yaml  |   5 +-
 .../roles/init/init-cert/defaults/main.yaml   |   4 +
 .../pki.yaml => init-cert/tasks/main.yaml}    |   5 +
 .../install/kubernetes/defaults/main.yaml     |   2 +-
 52 files changed, 1052 insertions(+), 15 deletions(-)
 create mode 100644 builtin/playbooks/capkk_binary_install.yaml
 create mode 100644 builtin/playbooks/capkk_bootstrap_ready.yaml
 create mode 100644 builtin/playbooks/capkk_check_connect.yaml
 create mode 100644 builtin/playbooks/capkk_delete_cluster.yaml
 create mode 100644 builtin/playbooks/capkk_etcd_binary_install.yaml
 create mode 100644 builtin/playbooks/capkk_preparation.yaml
 create mode 100644 builtin/playbooks/vars/cluster_requirements.yaml
 create mode 100644 builtin/playbooks/vars/init_artifact.yaml
 create mode 100644 builtin/roles/capkk/delete/docker/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/delete/etcd/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/delete/kubernetes/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/delete/network/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/download/curl/calicoctl/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/download/curl/cni/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/download/curl/containerd/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/download/curl/crictl/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/download/curl/cridockerd/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/download/curl/docker-compose/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/download/curl/docker/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/download/curl/etcd/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/download/curl/harbor/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/download/curl/helm/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/download/curl/keepalived/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/download/curl/kube/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/download/curl/registry/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/download/curl/runc/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/download/helm/cilium/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/download/helm/hybridnet/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/download/helm/kubeovn/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/download/helm/nfs-provisioner/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/install/etcd/defaults/main.yaml
 create mode 100644 builtin/roles/capkk/install/etcd/files/backup.service
 create mode 100644 builtin/roles/capkk/install/etcd/files/etcd.service
 create mode 100644 builtin/roles/capkk/install/etcd/tasks/backup_etcd.yaml
 create mode 100644 builtin/roles/capkk/install/etcd/tasks/install_etcd.yaml
 create mode 100644 builtin/roles/capkk/install/etcd/tasks/main.yaml
 create mode 100644 builtin/roles/capkk/install/etcd/templates/backup.sh
 create mode 100644 builtin/roles/capkk/install/etcd/templates/backup.timer
 create mode 100644 builtin/roles/capkk/install/etcd/templates/etcd.env
 create mode 100644 builtin/roles/init/init-cert/defaults/main.yaml
 rename builtin/roles/init/{init-artifact/tasks/pki.yaml => init-cert/tasks/main.yaml} (92%)

diff --git a/builtin/playbooks/artifact_export.yaml b/builtin/playbooks/artifact_export.yaml
index bc3ba97fe..f724bf6d1 100644
--- a/builtin/playbooks/artifact_export.yaml
+++ b/builtin/playbooks/artifact_export.yaml
@@ -2,6 +2,7 @@
     - localhost
   roles:
     - init/init-artifact
+    - init/init-cert
   tasks:
     - name: Export artifact
       command: |
diff --git a/builtin/playbooks/artifact_images.yaml b/builtin/playbooks/artifact_images.yaml
index 377afb268..a4892e3f9 100644
--- a/builtin/playbooks/artifact_images.yaml
+++ b/builtin/playbooks/artifact_images.yaml
@@ -3,4 +3,5 @@
   tags: ["always"]
   roles:
     - init/init-artifact
+    - init/init-cert
     - install/image-registry
diff --git a/builtin/playbooks/capkk_binary_install.yaml b/builtin/playbooks/capkk_binary_install.yaml
new file mode 100644
index 000000000..1e0fbaf09
--- /dev/null
+++ b/builtin/playbooks/capkk_binary_install.yaml
@@ -0,0 +1,14 @@
+---
+- name: NFS Install
+  hosts:
+    - nfs
+  gather_facts: true
+  roles:
+    - install/nfs
+
+#- name: Image Registry Install
+#  hosts:
+#    - image_registry
+#  gather_facts: true
+#  roles:
+#    - install/image-registry
\ No newline at end of file
diff --git a/builtin/playbooks/capkk_bootstrap_ready.yaml b/builtin/playbooks/capkk_bootstrap_ready.yaml
new file mode 100644
index 000000000..a8be54613
--- /dev/null
+++ b/builtin/playbooks/capkk_bootstrap_ready.yaml
@@ -0,0 +1,46 @@
+---
+- name: CNI Binary Install Pre-Check
+  hosts:
+    - k8s_cluster
+  gather_facts: true
+  vars_files:
+    - vars/cluster_requirements.yaml
+  tasks:
+    - name: Stop if arch supported
+      assert:
+        that: or (.cluster_require.supported_architectures.amd64 | has .os.architecture) (.cluster_require.supported_architectures.arm64 | has .os.architecture)
+        success_msg: |
+          {{- if .cluster_require.supported_architectures.amd64 | has .os.architecture }}
+          amd64
+          {{- else }}
+          arm64
+          {{- end }}
+        fail_msg: "{{ .os.architecture }} is not a known arch"
+      register: binary_type
+
+- name: Kubernetes Install
+  hosts:
+    - k8s_cluster
+  vars_files:
+    - vars/create_cluster_kubernetes.yaml
+  roles:
+    - install/cri
+    - install/kubernetes
+
+- name: Certs Distribution
+  hosts:
+    - kube_control_plane
+  roles:
+    - role: install/certs
+      when: .renew_certs.enabled
+
+- name: Install CNI
+  hosts:
+    - kube_control_plane|random
+  roles:
+    - addons/cni
+    - addons/kata
+    - addons/nfd
+    - addons/sc
+
+- import_playbook: hook/post_install.yaml
\ No newline at end of file
diff --git a/builtin/playbooks/capkk_check_connect.yaml b/builtin/playbooks/capkk_check_connect.yaml
new file mode 100644
index 000000000..cc4e29748
--- /dev/null
+++ b/builtin/playbooks/capkk_check_connect.yaml
@@ -0,0 +1,11 @@
+---
+- name: Check Connect
+  hosts: all
+  ignore_errors: true
+  tasks:
+    - name: Check Connect for Hosts
+      when: .inventory_name | ne "localhost"
+      command: echo 1
+#  post_tasks:
+#  - name: Compute Respond Time for Hosts
+#    command: ping -c 1 -W 1 k8s-master-node | grep time= | awk '{print $8}' | cut -d'=' -f2
\ No newline at end of file
diff --git a/builtin/playbooks/capkk_delete_cluster.yaml b/builtin/playbooks/capkk_delete_cluster.yaml
new file mode 100644
index 000000000..98600bdbc
--- /dev/null
+++ b/builtin/playbooks/capkk_delete_cluster.yaml
@@ -0,0 +1,18 @@
+---
+- name: Delete Kubernetes
+  hosts:
+    - k8s_cluster
+  roles:
+    - capkk/kubernetes
+
+- name: Delete Network
+  hosts:
+    - k8s_cluster
+  roles:
+    - capkk/network
+
+- name: Delete ETCD
+  hosts:
+    - etcd
+  roles:
+    - capkk/etcd
\ No newline at end of file
diff --git a/builtin/playbooks/capkk_etcd_binary_install.yaml b/builtin/playbooks/capkk_etcd_binary_install.yaml
new file mode 100644
index 000000000..c0a18a518
--- /dev/null
+++ b/builtin/playbooks/capkk_etcd_binary_install.yaml
@@ -0,0 +1,42 @@
+---
+- name: ETCD Binary Install Pre-Check
+  hosts:
+    - etcd
+  gather_facts: true
+  vars_files:
+    - vars/cluster_requirements.yaml
+  tasks:
+    - name: Stop if arch supported
+      assert:
+        that: or (.cluster_require.supported_architectures.amd64 | has .os.architecture) (.cluster_require.supported_architectures.arm64 | has .os.architecture)
+        success_msg: |
+          {{- if .cluster_require.supported_architectures.amd64 | has .os.architecture }}
+          amd64
+          {{- else }}
+          arm64
+          {{- end }}
+        fail_msg: "{{ .os.architecture }} is not a known arch"
+      register: binary_type
+
+# TODO: 需要讨论一个比较合适的 Install 流程，要统一安装目录等等
+#- name: ETCD Binary Download
+#  hosts:
+#    - etcd
+#  tags: ["etcd"]
+#  vars_files:
+#    - vars/init_artifact.yaml
+#  pre_tasks:
+#    - name: Create Directory
+#      tags: [ "always" ]
+#      command: |
+#        if [ ! -d "{{ .work_dir }}" ]; then
+#          mkdir -p {{ .work_dir }}/kubekey
+#        fi
+#  roles:
+#    - capkk/download/curl/etcd
+
+- name: ETCD Binary Install
+  hosts:
+    - etcd
+  roles:
+    - install/etcd
\ No newline at end of file
diff --git a/builtin/playbooks/capkk_preparation.yaml b/builtin/playbooks/capkk_preparation.yaml
new file mode 100644
index 000000000..30a9dcaf1
--- /dev/null
+++ b/builtin/playbooks/capkk_preparation.yaml
@@ -0,0 +1,34 @@
+---
+- import_playbook: hook/pre_install.yaml
+
+- name: Environment PreCheck
+  gather_facts: true
+  hosts:
+    - k8s_cluster
+    - etcd
+    - image_registry
+    - nfs
+  tags: [ "always" ]
+  roles:
+    - precheck/env_check
+
+- name: Initialize OS
+  gather_facts: true
+  hosts:
+    - k8s_cluster
+    - etcd
+    - image_registry
+  roles:
+    - init/init-os
+
+- name: Download Artifacts
+  hosts:
+    - localhost
+  roles:
+    - init/init-artifact
+
+- name: Generate Certs
+  hosts:
+    - localhost
+  roles:
+    - init/init-cert
\ No newline at end of file
diff --git a/builtin/playbooks/certs_renew.yaml b/builtin/playbooks/certs_renew.yaml
index 599d64668..e69cd3f10 100644
--- a/builtin/playbooks/certs_renew.yaml
+++ b/builtin/playbooks/certs_renew.yaml
@@ -4,7 +4,7 @@
   vars_files:
     - vars/certs_renew.yaml
   roles:
-    - init/init-artifact
+    - init/init-cert
 
 - hosts:
     - etcd
diff --git a/builtin/playbooks/create_cluster.yaml b/builtin/playbooks/create_cluster.yaml
index 1970b1034..25dcca595 100644
--- a/builtin/playbooks/create_cluster.yaml
+++ b/builtin/playbooks/create_cluster.yaml
@@ -18,11 +18,11 @@
   roles:
     - install/etcd
 
-- hosts:
-    - image_registry
-  gather_facts: true
-  roles:
-    - install/image-registry
+#- hosts:
+#    - image_registry
+#  gather_facts: true
+#  roles:
+#    - install/image-registry
 
 - hosts:
     - k8s_cluster
diff --git a/builtin/playbooks/hook/post_install.yaml b/builtin/playbooks/hook/post_install.yaml
index 25d37123f..d6be21cbb 100644
--- a/builtin/playbooks/hook/post_install.yaml
+++ b/builtin/playbooks/hook/post_install.yaml
@@ -4,14 +4,17 @@
     - all
   tasks:
     - name: Copy post install scripts to remote
-      ignore_errors: yes
+      ignore_errors: true
       copy:
         src: |
           {{ .work_dir }}/scripts/post_install_{{ .inventory_name }}.sh
         dest: |
           /etc/kubekey/scripts/post_install_{{ .inventory_name }}.sh
         mode: 0755
+      register: execute_result
+
     - name: Execute post install scripts
+      when: .execute_result.stderr | eq ""
       command: |
         for file in /etc/kubekey/scripts/post_install_*.sh; do
           if [ -f $file ]; then
diff --git a/builtin/playbooks/hook/pre_install.yaml b/builtin/playbooks/hook/pre_install.yaml
index 627f906c5..b9860bd17 100644
--- a/builtin/playbooks/hook/pre_install.yaml
+++ b/builtin/playbooks/hook/pre_install.yaml
@@ -4,14 +4,17 @@
     - all
   tasks:
     - name: Copy pre install scripts to remote
-      ignore_errors: yes
+      ignore_errors: true
       copy:
         src: |
           {{ .work_dir }}/scripts/pre_install_{{ .inventory_name }}.sh
         dest: |
           /etc/kubekey/scripts/pre_install_{{ .inventory_name }}.sh
         mode: 0755
+      register: execute_result
+
     - name: Execute pre install scripts
+      when: .execute_result.stderr | eq ""
       command: |
         for file in /etc/kubekey/scripts/pre_install_*.sh; do
           if [ -f $file ]; then
diff --git a/builtin/playbooks/init_os.yaml b/builtin/playbooks/init_os.yaml
index 7d8d03983..35c0ca6d5 100644
--- a/builtin/playbooks/init_os.yaml
+++ b/builtin/playbooks/init_os.yaml
@@ -3,6 +3,7 @@
     - localhost
   roles:
     - init/init-artifact
+    - init/init-cert
 
 - hosts:
     - etcd
diff --git a/builtin/playbooks/init_registry.yaml b/builtin/playbooks/init_registry.yaml
index 11cc3859d..63fa56ca9 100644
--- a/builtin/playbooks/init_registry.yaml
+++ b/builtin/playbooks/init_registry.yaml
@@ -5,6 +5,7 @@
    - localhost
   roles:
    - init/init-artifact
+   - init/init-cert
 
 - hosts:
     - image_registry
diff --git a/builtin/playbooks/vars/cluster_requirements.yaml b/builtin/playbooks/vars/cluster_requirements.yaml
new file mode 100644
index 000000000..13143e16b
--- /dev/null
+++ b/builtin/playbooks/vars/cluster_requirements.yaml
@@ -0,0 +1,8 @@
+cluster_require:
+  supported_architectures:
+    amd64:
+      - amd64
+      - x86_64
+    arm64:
+      - arm64
+      - aarch64
\ No newline at end of file
diff --git a/builtin/playbooks/vars/init_artifact.yaml b/builtin/playbooks/vars/init_artifact.yaml
new file mode 100644
index 000000000..b68afff4a
--- /dev/null
+++ b/builtin/playbooks/vars/init_artifact.yaml
@@ -0,0 +1,223 @@
+work_dir: /kubekey
+artifact:
+  arch: [ "amd64" ]
+  # offline artifact package for kk.
+  artifact_file: ""
+  # the md5_file of artifact_file.
+  artifact_md5: ""
+  artifact_url:
+    etcd:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/etcd/release/download/{{ .etcd_version }}/etcd-{{ .etcd_version }}-linux-amd64.tar.gz
+        {{- else }}
+        https://github.com/etcd-io/etcd/releases/download/{{ .etcd_version }}/etcd-{{ .etcd_version }}-linux-amd64.tar.gz
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/etcd/release/download/{{ .etcd_version }}/etcd-{{ .etcd_version }}-linux-arm64.tar.gz
+        {{- else }}
+        https://github.com/etcd-io/etcd/releases/download/{{ .etcd_version }}/etcd-{{ .etcd_version }}-linux-arm64.tar.gz
+        {{- end }}
+    kubeadm:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/release/{{ .kube_version }}/bin/linux/amd64/kubeadm
+        {{- else }}
+        https://storage.googleapis.com/kubernetes-release/release/{{ .kube_version }}/bin/linux/amd64/kubeadm
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/release/{{ .kube_version }}/bin/linux/arm64/kubeadm
+        {{- else }}
+        https://storage.googleapis.com/kubernetes-release/release/{{ .kube_version }}/bin/linux/arm64/kubeadm
+        {{- end }}
+    kubelet:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/release/{{ .kube_version }}/bin/linux/amd64/kubelet
+        {{- else }}
+        https://storage.googleapis.com/kubernetes-release/release/{{ .kube_version }}/bin/linux/amd64/kubelet
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/release/{{ .kube_version }}/bin/linux/arm64/kubelet
+        {{- else }}
+        https://storage.googleapis.com/kubernetes-release/release/{{ .kube_version }}/bin/linux/arm64/kubelet
+        {{- end }}
+    kubectl:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/release/{{ .kube_version }}/bin/linux/amd64/kubectl
+        {{- else }}
+        https://storage.googleapis.com/kubernetes-release/release/{{ .kube_version }}/bin/linux/amd64/kubectl
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/release/{{ .kube_version }}/bin/linux/arm64/kubectl
+        {{- else }}
+        https://storage.googleapis.com/kubernetes-release/release/{{ .kube_version }}/bin/linux/arm64/kubectl
+        {{- end }}
+    cni:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://github.com/containernetworking/plugins/releases/download/{{ .cni_version }}/cni-plugins-linux-amd64-{{ .cni_version }}.tgz
+        {{- else }}
+        https://containernetworking.pek3b.qingstor.com/plugins/releases/download/{{ .cni_version }}/cni-plugins-linux-amd64-{{ .cni_version }}.tgz
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://github.com/containernetworking/plugins/releases/download/{{ .cni_version }}/cni-plugins-linux-arm64-{{ .cni_version }}.tgz
+        {{- else }}
+        https://containernetworking.pek3b.qingstor.com/plugins/releases/download/{{ .cni_version }}/cni-plugins-linux-arm64-{{ .cni_version }}.tgz
+        {{- end }}
+    helm:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-helm.pek3b.qingstor.com/helm-{{ .helm_version }}-linux-amd64.tar.gz
+        {{- else }}
+        https://get.helm.sh/helm-{{ .helm_version }}-linux-amd64.tar.gz
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-helm.pek3b.qingstor.com/helm-{{ .helm_version }}-linux-arm64.tar.gz
+        {{- else }}
+        https://get.helm.sh/helm-{{ .helm_version }}-linux-arm64.tar.gz
+        {{- end }}
+    crictl:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/cri-tools/releases/download/{{ .crictl_version }}/crictl-{{ .crictl_version }}-linux-amd64.tar.gz
+        {{- else }}
+        https://github.com/kubernetes-sigs/cri-tools/releases/download/{{ .crictl_version }}/crictl-{{ .crictl_version }}-linux-amd64.tar.gz
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/cri-tools/releases/download/{{ .crictl_version }}/crictl-{{ .crictl_version }}-linux-arm64.tar.gz
+        {{- else }}
+        https://github.com/kubernetes-sigs/cri-tools/releases/download/{{ .crictl_version }}/crictl-{{ .crictl_version }}-linux-arm64.tar.gz
+        {{- end }}
+    docker:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://mirrors.aliyun.com/docker-ce/linux/static/stable/x86_64/docker-{{ .docker_version }}.tgz
+        {{- else }}
+        https://download.docker.com/linux/static/stable/x86_64/docker-{{ .docker_version }}.tgz
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://mirrors.aliyun.com/docker-ce/linux/static/stable/aarch64/docker-{{ .docker_version }}.tgz
+        {{- else }}
+        https://download.docker.com/linux/static/stable/aarch64/docker-{{ .docker_version }}.tgz
+        {{- end }}
+    cridockerd:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/releases/download/{{ .cridockerd_version }}/cri-dockerd-{{ .cridockerd_version | default "" | trimPrefix "v" }}.amd64.tgz
+        {{- else }}
+        https://github.com/Mirantis/cri-dockerd/releases/download/{{ .cridockerd_version }}/cri-dockerd-{{ .cridockerd_version | default "" | trimPrefix "v" }}.amd64.tgz
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/releases/download/{{ .cridockerd_version }}/cri-dockerd-{{ .cridockerd_version | default "" | trimPrefix "v" }}.arm64.tgz
+        {{- else }}
+        https://github.com/Mirantis/cri-dockerd/releases/download/{{ .cridockerd_version }}/cri-dockerd-{{ .cridockerd_version | default "" | trimPrefix "v" }}.arm64.tgz
+        {{- end }}
+    containerd:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/containerd/containerd/releases/download/{{ .containerd_version }}/containerd-{{ .containerd_version | default "" | trimPrefix "v" }}-linux-amd64.tar.gz
+        {{- else }}
+        https://github.com/containerd/containerd/releases/download/{{ .containerd_version }}/containerd-{{ .containerd_version | default "" | trimPrefix "v" }}-linux-amd64.tar.gz
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/containerd/containerd/releases/download/{{ .containerd_version }}/containerd-{{ .containerd_version | default "" | trimPrefix "v" }}-linux-arm64.tar.gz
+        {{- else }}
+        https://github.com/containerd/containerd/releases/download/{{ .containerd_version }}/containerd-{{ .containerd_version | default "" | trimPrefix "v"  }}-linux-arm64.tar.gz
+        {{- end }}
+    runc:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/opencontainers/runc/releases/download/{{ .runc_version }}/runc.amd64
+        {{- else }}
+        https://github.com/opencontainers/runc/releases/download/{{ .runc_version }}/runc.amd64
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/opencontainers/runc/releases/download/{{ .runc_version }}/runc.arm64
+        {{- else }}
+        https://github.com/opencontainers/runc/releases/download/{{ .runc_version }}/runc.arm64
+        {{- end }}
+    calicoctl:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/projectcalico/calico/releases/download/{{ .calico_version }}/calicoctl-linux-amd64
+        {{- else }}
+        https://github.com/projectcalico/calico/releases/download/{{ .calico_version }}/calicoctl-linux-amd64
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/projectcalico/calico/releases/download/{{ .calico_version }}/calicoctl-linux-arm64
+        {{- else }}
+        https://github.com/projectcalico/calico/releases/download/{{ .calico_version }}/calicoctl-linux-arm64
+        {{- end }}
+    dockercompose:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/docker/compose/releases/download/{{ .dockercompose_version }}/docker-compose-linux-x86_64
+        {{- else }}
+        https://github.com/docker/compose/releases/download/{{ .dockercompose_version }}/docker-compose-linux-x86_64
+        {{- end }}
+      arm64: |
+        {{- if .kkzone | eq "cn" }}
+        https://kubernetes-release.pek3b.qingstor.com/docker/compose/releases/download/{{ .dockercompose_version }}/docker-compose-linux-aarch64
+        {{- else }}
+        https://github.com/docker/compose/releases/download/{{ .dockercompose_version }}/docker-compose-linux-aarch64
+        {{- end }}
+    #    registry:
+    #      amd64: |
+    #        {{- if .kkzone | eq "cn" }}
+    #        https://kubernetes-release.pek3b.qingstor.com/registry/{{ .registry_version }}/registry-{{ .registry_version }}-linux-amd64.tgz
+    #        {{- else }}
+    #        https://github.com/kubesphere/kubekey/releases/download/{{ .registry_version }}/registry-{{ .registry_version }}-linux-amd64.tgz
+    #        {{- end }}
+    #      arm64: |
+    #        {{- if .kkzone | eq "cn" }}
+    #        https://kubernetes-release.pek3b.qingstor.com/registry/{{ .registry_version }}/registry-{{ .registry_version }}-linux-arm64.tgz
+    #        {{- else }}
+    #        https://github.com/kubesphere/kubekey/releases/download/{{ .registry_version }}/registry-{{ .registry_version }}-linux-arm64.tgz
+    #        {{- end }}
+    harbor:
+      amd64: |
+        {{- if .kkzone | eq "cn" }}
+        https://github.com/goharbor/harbor/releases/download/{{ .harbor_version }}/harbor-offline-installer-{{ .harbor_version }}.tgz
+        {{- else }}
+        https://github.com/goharbor/harbor/releases/download/{{ .harbor_version }}/harbor-offline-installer-{{ .harbor_version }}.tgz
+        {{- end }}
+    #      arm64: |
+    #        {{- if .kkzone | eq "cn" }}
+    #        https://github.com/goharbor/harbor/releases/download/{{ .harbor_version }}/harbor-{{ .harbor_version }}-linux-arm64.tgz
+    #        {{- else }}
+    #        https://github.com/goharbor/harbor/releases/download/{{ .harbor_version }}/harbor-{{ .harbor_version }}-linux-arm64.tgz
+    #        {{- end }}
+    #    keepalived:
+    #      amd64: |
+    #        {{- if .kkzone | eq "cn" }}
+    #        https://kubernetes-release.pek3b.qingstor.com/osixia/keepalived/releases/download/{{ .keepalived_version }}/keepalived-{{ .keepalived_version }}-linux-amd64.tgz
+    #        {{- else }}
+    #        https://github.com/osixia/keepalived/releases/download/{{ .keepalived_version }}/keepalived-{{ .keepalived_version }}-linux-amd64.tgz
+    #        {{- end }}
+    #      arm64: |
+    #        {{- if .kkzone | eq "cn" }}
+    #        https://kubernetes-release.pek3b.qingstor.com/osixia/keepalived/releases/download/{{ .keepalived_version }}/keepalived-{{ .keepalived_version }}-linux-arm64.tgz
+    #        {{- else }}
+    #        https://github.com/osixia/keepalived/releases/download/{{ .keepalived_version }}/keepalived-{{ .keepalived_version }}-linux-arm64.tgz
+    #        {{- end }}
+    cilium: https://helm.cilium.io/cilium-{{ .cilium_version }}.tgz
+    kubeovn: https://kubeovn.github.io/kube-ovn/kube-ovn-{{ .kubeovn_version }}.tgz
+    hybridnet: https://github.com/alibaba/hybridnet/releases/download/helm-chart-{{ .hybridnet_version }}/hybridnet-{{ .hybridnet_version }}.tgz
+    nfs_provisioner: https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/releases/download/nfs-subdir-external-provisioner-4.0.18/nfs-subdir-external-provisioner-{{ .nfs_provisioner_version }}.tgz
+  images:
+    auth: []
+    list: []
diff --git a/builtin/roles/capkk/delete/docker/tasks/main.yaml b/builtin/roles/capkk/delete/docker/tasks/main.yaml
new file mode 100644
index 000000000..507f587b8
--- /dev/null
+++ b/builtin/roles/capkk/delete/docker/tasks/main.yaml
@@ -0,0 +1,15 @@
+---
+- name: Stop Docker Service
+  command: systemctl stop docker && systemctl disable docker
+
+- name: Delete Binary ETCD
+  command: rm /usr/local/bin/etcd && rm /usr/local/bin/etcdctl
+
+- name: Delete ETCD Certs
+  command: rm -r /etc/ssl/etcd/
+
+- name: Delete ETCD service file and env file
+  command: rm /etc/systemd/system/etcd.service && rm /etc/etcd.env
+
+- name: Delete Directories
+  command: rm -rf /var/lib/etcd/
\ No newline at end of file
diff --git a/builtin/roles/capkk/delete/etcd/tasks/main.yaml b/builtin/roles/capkk/delete/etcd/tasks/main.yaml
new file mode 100644
index 000000000..28156f479
--- /dev/null
+++ b/builtin/roles/capkk/delete/etcd/tasks/main.yaml
@@ -0,0 +1,15 @@
+---
+- name: Stop ETCD Service
+  command: systemctl stop etcd && systemctl disable etcd
+
+- name: Delete Binary ETCD
+  command: rm /usr/local/bin/etcd && rm /usr/local/bin/etcdctl
+
+- name: Delete ETCD Certs
+  command: rm -r /etc/ssl/etcd/
+
+- name: Delete ETCD service file and env file
+  command: rm /etc/systemd/system/etcd.service && rm /etc/etcd.env
+
+- name: Delete Directories
+  command: rm -rf /var/lib/etcd/
\ No newline at end of file
diff --git a/builtin/roles/capkk/delete/kubernetes/tasks/main.yaml b/builtin/roles/capkk/delete/kubernetes/tasks/main.yaml
new file mode 100644
index 000000000..0bf553bbb
--- /dev/null
+++ b/builtin/roles/capkk/delete/kubernetes/tasks/main.yaml
@@ -0,0 +1,21 @@
+---
+- name: Cordon Node
+  command: kubectl cordon {{ .inventory_name }}
+
+- name: Drain Node
+  command: kubectl drain {{ .inventory_name }} --ignore-daemonsets --delete-emptydir-data
+
+- name: Delete Node
+  command: kubectl delete node {{ .inventory_name }}
+
+- name: Reset by Kubeadm
+  command: kubeadm reset -f
+
+- name: Remove Kube Services
+  command: rm -rf /etc/systemd/system/kube*
+
+- name: Delete Directories
+  command: |
+    rm -rf /etc/kubernetes/
+    rm -rf /var/lib/kubelet/
+    rm -rf {{ .work_dir }}
diff --git a/builtin/roles/capkk/delete/network/tasks/main.yaml b/builtin/roles/capkk/delete/network/tasks/main.yaml
new file mode 100644
index 000000000..4d8606259
--- /dev/null
+++ b/builtin/roles/capkk/delete/network/tasks/main.yaml
@@ -0,0 +1,13 @@
+---
+- name: Reset iptables
+  ignore_errors: true
+  command: iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X
+
+- name: Reset IPVS
+  ignore_errors: true
+  command: ipvsadm -C
+
+- name: Delete net.d
+  command: |
+    rm -rf /etc/cni/net.d/
+    rm -rf /var/lib/cni/
\ No newline at end of file
diff --git a/builtin/roles/capkk/download/curl/calicoctl/tasks/main.yaml b/builtin/roles/capkk/download/curl/calicoctl/tasks/main.yaml
new file mode 100644
index 000000000..4c955c04e
--- /dev/null
+++ b/builtin/roles/capkk/download/curl/calicoctl/tasks/main.yaml
@@ -0,0 +1,18 @@
+---
+- name: Check binaries for calicoctl
+  tags: ["calicoctl"]
+  command: |
+    artifact_name=calicoctl
+    artifact_path={{ .work_dir }}/kubekey/cni/{{ .calico_version }}/{{ .item }}
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.calicoctl .item }})
+      if [ $http_code != 200 ]; then
+        echo "http code is $http_code"
+        exit 1
+      fi
+      curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.calicoctl .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .calico_version (ne .calico_version "")
\ No newline at end of file
diff --git a/builtin/roles/capkk/download/curl/cni/tasks/main.yaml b/builtin/roles/capkk/download/curl/cni/tasks/main.yaml
new file mode 100644
index 000000000..71d7548aa
--- /dev/null
+++ b/builtin/roles/capkk/download/curl/cni/tasks/main.yaml
@@ -0,0 +1,18 @@
+---
+- name: Check binaries for cni
+  tags: ["cni"]
+  command: |
+    artifact_name={{ get .artifact.artifact_url.cni .item | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/cni/{{ .cni_version }}/{{ .item }}
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+        http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.cni .item }})
+        if [ $http_code != 200 ]; then
+          echo "http code is $http_code"
+          exit 1
+        fi
+      curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.cni .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .cni_version (ne .cni_version "")
\ No newline at end of file
diff --git a/builtin/roles/capkk/download/curl/containerd/tasks/main.yaml b/builtin/roles/capkk/download/curl/containerd/tasks/main.yaml
new file mode 100644
index 000000000..2f381233b
--- /dev/null
+++ b/builtin/roles/capkk/download/curl/containerd/tasks/main.yaml
@@ -0,0 +1,18 @@
+---
+- name: Check binaries for containerd
+  tags: ["containerd"]
+  command: |
+    artifact_name={{ get .artifact.artifact_url.containerd .item | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/containerd/{{ .containerd_version }}/{{ .item }}
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.containerd .item }})
+      if [ $http_code != 200 ]; then
+        echo "http code is $http_code"
+        exit 1
+      fi
+      curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.containerd .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .containerd_version (ne .containerd_version "")
\ No newline at end of file
diff --git a/builtin/roles/capkk/download/curl/crictl/tasks/main.yaml b/builtin/roles/capkk/download/curl/crictl/tasks/main.yaml
new file mode 100644
index 000000000..9431311d2
--- /dev/null
+++ b/builtin/roles/capkk/download/curl/crictl/tasks/main.yaml
@@ -0,0 +1,18 @@
+---
+- name: Check binaries for crictl
+  tags: ["crictl"]
+  command: |
+    artifact_name={{ get .artifact.artifact_url.crictl .item | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/crictl/{{ .crictl_version }}/{{ .item }}
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.crictl .item }})
+      if [ $http_code != 200 ]; then
+        echo "http code is $http_code"
+        exit 1
+      fi
+      curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.crictl .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .crictl_version (ne .crictl_version "")
\ No newline at end of file
diff --git a/builtin/roles/capkk/download/curl/cridockerd/tasks/main.yaml b/builtin/roles/capkk/download/curl/cridockerd/tasks/main.yaml
new file mode 100644
index 000000000..c50f4e024
--- /dev/null
+++ b/builtin/roles/capkk/download/curl/cridockerd/tasks/main.yaml
@@ -0,0 +1,17 @@
+---
+- name: Check binaries for containerd
+  command: |
+    artifact_name={{ get .artifact.artifact_url.containerd .item | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/containerd/{{ .containerd_version }}/{{ .item }}
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.containerd .item }})
+      if [ $http_code != 200 ]; then
+        echo "http code is $http_code"
+        exit 1
+      fi
+      curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.containerd .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .containerd_version (ne .containerd_version "")
\ No newline at end of file
diff --git a/builtin/roles/capkk/download/curl/docker-compose/tasks/main.yaml b/builtin/roles/capkk/download/curl/docker-compose/tasks/main.yaml
new file mode 100644
index 000000000..766a30410
--- /dev/null
+++ b/builtin/roles/capkk/download/curl/docker-compose/tasks/main.yaml
@@ -0,0 +1,13 @@
+---
+- name: Check binaries for docker-compose
+  tags: ["docker-compose"]
+  command: |
+    compose_name=docker-compose
+    compose_path={{ .work_dir }}/kubekey/image-registry/docker-compose/{{ .dockercompose_version }}/{{ .item }}
+    if [ ! -f $compose_path/$compose_name ]; then
+      mkdir -p $compose_path
+      # download online
+      curl -L -o $compose_path/$compose_name {{ get .artifact.artifact_url.dockercompose .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .dockercompose_version (ne .dockercompose_version "")
\ No newline at end of file
diff --git a/builtin/roles/capkk/download/curl/docker/tasks/main.yaml b/builtin/roles/capkk/download/curl/docker/tasks/main.yaml
new file mode 100644
index 000000000..e8c4b8296
--- /dev/null
+++ b/builtin/roles/capkk/download/curl/docker/tasks/main.yaml
@@ -0,0 +1,18 @@
+---
+- name: Check binaries for docker
+  tags: ["docker"]
+  command: |
+    artifact_name={{ get .artifact.artifact_url.docker .item | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/docker/{{ .docker_version }}/{{ .item }}
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.docker .item }})
+      if [ $http_code != 200 ]; then
+        echo "http code is $http_code"
+        exit 1
+      fi
+      curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.docker .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .docker_version (ne .docker_version "")
\ No newline at end of file
diff --git a/builtin/roles/capkk/download/curl/etcd/tasks/main.yaml b/builtin/roles/capkk/download/curl/etcd/tasks/main.yaml
new file mode 100644
index 000000000..4ccd929a1
--- /dev/null
+++ b/builtin/roles/capkk/download/curl/etcd/tasks/main.yaml
@@ -0,0 +1,18 @@
+---
+- name: Check binaries for etcd
+  tags: ["etcd"]
+  command: |
+    artifact_name={{ get .artifact.artifact_url.etcd .item | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/etcd/{{ .etcd_version }}/{{ .item }}
+    if [ ! -f $artifact_path/$artifact_name ]; then
+        mkdir -p $artifact_path
+        # download online
+        http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.etcd .item }})
+        if [ $http_code != 200 ]; then
+          echo "http code is $http_code"
+          exit 1
+        fi
+        curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.etcd .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .etcd_version (ne .etcd_version "")
\ No newline at end of file
diff --git a/builtin/roles/capkk/download/curl/harbor/tasks/main.yaml b/builtin/roles/capkk/download/curl/harbor/tasks/main.yaml
new file mode 100644
index 000000000..860e586f0
--- /dev/null
+++ b/builtin/roles/capkk/download/curl/harbor/tasks/main.yaml
@@ -0,0 +1,18 @@
+---
+- name: Check binaries for harbor
+  tags: ["harbor"]
+  command: |
+    harbor_name={{ get .artifact.artifact_url.harbor .item | splitList "/" | last }}
+    harbor_path={{ .work_dir }}/kubekey/image-registry/harbor/{{ .harbor_version }}/{{ .item }}
+    if [ ! -f $harbor_path/$harbor_name ]; then
+      mkdir -p $harbor_path
+      # download online
+      http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.harbor .item }})
+      if [ $http_code != 200 ]; then
+        echo "http code is $http_code"
+        exit 1
+      fi
+      curl -L -o $harbor_path/$harbor_name {{ get .artifact.artifact_url.harbor .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .harbor_version (ne .harbor_version "")
\ No newline at end of file
diff --git a/builtin/roles/capkk/download/curl/helm/tasks/main.yaml b/builtin/roles/capkk/download/curl/helm/tasks/main.yaml
new file mode 100644
index 000000000..dbafa08bf
--- /dev/null
+++ b/builtin/roles/capkk/download/curl/helm/tasks/main.yaml
@@ -0,0 +1,18 @@
+---
+- name: Check binaries for helm
+  tags: ["helm"]
+  command: |
+    artifact_name={{ get .artifact.artifact_url.helm .item | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/helm/{{ .helm_version }}/{{ .item }}
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.helm .item }})
+      if [ $http_code != 200 ]; then
+        echo "http code is $http_code"
+        exit 1
+      fi
+      curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.helm .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .helm_version (ne .helm_version "")
\ No newline at end of file
diff --git a/builtin/roles/capkk/download/curl/keepalived/tasks/main.yaml b/builtin/roles/capkk/download/curl/keepalived/tasks/main.yaml
new file mode 100644
index 000000000..fa1c3a809
--- /dev/null
+++ b/builtin/roles/capkk/download/curl/keepalived/tasks/main.yaml
@@ -0,0 +1,18 @@
+---
+- name: Check binaries for keepalived
+  tags: ["keepalived"]
+  command: |
+    artifact_name={{ get .artifact.artifact_url.keepalived .item | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/image-registry/keepalived/{{ .keepalived_version }}/{{ .item }}
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.keepalived .item }})
+      if [ $http_code != 200 ]; then
+        echo "http code is $http_code"
+        exit 1
+      fi
+      curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.keepalived .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .keepalived_version (ne .keepalived_version "")
\ No newline at end of file
diff --git a/builtin/roles/capkk/download/curl/kube/tasks/main.yaml b/builtin/roles/capkk/download/curl/kube/tasks/main.yaml
new file mode 100644
index 000000000..6ba6f2d55
--- /dev/null
+++ b/builtin/roles/capkk/download/curl/kube/tasks/main.yaml
@@ -0,0 +1,37 @@
+---
+- name: Check binaries for kube
+  tags: ["kube"]
+  command: |
+    kube_path={{ .work_dir }}/kubekey/kube/{{ .kube_version }}/{{ .item }}
+    if [ ! -f $kube_path/kubelet ]; then
+        mkdir -p $kube_path
+        # download online
+        http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.kubelet .item }})
+        if [ $http_code != 200 ]; then
+          echo "http code is $http_code"
+          exit 1
+        fi
+        curl -L -o $kube_path/kubelet {{ get .artifact.artifact_url.kubelet .item }}
+    fi
+    if [ ! -f $kube_path/kubeadm ]; then
+        mkdir -p $kube_path
+        # download online
+        http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.kubeadm .item }})
+        if [ $http_code != 200 ]; then
+          echo "http code is $http_code"
+          exit 1
+        fi
+        curl -L -o $kube_path/kubeadm {{ get .artifact.artifact_url.kubeadm .item }}
+    fi
+    if [ ! -f $kube_path/kubectl ]; then
+        mkdir -p $kube_path
+        # download online
+        http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.kubectl .item }})
+        if [ $http_code != 200 ]; then
+          echo "http code is $http_code"
+          exit 1
+        fi
+        curl -L -o $kube_path/kubectl {{ get .artifact.artifact_url.kubectl .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .kube_version (ne .kube_version "")
\ No newline at end of file
diff --git a/builtin/roles/capkk/download/curl/registry/tasks/main.yaml b/builtin/roles/capkk/download/curl/registry/tasks/main.yaml
new file mode 100644
index 000000000..4cbee57d8
--- /dev/null
+++ b/builtin/roles/capkk/download/curl/registry/tasks/main.yaml
@@ -0,0 +1,18 @@
+---
+- name: Check binaries for registry
+  tags: ["registry"]
+  command: |
+    artifact_name={{ get .artifact.artifact_url.registry .item | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/image-registry/registry/{{ .registry_version }}/{{ .item }}
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.registry .item }})
+      if [ $http_code != 200 ]; then
+        echo "http code is $http_code"
+        exit 1
+      fi
+      curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.registry .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .registry_version (ne .registry_version "")
\ No newline at end of file
diff --git a/builtin/roles/capkk/download/curl/runc/tasks/main.yaml b/builtin/roles/capkk/download/curl/runc/tasks/main.yaml
new file mode 100644
index 000000000..964b5ef16
--- /dev/null
+++ b/builtin/roles/capkk/download/curl/runc/tasks/main.yaml
@@ -0,0 +1,18 @@
+---
+- name: Check binaries for runc
+  tags: ["runc"]
+  command: |
+    artifact_name={{ get .artifact.artifact_url.runc .item | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/runc/{{ .runc_version }}/{{ .item }}
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.runc .item }})
+      if [ $http_code != 200 ]; then
+        echo "http code is $http_code"
+        exit 1
+      fi
+      curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.runc .item }}
+    fi
+  loop: "{{ .artifact.arch | toJson }}"
+  when: and .runc_version (ne .runc_version "")
\ No newline at end of file
diff --git a/builtin/roles/capkk/download/helm/cilium/tasks/main.yaml b/builtin/roles/capkk/download/helm/cilium/tasks/main.yaml
new file mode 100644
index 000000000..5a6df2a1f
--- /dev/null
+++ b/builtin/roles/capkk/download/helm/cilium/tasks/main.yaml
@@ -0,0 +1,12 @@
+---
+- name: Check binaries for cilium
+  tags: ["cilium"]
+  command: |
+    artifact_name={{ .artifact.artifact_url.cilium | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/cni
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      cd $artifact_path && helm pull {{ .artifact.artifact_url.cilium }}
+    fi
+  when: and .cilium_version (ne .cilium_version "")
\ No newline at end of file
diff --git a/builtin/roles/capkk/download/helm/hybridnet/tasks/main.yaml b/builtin/roles/capkk/download/helm/hybridnet/tasks/main.yaml
new file mode 100644
index 000000000..29bd77d59
--- /dev/null
+++ b/builtin/roles/capkk/download/helm/hybridnet/tasks/main.yaml
@@ -0,0 +1,12 @@
+---
+- name: Check binaries for hybridnet
+  tags: ["hybridnet"]
+  command: |
+    artifact_name={{ .artifact.artifact_url.hybridnet | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/cni
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      cd $artifact_path && helm pull {{ .artifact.artifact_url.hybridnet }}
+    fi
+  when: and .hybridnet_version (ne .hybridnet_version "")
\ No newline at end of file
diff --git a/builtin/roles/capkk/download/helm/kubeovn/tasks/main.yaml b/builtin/roles/capkk/download/helm/kubeovn/tasks/main.yaml
new file mode 100644
index 000000000..74f11da35
--- /dev/null
+++ b/builtin/roles/capkk/download/helm/kubeovn/tasks/main.yaml
@@ -0,0 +1,12 @@
+---
+- name: Check binaries for kubeovn
+  tags: ["kubeovn"]
+  command: |
+    artifact_name={{ .artifact.artifact_url.kubeovn | splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/cni
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      cd $artifact_path && helm pull {{ .artifact.artifact_url.kubeovn }}
+    fi
+  when: and .kubeovn_version (ne .kubeovn_version "")
\ No newline at end of file
diff --git a/builtin/roles/capkk/download/helm/nfs-provisioner/tasks/main.yaml b/builtin/roles/capkk/download/helm/nfs-provisioner/tasks/main.yaml
new file mode 100644
index 000000000..50a05b7ef
--- /dev/null
+++ b/builtin/roles/capkk/download/helm/nfs-provisioner/tasks/main.yaml
@@ -0,0 +1,12 @@
+---
+- name: Check binaries for nfs-provisioner
+  tags: ["nfs-provisioner"]
+  command: |
+    artifact_name={{ .artifact.artifact_url.nfs_provisioner |splitList "/" | last }}
+    artifact_path={{ .work_dir }}/kubekey/sc
+    if [ ! -f $artifact_path/$artifact_name ]; then
+      mkdir -p $artifact_path
+      # download online
+      cd $artifact_path && helm pull {{ .artifact.artifact_url.nfs_provisioner }}
+    fi
+  when: and .nfs_provisioner_version (ne .nfs_provisioner_version "")
\ No newline at end of file
diff --git a/builtin/roles/capkk/install/etcd/defaults/main.yaml b/builtin/roles/capkk/install/etcd/defaults/main.yaml
new file mode 100644
index 000000000..6f017bcc1
--- /dev/null
+++ b/builtin/roles/capkk/install/etcd/defaults/main.yaml
@@ -0,0 +1,27 @@
+etcd:
+  #    endpoints: ["https://172.1.1.1:2379"]
+  # etcd binary
+  state: new
+#  env config
+  env:
+    election_timeout: 5000
+    heartbeat_interval: 250
+    compaction_retention: 8
+    snapshot_count: 10000
+    data_dir: /var/lib/etcd
+    token: k8s_etcd
+#    metrics: basic
+#    quota_backend_bytes: 100
+#    max_request_bytes: 100
+#    max_snapshots: 100
+#    max_wals: 5
+#    log_level: info
+#    unsupported_arch: arm64
+#  backup config
+  backup:
+    backup_dir: /var/lib/etcd-backup
+    keep_backup_number: 5
+    etcd_backup_script: "backup.sh"
+    on_calendar: "*-*-* *:00/30:00"
+  performance: false
+  traffic_priority: false
diff --git a/builtin/roles/capkk/install/etcd/files/backup.service b/builtin/roles/capkk/install/etcd/files/backup.service
new file mode 100644
index 000000000..a03f4226a
--- /dev/null
+++ b/builtin/roles/capkk/install/etcd/files/backup.service
@@ -0,0 +1,5 @@
+[Unit]
+Description=Backup ETCD
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/kube-scripts/backup_etcd.sh
diff --git a/builtin/roles/capkk/install/etcd/files/etcd.service b/builtin/roles/capkk/install/etcd/files/etcd.service
new file mode 100644
index 000000000..178c3e246
--- /dev/null
+++ b/builtin/roles/capkk/install/etcd/files/etcd.service
@@ -0,0 +1,18 @@
+[Unit]
+Description=etcd
+After=network.target
+
+[Service]
+User=root
+Type=notify
+Nice=-20
+OOMScoreAdjust=-1000
+EnvironmentFile=/etc/etcd.env
+ExecStart=/usr/local/bin/etcd
+NotifyAccess=all
+RestartSec=10s
+LimitNOFILE=40000
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
diff --git a/builtin/roles/capkk/install/etcd/tasks/backup_etcd.yaml b/builtin/roles/capkk/install/etcd/tasks/backup_etcd.yaml
new file mode 100644
index 000000000..772071b37
--- /dev/null
+++ b/builtin/roles/capkk/install/etcd/tasks/backup_etcd.yaml
@@ -0,0 +1,21 @@
+---
+- name: Sync custom backup etcd script
+  template:
+    src: |
+      {{ .etcd.backup.etcd_backup_script }}
+    dest: /usr/local/bin/kube-scripts/backup-etcd.sh
+    mode: 777
+
+- name: Generate backup etcd service
+  copy:
+    src: backup.service
+    dest: /etc/systemd/system/backup-etcd.service
+
+- name: Generate backup etcd timer
+  template:
+    src: backup.timer
+    dest: /etc/systemd/system/backup-etcd.timer
+
+- name: Enable etcd timer
+  command: |
+    systemctl daemon-reload && systemctl enable --now backup-etcd.timer
diff --git a/builtin/roles/capkk/install/etcd/tasks/install_etcd.yaml b/builtin/roles/capkk/install/etcd/tasks/install_etcd.yaml
new file mode 100644
index 000000000..b50204c8b
--- /dev/null
+++ b/builtin/roles/capkk/install/etcd/tasks/install_etcd.yaml
@@ -0,0 +1,55 @@
+---
+- name: Sync etcd binary to node
+  copy:
+    src: |
+      {{ .work_dir }}/kubekey/etcd/{{ .etcd_version }}/{{ .binary_type.stdout }}/etcd-{{ .etcd_version }}-linux-{{ .binary_type.stdout }}.tar.gz
+    dest: |
+      /tmp/kubekey/etcd-{{ .etcd_version }}-linux-{{ .binary_type.stdout }}.tar.gz
+
+- name: Extract etcd binary
+  command: |
+    tar --strip-components=1 -C /usr/local/bin/ -xvf /tmp/kubekey/etcd-{{ .etcd_version }}-linux-{{ .binary_type.stdout }}.tar.gz \
+      --wildcards etcd-{{ .etcd_version }}-linux-{{ .binary_type.stdout }}/etcd*
+
+- name: Sync ca file to remote
+  copy:
+    src: /etc/kubernetes/pki/ca/tls.crt
+    dest: /etc/ssl/etcd/ssl/ca.crt
+
+- name: Sync etcd cert file to remote
+  copy:
+    src: /etc/kubernetes/pki/etcd/tls.crt
+    dest: /etc/ssl/etcd/ssl/server.crt
+
+- name: Sync etcd key file to remote
+  copy:
+    src: /etc/kubernetes/pki/etcd/tls.key
+    dest: /etc/ssl/etcd/ssl/server.key
+
+- name: Generate etcd env file
+  template:
+    src: etcd.env
+    dest: /etc/etcd.env
+
+- name: Generate etcd systemd service file
+  copy:
+    src: etcd.service
+    dest: /etc/systemd/system/etcd.service
+
+# refer: https://etcd.io/docs/v3.5/tuning/
+- name: Set cpu to performance
+  command: |
+    echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
+  when: .etcd.performance
+
+- name: Set Traffic Priority
+  command: |
+    tc qdisc add dev eth0 root handle 1: prio bands 3
+    tc filter add dev eth0 parent 1: protocol ip prio 1 u32 match ip sport 2380 0xffff flowid 1:1
+    tc filter add dev eth0 parent 1: protocol ip prio 1 u32 match ip dport 2380 0xffff flowid 1:1
+    tc filter add dev eth0 parent 1: protocol ip prio 2 u32 match ip sport 2379 0xffff flowid 1:1
+    tc filter add dev eth0 parent 1: protocol ip prio 2 u32 match ip dport 2379 0xffff flowid 1:1
+  when: .etcd.traffic_priority
+
+- name: Start etcd service
+  command: systemctl daemon-reload && systemctl start etcd && systemctl enable etcd
diff --git a/builtin/roles/capkk/install/etcd/tasks/main.yaml b/builtin/roles/capkk/install/etcd/tasks/main.yaml
new file mode 100644
index 000000000..2c3531d72
--- /dev/null
+++ b/builtin/roles/capkk/install/etcd/tasks/main.yaml
@@ -0,0 +1,27 @@
+---
+- name: Check if etcd is installed
+  ignore_errors: true
+  command: etcd --version
+  run_once: true
+  register: etcd_install_version
+
+- name: Install etcd
+  when: |
+    or (.etcd_install_version.stderr | ne "") (.etcd_install_version.stdout | hasPrefix (printf "etcd Version: %s\n" (.etcd_version | default "" | trimPrefix "v")) | not)
+  block:
+    - name: Init etcd
+      block:
+        - name: Add etcd user
+          command: |
+            useradd -M -c 'Etcd user' -s /sbin/nologin -r etcd || :
+        - name: Create etcd directories
+          command: |
+            if [ ! -d "{{ .item }}" ]; then
+              mkdir -p {{ .item }} && chown -R etcd {{ .item }}
+            fi
+          loop:
+           - "/var/lib/etcd"
+
+    - include_tasks: install_etcd.yaml
+
+    - include_tasks: backup_etcd.yaml
diff --git a/builtin/roles/capkk/install/etcd/templates/backup.sh b/builtin/roles/capkk/install/etcd/templates/backup.sh
new file mode 100644
index 000000000..61e70235a
--- /dev/null
+++ b/builtin/roles/capkk/install/etcd/templates/backup.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+ETCDCTL_PATH='/usr/local/bin/etcdctl'
+ENDPOINTS='https://{{ .internal_ipv4 }}:2379'
+ETCD_DATA_DIR="{{ .etcd.env.data_dir }}"
+BACKUP_DIR="{{ .etcd.backup.backup_dir }}/etcd-$(date +%Y-%m-%d-%H-%M-%S)"
+KEEPBACKUPNUMBER='{{ .etcd.backup.keep_backup_number }}'
+((KEEPBACKNUMBER++))
+
+ETCDCTL_CERT="/etc/ssl/etcd/ssl/server.crt"
+ETCDCTL_KEY="/etc/ssl/etcd/ssl/server.key"
+ETCDCTL_CA_FILE="/etc/ssl/etcd/ssl/ca.crt"
+
+[ ! -d $BACKUP_DIR ] && mkdir -p $BACKUP_DIR
+
+export ETCDCTL_API=2;$ETCDCTL_PATH backup --data-dir $ETCD_DATA_DIR --backup-dir $BACKUP_DIR
+
+sleep 3
+
+{
+export ETCDCTL_API=3;$ETCDCTL_PATH --endpoints="$ENDPOINTS" snapshot save $BACKUP_DIR/snapshot.db \
+                                   --cacert="$ETCDCTL_CA_FILE" \
+                                   --cert="$ETCDCTL_CERT" \
+                                   --key="$ETCDCTL_KEY"
+} > /dev/null
+
+sleep 3
+
+cd $BACKUP_DIR/../ && ls -lt |awk '{if(NR > '$KEEPBACKUPNUMBER'){print "rm -rf "$9}}'|sh
diff --git a/builtin/roles/capkk/install/etcd/templates/backup.timer b/builtin/roles/capkk/install/etcd/templates/backup.timer
new file mode 100644
index 000000000..4b73c6bc2
--- /dev/null
+++ b/builtin/roles/capkk/install/etcd/templates/backup.timer
@@ -0,0 +1,7 @@
+[Unit]
+Description=Timer to backup ETCD
+[Timer]
+OnCalendar={{ .etcd.backup.on_calendar }}
+Unit=backup-etcd.service
+[Install]
+WantedBy=multi-user.target
diff --git a/builtin/roles/capkk/install/etcd/templates/etcd.env b/builtin/roles/capkk/install/etcd/templates/etcd.env
new file mode 100644
index 000000000..ed85a2097
--- /dev/null
+++ b/builtin/roles/capkk/install/etcd/templates/etcd.env
@@ -0,0 +1,57 @@
+ETCD_DATA_DIR={{ .etcd.env.data_dir }}
+ETCD_ADVERTISE_CLIENT_URLS={{ printf "https://%s:2379" .internal_ipv4 }}
+ETCD_INITIAL_ADVERTISE_PEER_URLS={{ printf "https://%s:2380" .internal_ipv4 }}
+ETCD_INITIAL_CLUSTER_STATE={{ .etcd.state }}
+ETCD_LISTEN_CLIENT_URLS={{ printf "https://%s:2379" .internal_ipv4 }},https://127.0.0.1:2379
+ETCD_INITIAL_CLUSTER_TOKEN={{ .etcd.env.token }}
+ETCD_LISTEN_PEER_URLS={{ printf "https://%s:2380" .internal_ipv4 }}
+ETCD_NAME={{ .hostname }}
+ETCD_PROXY=off
+ETCD_ENABLE_V2=true
+{{- $ips := list }}
+{{- range .groups.etcd | default list }}
+    {{- $ips = append $ips (printf "%s=https://%s:2380" (index $.inventory_hosts . "hostname") (index $.inventory_hosts . "internal_ipv4")) }}
+{{- end }}
+ETCD_INITIAL_CLUSTER={{ $ips | join "," }}
+ETCD_ELECTION_TIMEOUT={{ .etcd.env.election_timeout }}
+ETCD_HEARTBEAT_INTERVAL={{ .etcd.env.heartbeat_interval }}
+ETCD_AUTO_COMPACTION_RETENTION={{ .etcd.env.compaction_retention }}
+ETCD_SNAPSHOT_COUNT={{ .etcd.env.snapshot_count }}
+{{- if .etcd.metrics }}
+ETCD_METRICS={{ .etcd.env.metrics }}
+{{- end }}
+{{- if .etcd.env.quota_backend_bytes }}
+ETCD_QUOTA_BACKEND_BYTES={{ .etcd.env.quota_backend_bytes }}
+{{- end }}
+{{- if .etcd.env.max_request_bytes }}
+ETCD_MAX_REQUEST_BYTES={{ .etcd.env.max_request_bytes }}
+{{- end }}
+{{- if .etcd.env.max_snapshots }}
+ETCD_MAX_SNAPSHOTS={{ .etcd.env.max_snapshots }}
+{{- end }}
+{{- if .etcd.env.max_wals }}
+ETCD_MAX_WALS={{ .etcd.env.max_wals }}
+{{- end }}
+{{- if .etcd.env.log_level }}
+ETCD_LOG_LEVEL={{ .etcd.env.log_level }}
+{{- end }}
+{{- if .etcd.env.unsupported_arch }}
+ETCD_UNSUPPORTED_ARCH={{ .etcd.env.unsupported_arch }}
+{{- end }}
+
+# TLS settings
+ETCD_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.crt
+ETCD_CERT_FILE=/etc/ssl/etcd/ssl/server.crt
+ETCD_KEY_FILE=/etc/ssl/etcd/ssl/server.key
+ETCD_CLIENT_CERT_AUTH=true
+
+ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.crt
+ETCD_PEER_CERT_FILE=/etc/ssl/etcd/ssl/server.crt
+ETCD_PEER_KEY_FILE=/etc/ssl/etcd/ssl/server.key
+ETCD_PEER_CLIENT_CERT_AUTH=true
+
+# CLI settings
+ETCDCTL_ENDPOINTS=https://127.0.0.1:2379
+ETCDCTL_CACERT=/etc/ssl/etcd/ssl/ca.crt
+ETCDCTL_CERT=/etc/ssl/etcd/ssl/server.crt
+ETCDCTL_KEY=/etc/ssl/etcd/ssl/server.key
diff --git a/builtin/roles/init/init-artifact/defaults/main.yaml b/builtin/roles/init/init-artifact/defaults/main.yaml
index ed991199d..5e550d664 100644
--- a/builtin/roles/init/init-artifact/defaults/main.yaml
+++ b/builtin/roles/init/init-artifact/defaults/main.yaml
@@ -5,8 +5,6 @@ artifact:
   artifact_file: ""
   # the md5_file of artifact_file.
   artifact_md5: ""
-  # how to generate cert file.support: IfNotPresent, Always
-  gen_cert_policy: IfNotPresent
   artifact_url:
     etcd:
       amd64: |
diff --git a/builtin/roles/init/init-artifact/tasks/main.yaml b/builtin/roles/init/init-artifact/tasks/main.yaml
index 43f69f4c1..4d5610295 100644
--- a/builtin/roles/init/init-artifact/tasks/main.yaml
+++ b/builtin/roles/init/init-artifact/tasks/main.yaml
@@ -28,11 +28,8 @@
           manifests: "{{ .image_manifests | toJson }}"
       when: .image_manifests | default list | len | lt 0
 
-- include_tasks: pki.yaml
-  tags: ["certs"]
-
 - name: Chown work_dir to sudo
   tags: ["always"]
-  ignore_errors: yes
+  ignore_errors: true
   command: |
     chown -R ${SUDO_UID}:${SUDO_GID} {{ .work_dir }}
diff --git a/builtin/roles/init/init-cert/defaults/main.yaml b/builtin/roles/init/init-cert/defaults/main.yaml
new file mode 100644
index 000000000..2a049c9c3
--- /dev/null
+++ b/builtin/roles/init/init-cert/defaults/main.yaml
@@ -0,0 +1,4 @@
+work_dir: /kubekey
+artifact:
+  # how to generate cert file.support: IfNotPresent, Always
+  gen_cert_policy: IfNotPresent
\ No newline at end of file
diff --git a/builtin/roles/init/init-artifact/tasks/pki.yaml b/builtin/roles/init/init-cert/tasks/main.yaml
similarity index 92%
rename from builtin/roles/init/init-artifact/tasks/pki.yaml
rename to builtin/roles/init/init-cert/tasks/main.yaml
index 391595d0d..1f89162fb 100644
--- a/builtin/roles/init/init-artifact/tasks/pki.yaml
+++ b/builtin/roles/init/init-cert/tasks/main.yaml
@@ -50,3 +50,8 @@
     out_cert: |
       {{ .work_dir }}/kubekey/pki/image_registry.crt
   when: and .groups.image_registry (.groups.image_registry | default list | len | lt 0)
+
+#- name: Chown pki to sudo
+#  ignore_errors: yes
+#  command: |
+#    chown -R ${SUDO_UID}:${SUDO_GID} {{ .work_dir }}/kubekey/pki
diff --git a/builtin/roles/install/kubernetes/defaults/main.yaml b/builtin/roles/install/kubernetes/defaults/main.yaml
index 4796709f0..45e2b375a 100644
--- a/builtin/roles/install/kubernetes/defaults/main.yaml
+++ b/builtin/roles/install/kubernetes/defaults/main.yaml
@@ -18,7 +18,7 @@ kubernetes:
     pod_cidr: 10.233.64.0/18
     service_cidr: 10.233.0.0/18
     dns_image: |
-      {{ .k8s_registry }}/coredns/coredns:1.8.6
+      {{ .k8s_registry }}/coredns/coredns:v1.8.6
     dns_cache_image: |
       {{ .dockerio_registry }}/kubesphere/k8s-dns-node-cache:1.22.20
     dns_service_ip: |

From d38f11d573f664599ee400aac2ee96e0fb881e01 Mon Sep 17 00:00:00 2001
From: DingYongliang <1521323439@qq.com>
Date: Tue, 29 Oct 2024 17:17:10 +0800
Subject: [PATCH 2/2] feat: bug fix and add tags for playbook

---
 builtin/playbooks/capkk_bootstrap_ready.yaml  |  2 +-
 builtin/playbooks/capkk_delete_cluster.yaml   |  6 +++---
 .../playbooks/capkk_etcd_binary_install.yaml  | 19 ++-----------------
 .../init-artifact/tasks/download_by_curl.yaml | 14 ++++++++++++++
 .../init-artifact/tasks/download_by_helm.yaml |  4 ++++
 builtin/roles/init/init-cert/tasks/main.yaml  |  8 ++++----
 .../templates/kubeadm/kubeadm-init.v1beta2    |  2 +-
 .../templates/kubeadm/kubeadm-init.v1beta3    |  2 +-
 8 files changed, 30 insertions(+), 27 deletions(-)

diff --git a/builtin/playbooks/capkk_bootstrap_ready.yaml b/builtin/playbooks/capkk_bootstrap_ready.yaml
index a8be54613..994bdeb18 100644
--- a/builtin/playbooks/capkk_bootstrap_ready.yaml
+++ b/builtin/playbooks/capkk_bootstrap_ready.yaml
@@ -25,7 +25,7 @@
     - vars/create_cluster_kubernetes.yaml
   roles:
     - install/cri
-    - install/kubernetes
+    - capkk/install/kubernetes
 
 - name: Certs Distribution
   hosts:
diff --git a/builtin/playbooks/capkk_delete_cluster.yaml b/builtin/playbooks/capkk_delete_cluster.yaml
index 98600bdbc..62eff1664 100644
--- a/builtin/playbooks/capkk_delete_cluster.yaml
+++ b/builtin/playbooks/capkk_delete_cluster.yaml
@@ -3,16 +3,16 @@
   hosts:
     - k8s_cluster
   roles:
-    - capkk/kubernetes
+    - capkk/delete/kubernetes
 
 - name: Delete Network
   hosts:
     - k8s_cluster
   roles:
-    - capkk/network
+    - capkk/delete/network
 
 - name: Delete ETCD
   hosts:
     - etcd
   roles:
-    - capkk/etcd
\ No newline at end of file
+    - capkk/delete/etcd
\ No newline at end of file
diff --git a/builtin/playbooks/capkk_etcd_binary_install.yaml b/builtin/playbooks/capkk_etcd_binary_install.yaml
index c0a18a518..a6e70d684 100644
--- a/builtin/playbooks/capkk_etcd_binary_install.yaml
+++ b/builtin/playbooks/capkk_etcd_binary_install.yaml
@@ -1,5 +1,6 @@
 ---
 - name: ETCD Binary Install Pre-Check
+  when: .kubernetes.etcd.deployment_type | eq "external"
   hosts:
     - etcd
   gather_facts: true
@@ -18,24 +19,8 @@
         fail_msg: "{{ .os.architecture }} is not a known arch"
       register: binary_type
 
-# TODO: 需要讨论一个比较合适的 Install 流程，要统一安装目录等等
-#- name: ETCD Binary Download
-#  hosts:
-#    - etcd
-#  tags: ["etcd"]
-#  vars_files:
-#    - vars/init_artifact.yaml
-#  pre_tasks:
-#    - name: Create Directory
-#      tags: [ "always" ]
-#      command: |
-#        if [ ! -d "{{ .work_dir }}" ]; then
-#          mkdir -p {{ .work_dir }}/kubekey
-#        fi
-#  roles:
-#    - capkk/download/curl/etcd
-
 - name: ETCD Binary Install
+  when: .kubernetes.etcd.deployment_type | eq "external"
   hosts:
     - etcd
   roles:
diff --git a/builtin/roles/init/init-artifact/tasks/download_by_curl.yaml b/builtin/roles/init/init-artifact/tasks/download_by_curl.yaml
index ff1ce4e3e..0397f5a2d 100644
--- a/builtin/roles/init/init-artifact/tasks/download_by_curl.yaml
+++ b/builtin/roles/init/init-artifact/tasks/download_by_curl.yaml
@@ -1,5 +1,6 @@
 ---
 - name: Check binaries for etcd
+  tags: ["etcd"]
   command: |
     artifact_name={{ get .artifact.artifact_url.etcd .item | splitList "/" | last }}
     artifact_path={{ .work_dir }}/kubekey/etcd/{{ .etcd_version }}/{{ .item }}
@@ -17,6 +18,7 @@
   when: and .etcd_version (ne .etcd_version "")
 
 - name: Check binaries for kube
+  tags: ["kube"]
   command: |
     kube_path={{ .work_dir }}/kubekey/kube/{{ .kube_version }}/{{ .item }}
     if [ ! -f $kube_path/kubelet ]; then
@@ -53,6 +55,7 @@
   when: and .kube_version (ne .kube_version "")
 
 - name: Check binaries for cni
+  tags: ["cni"]
   command: |
     artifact_name={{ get .artifact.artifact_url.cni .item | splitList "/" | last }}
     artifact_path={{ .work_dir }}/kubekey/cni/{{ .cni_version }}/{{ .item }}
@@ -70,6 +73,7 @@
   when: and .cni_version (ne .cni_version "")
 
 - name: Check binaries for helm
+  tags: ["helm"]
   command: |
     artifact_name={{ get .artifact.artifact_url.helm .item | splitList "/" | last }}
     artifact_path={{ .work_dir }}/kubekey/helm/{{ .helm_version }}/{{ .item }}
@@ -87,6 +91,7 @@
   when: and .helm_version (ne .helm_version "")
 
 - name: Check binaries for crictl
+  tags: ["crictl"]
   command: |
     artifact_name={{ get .artifact.artifact_url.crictl .item | splitList "/" | last }}
     artifact_path={{ .work_dir }}/kubekey/crictl/{{ .crictl_version }}/{{ .item }}
@@ -104,6 +109,7 @@
   when: and .crictl_version (ne .crictl_version "")
 
 - name: Check binaries for docker
+  tags: ["docker"]
   command: |
     artifact_name={{ get .artifact.artifact_url.docker .item | splitList "/" | last }}
     artifact_path={{ .work_dir }}/kubekey/docker/{{ .docker_version }}/{{ .item }}
@@ -121,6 +127,7 @@
   when: and .docker_version (ne .docker_version "")
 
 - name: Check binaries for cridockerd
+  tags: ["cridockerd"]
   command: |
     artifact_name={{ get .artifact.artifact_url.cridockerd .item | splitList "/" | last }}
     artifact_path={{ .work_dir }}/kubekey/cri-dockerd/{{ .cridockerd_version }}/{{ .item }}
@@ -138,6 +145,7 @@
   when: and .cridockerd_version (ne .docker_version "")
 
 - name: Check binaries for containerd
+  tags: ["containerd"]
   command: |
     artifact_name={{ get .artifact.artifact_url.containerd .item | splitList "/" | last }}
     artifact_path={{ .work_dir }}/kubekey/containerd/{{ .containerd_version }}/{{ .item }}
@@ -155,6 +163,7 @@
   when: and .containerd_version (ne .containerd_version "")
 
 - name: Check binaries for runc
+  tags: ["runc"]
   command: |
     artifact_name={{ get .artifact.artifact_url.runc .item | splitList "/" | last }}
     artifact_path={{ .work_dir }}/kubekey/runc/{{ .runc_version }}/{{ .item }}
@@ -172,6 +181,7 @@
   when: and .runc_version (ne .runc_version "")
 
 - name: Check binaries for calicoctl
+  tags: ["calicoctl"]
   command: |
     artifact_name=calicoctl
     artifact_path={{ .work_dir }}/kubekey/cni/{{ .calico_version }}/{{ .item }}
@@ -189,6 +199,7 @@
   when: and .calico_version (ne .calico_version "")
 
 - name: Check binaries for registry
+  tags: ["registry"]
   command: |
     artifact_name={{ get .artifact.artifact_url.registry .item | splitList "/" | last }}
     artifact_path={{ .work_dir }}/kubekey/image-registry/registry/{{ .registry_version }}/{{ .item }}
@@ -206,6 +217,7 @@
   when: and .registry_version (ne .registry_version "")
 
 - name: Check binaries for docker-compose
+  tags: ["docker-compose"]
   command: |
     compose_name=docker-compose
     compose_path={{ .work_dir }}/kubekey/image-registry/docker-compose/{{ .dockercompose_version }}/{{ .item }}
@@ -218,6 +230,7 @@
   when: and .dockercompose_version (ne .dockercompose_version "")
 
 - name: Check binaries for harbor
+  tags: ["harbor"]
   command: |
     harbor_name={{ get .artifact.artifact_url.harbor .item | splitList "/" | last }}
     harbor_path={{ .work_dir }}/kubekey/image-registry/harbor/{{ .harbor_version }}/{{ .item }}
@@ -235,6 +248,7 @@
   when: and .harbor_version (ne .harbor_version "")
 
 - name: Check binaries for keepalived
+  tags: ["keepalived"]
   command: |
     artifact_name={{ get .artifact.artifact_url.keepalived .item | splitList "/" | last }}
     artifact_path={{ .work_dir }}/kubekey/image-registry/keepalived/{{ .keepalived_version }}/{{ .item }}
diff --git a/builtin/roles/init/init-artifact/tasks/download_by_helm.yaml b/builtin/roles/init/init-artifact/tasks/download_by_helm.yaml
index 0e462983a..254617997 100644
--- a/builtin/roles/init/init-artifact/tasks/download_by_helm.yaml
+++ b/builtin/roles/init/init-artifact/tasks/download_by_helm.yaml
@@ -1,5 +1,6 @@
 ---
 - name: Check binaries for cilium
+  tags: ["cilium"]
   command: |
     artifact_name={{ .artifact.artifact_url.cilium | splitList "/" | last }}
     artifact_path={{ .work_dir }}/kubekey/cni
@@ -11,6 +12,7 @@
   when: and .cilium_version (ne .cilium_version "")
 
 - name: Check binaries for kubeovn
+  tags: ["kubeovn"]
   command: |
     artifact_name={{ .artifact.artifact_url.kubeovn | splitList "/" | last }}
     artifact_path={{ .work_dir }}/kubekey/cni
@@ -22,6 +24,7 @@
   when: and .kubeovn_version (ne .kubeovn_version "")
 
 - name: Check binaries for hybridnet
+  tags: ["hybridnet"]
   command: |
     artifact_name={{ .artifact.artifact_url.hybridnet | splitList "/" | last }}
     artifact_path={{ .work_dir }}/kubekey/cni
@@ -33,6 +36,7 @@
   when: and .hybridnet_version (ne .hybridnet_version "")
 
 - name: Check binaries for nfs_provisioner
+  tags: ["nfs_provisioner"]
   command: |
     artifact_name={{ .artifact.artifact_url.nfs_provisioner |splitList "/" | last }}
     artifact_path={{ .work_dir }}/kubekey/sc
diff --git a/builtin/roles/init/init-cert/tasks/main.yaml b/builtin/roles/init/init-cert/tasks/main.yaml
index 1f89162fb..b54890122 100644
--- a/builtin/roles/init/init-cert/tasks/main.yaml
+++ b/builtin/roles/init/init-cert/tasks/main.yaml
@@ -51,7 +51,7 @@
       {{ .work_dir }}/kubekey/pki/image_registry.crt
   when: and .groups.image_registry (.groups.image_registry | default list | len | lt 0)
 
-#- name: Chown pki to sudo
-#  ignore_errors: yes
-#  command: |
-#    chown -R ${SUDO_UID}:${SUDO_GID} {{ .work_dir }}/kubekey/pki
+- name: Chown pki to sudo
+  ignore_errors: yes
+  command: |
+    chown -R ${SUDO_UID}:${SUDO_GID} {{ .work_dir }}/kubekey/pki
diff --git a/builtin/roles/install/kubernetes/templates/kubeadm/kubeadm-init.v1beta2 b/builtin/roles/install/kubernetes/templates/kubeadm/kubeadm-init.v1beta2
index a3986464f..645cdfd2a 100644
--- a/builtin/roles/install/kubernetes/templates/kubeadm/kubeadm-init.v1beta2
+++ b/builtin/roles/install/kubernetes/templates/kubeadm/kubeadm-init.v1beta2
@@ -8,7 +8,7 @@ etcd:
     imageTag: {{ .kubernetes.etcd.image | splitList ":" | last }}
     serverCertSANs:
     {{- range .groups.etcd | default list }}
-    - https://{{ index $.inventory_hosts . "internal_ipv4" }}:2379
+    - {{ index $.inventory_hosts . "internal_ipv4" }}
     {{- end }}
 {{- else }}
   external:
diff --git a/builtin/roles/install/kubernetes/templates/kubeadm/kubeadm-init.v1beta3 b/builtin/roles/install/kubernetes/templates/kubeadm/kubeadm-init.v1beta3
index ec02d2ff9..cf96ecf06 100644
--- a/builtin/roles/install/kubernetes/templates/kubeadm/kubeadm-init.v1beta3
+++ b/builtin/roles/install/kubernetes/templates/kubeadm/kubeadm-init.v1beta3
@@ -8,7 +8,7 @@ etcd:
     imageTag: {{ .kubernetes.etcd.image | splitList ":" | last }}
     serverCertSANs:
     {{- range .groups.etcd | default list }}
-    - https://{{ index $.inventory_hosts . "internal_ipv4" }}:2379
+    - {{ index $.inventory_hosts . "internal_ipv4" }}
     {{- end }}
 {{- else }}
   external: