mirror of
https://github.com/kubesphere/kubekey.git
synced 2025-12-25 17:12:50 +00:00
Merge c044b8aa26 into 0cd937c4e3
This commit is contained in:
liujian
GitHub
commit
1bd909658e
|
|
@ -36,7 +36,86 @@
|
|||
- etcd
|
||||
gather_facts: true
|
||||
roles:
|
||||
- etcd
|
||||
- role: etcd
|
||||
when:
|
||||
- .etcd.deployment_type | eq "external"
|
||||
|
||||
- hosts:
|
||||
- kube_control_plane
|
||||
tasks:
|
||||
- name: AddNodes | Check if should update apiserver certificates
|
||||
run_once: true
|
||||
add_hostvars:
|
||||
hosts: kube_control_plane
|
||||
vars:
|
||||
need_installed_etcd: >-
|
||||
{{- $needInstalled := list -}}
|
||||
{{- range .groups.etcd -}}
|
||||
{{- if and ((index $.hostvars . "etcd_install_LoadState" "stdout") | eq "not-found") ($.delete_nodes | default list | has . | not) -}}
|
||||
{{- $needInstalled = append $needInstalled . -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{ $needInstalled | toJson }}
|
||||
- name: AddNodes | Update apiserver etcd certificates
|
||||
when:
|
||||
- .need_installed_etcd | fromJson | empty | not
|
||||
- .etcd.deployment_type | eq "external"
|
||||
block:
|
||||
- name: AddNodes | Copy etcd CA certificate to control plane node
|
||||
copy:
|
||||
src: >-
|
||||
{{ .etcd.ca_file }}
|
||||
dest: /etc/kubernetes/pki/etcd/ca.crt
|
||||
- name: AddNodes | Copy etcd client certificate to control plane node
|
||||
copy:
|
||||
src: >-
|
||||
{{ .etcd.cert_file }}
|
||||
dest: /etc/kubernetes/pki/etcd/client.crt
|
||||
- name: AddNodes | Copy etcd client key to control plane node
|
||||
copy:
|
||||
src: >-
|
||||
{{ .etcd.key_file }}
|
||||
dest: /etc/kubernetes/pki/etcd/client.key
|
||||
- name: AddNodes | update ks-apiserver
|
||||
command: |
|
||||
{{- $endpoints := list -}}
|
||||
{{- range .groups.etcd | default list -}}
|
||||
{{- $endpoints = append $endpoints (printf "https://%s:2379" (index $.hostvars . "internal_ipv4")) -}}
|
||||
{{- end -}}
|
||||
ETCD_ENDPOINTS="{{ join "," $endpoints }}"
|
||||
|
||||
if ! grep -q 'ClusterConfiguration' /etc/kubernetes/kubeadm-config.yaml 2>/dev/null; then
|
||||
kubectl get cm kubeadm-config -n kube-system -o=jsonpath='{.data.ClusterConfiguration}' > /etc/kubernetes/kubeadm-config.yaml
|
||||
fi
|
||||
|
||||
awk -v ep="$ETCD_ENDPOINTS" '
|
||||
BEGIN {
|
||||
n = split(ep, arr, ",")
|
||||
for (i = 1; i <= n; i++) {
|
||||
print " - " arr[i]
|
||||
}
|
||||
}
|
||||
' > /etc/kubernetes/kubeadm_new_endpoints.yaml
|
||||
# delete old endpoint
|
||||
sed -i '/^[[:space:]]*endpoints:/{
|
||||
:loop
|
||||
N
|
||||
s/\n[[:space:]]\+-.*//; t loop
|
||||
s/\n[[:space:]]*\n/\n/g
|
||||
P
|
||||
D
|
||||
}' /etc/kubernetes/kubeadm-config.yaml
|
||||
# insert new endpoint
|
||||
sed -i "/^[[:space:]]*endpoints:/r /etc/kubernetes/kubeadm_new_endpoints.yaml" /etc/kubernetes/kubeadm-config.yaml
|
||||
rm /etc/kubernetes/kubeadm_new_endpoints.yaml
|
||||
# update kubeadm-config
|
||||
{{- if .kubernetes.kube_version | semverCompare "<v1.27.0" }}
|
||||
kubeadm config upload from-file --config /etc/kubernetes/kubeadm-config.yaml
|
||||
{{- else }}
|
||||
kubeadm init phase upload-config kubeadm --config /etc/kubernetes/kubeadm-config.yaml
|
||||
{{- end }}
|
||||
# regenerate kube-apiserver
|
||||
kubeadm init phase control-plane apiserver --config /etc/kubernetes/kubeadm-config.yaml
|
||||
|
||||
- hosts:
|
||||
- k8s_cluster
|
||||
|
|
|
|||
|
|
@ -35,7 +35,8 @@
|
|||
- hosts:
|
||||
- etcd
|
||||
roles:
|
||||
- etcd
|
||||
- role: etcd
|
||||
when: .etcd.deployment_type | eq "external"
|
||||
|
||||
# Install the private image registry
|
||||
- hosts:
|
||||
|
|
|
|||
|
|
@ -31,8 +31,10 @@
|
|||
- hosts:
|
||||
- etcd
|
||||
roles:
|
||||
- role: uninstall/etcd
|
||||
when: .delete.etcd
|
||||
- role: etcd/scaling_down
|
||||
when:
|
||||
- .delete.etcd
|
||||
- .etcd.deployment_type | eq "external"
|
||||
|
||||
- hosts:
|
||||
- image_registry
|
||||
|
|
|
|||
|
|
@ -9,12 +9,18 @@
|
|||
gather_facts: true
|
||||
roles:
|
||||
- defaults
|
||||
- precheck
|
||||
|
||||
- hosts:
|
||||
- etcd
|
||||
roles:
|
||||
- role: etcd
|
||||
when:
|
||||
- .delete.etcd
|
||||
- .etcd.deployment_type | eq "external"
|
||||
|
||||
- hosts:
|
||||
- kube_control_plane
|
||||
gather_facts: true
|
||||
tasks:
|
||||
pre_tasks:
|
||||
- name: DeleteNode | Ensure at least one control plane node remains in the cluster
|
||||
run_once: true
|
||||
command: |
|
||||
|
|
@ -28,6 +34,86 @@
|
|||
echo "At least one control plane node must be retained in the cluster." >&2
|
||||
exit 1
|
||||
{{- end }}
|
||||
tasks:
|
||||
- name: DeleteNode | Update etcd certificate for kube_control_plane
|
||||
when:
|
||||
- .delete.etcd
|
||||
- .etcd.deployment_type | eq "external"
|
||||
block:
|
||||
- name: DeleteNode | Check if should update apiserver certificates
|
||||
run_once: true
|
||||
add_hostvars:
|
||||
hosts: kube_control_plane
|
||||
vars:
|
||||
need_uninstall_etcd: >-
|
||||
{{- $needUnInstalled := list -}}
|
||||
{{- range .groups.etcd -}}
|
||||
{{- if $.delete_nodes | default list | has . -}}
|
||||
{{- $needUnInstalled = append $needUnInstalled . -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{ $needUnInstalled | toJson }}
|
||||
- name: DeleteNode | Update apiserver etcd certificates
|
||||
when:
|
||||
- .need_uninstall_etcd | fromJson | empty | not
|
||||
block:
|
||||
- name: DeleteNode | Copy etcd CA certificate to control plane node
|
||||
copy:
|
||||
src: >-
|
||||
{{ .etcd.ca_file }}
|
||||
dest: /etc/kubernetes/pki/etcd/ca.crt
|
||||
- name: DeleteNode | Copy etcd client certificate to control plane node
|
||||
copy:
|
||||
src: >-
|
||||
{{ .etcd.cert_file }}
|
||||
dest: /etc/kubernetes/pki/etcd/client.crt
|
||||
- name: DeleteNode | Copy etcd client key to control plane node
|
||||
copy:
|
||||
src: >-
|
||||
{{ .etcd.key_file }}
|
||||
dest: /etc/kubernetes/pki/etcd/client.key
|
||||
- name: DeleteNode | update ks-apiserver
|
||||
command: |
|
||||
{{- $endpoints := list -}}
|
||||
{{- range .groups.etcd | default list -}}
|
||||
{{- if $.need_uninstall_etcd | fromJson | has . | not -}}
|
||||
{{- $endpoints = append $endpoints (printf "https://%s:2379" (index $.hostvars . "internal_ipv4")) -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
ETCD_ENDPOINTS="{{ join "," $endpoints }}"
|
||||
|
||||
if ! grep -q 'ClusterConfiguration' /etc/kubernetes/kubeadm-config.yaml 2>/dev/null; then
|
||||
kubectl get cm kubeadm-config -n kube-system -o=jsonpath='{.data.ClusterConfiguration}' > /etc/kubernetes/kubeadm-config.yaml
|
||||
fi
|
||||
|
||||
awk -v ep="$ETCD_ENDPOINTS" '
|
||||
BEGIN {
|
||||
n = split(ep, arr, ",")
|
||||
for (i = 1; i <= n; i++) {
|
||||
print " - " arr[i]
|
||||
}
|
||||
}
|
||||
' > /etc/kubernetes/kubeadm_new_endpoints.yaml
|
||||
# delete old endpoint
|
||||
sed -i '/^[[:space:]]*endpoints:/{
|
||||
:loop
|
||||
N
|
||||
s/\n[[:space:]]\+-.*//; t loop
|
||||
s/\n[[:space:]]*\n/\n/g
|
||||
P
|
||||
D
|
||||
}' /etc/kubernetes/kubeadm-config.yaml
|
||||
# insert new endpoint
|
||||
sed -i "/^[[:space:]]*endpoints:/r /etc/kubernetes/kubeadm_new_endpoints.yaml" /etc/kubernetes/kubeadm-config.yaml
|
||||
rm /etc/kubernetes/kubeadm_new_endpoints.yaml
|
||||
# update kubeadm-config
|
||||
{{- if .kubernetes.kube_version | semverCompare "<v1.27.0" }}
|
||||
kubeadm config upload from-file --config /etc/kubernetes/kubeadm-config.yaml
|
||||
{{- else }}
|
||||
kubeadm init phase upload-config kubeadm --config /etc/kubernetes/kubeadm-config.yaml
|
||||
{{- end }}
|
||||
# regenerate kube-apiserver
|
||||
kubeadm init phase control-plane apiserver --config /etc/kubernetes/kubeadm-config.yaml
|
||||
|
||||
- hosts:
|
||||
- k8s_cluster
|
||||
|
|
@ -67,14 +153,6 @@
|
|||
- .delete.dns
|
||||
- .delete_nodes | default list | has .inventory_hostname
|
||||
|
||||
- hosts:
|
||||
- etcd
|
||||
roles:
|
||||
- role: uninstall/etcd
|
||||
when:
|
||||
- .delete.etcd
|
||||
- .delete_nodes | default list | has .inventory_hostname
|
||||
|
||||
- hosts:
|
||||
- image_registry
|
||||
roles:
|
||||
|
|
|
|||
|
|
@ -98,3 +98,4 @@ image_manifests:
|
|||
- docker.io/library/haproxy:2.9.6-alpine
|
||||
- docker.io/plndr/kube-vip:{{ .kubernetes.control_plane_endpoint.kube_vip.image.tag }}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -12,11 +12,6 @@
|
|||
loop:
|
||||
- "{{ .etcd.env.data_dir }}"
|
||||
|
||||
- name: Install | Generate etcd environment configuration file
|
||||
template:
|
||||
src: etcd.env
|
||||
dest: /etc/etcd.env
|
||||
|
||||
- name: Install | Deploy etcd systemd service file
|
||||
copy:
|
||||
src: etcd.service
|
||||
|
|
@ -39,3 +34,7 @@
|
|||
|
||||
- name: Install | Start and enable etcd systemd service
|
||||
command: systemctl daemon-reload && systemctl start etcd && systemctl enable etcd
|
||||
|
||||
- name: Install | Refresh etcd.env to set ETCD_INITIAL_CLUSTER_STATE=existing
|
||||
command: sed -i 's/^ETCD_INITIAL_CLUSTER_STATE=new$/ETCD_INITIAL_CLUSTER_STATE=existing/' /etc/etcd.env
|
||||
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
---
|
||||
- include_tasks: install.yaml
|
||||
|
||||
- include_tasks: backup_service.yaml
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
---
|
||||
dependencies:
|
||||
- role: etcd/prepare
|
||||
|
||||
- role: etcd/upgrade
|
||||
when:
|
||||
- .etcd_install_LoadState.stdout | eq "loaded"
|
||||
- .etcd.etcd_version | semverCompare (printf ">v%s" (index .etcd_install_version "stdout" "etcd Version"))
|
||||
|
||||
- role: etcd/scaling_up
|
||||
when:
|
||||
- .installed_etcd | empty | not
|
||||
- .need_installed_etcd | fromJson | empty | not
|
||||
|
||||
- role: etcd/scaling_down
|
||||
when:
|
||||
- .need_uninstall_etcd | fromJson | empty | not
|
||||
|
||||
- role: etcd/install
|
||||
when:
|
||||
- .etcd_install_LoadState.stdout | eq "not-found"
|
||||
- .need_uninstall_etcd | fromJson | has .inventory_hostname | not
|
||||
|
||||
|
|
@ -6,31 +6,39 @@
|
|||
fail_msg: >-
|
||||
etcd service is installed but not running
|
||||
|
||||
- name: Prepare | Set etcd node parameters
|
||||
- name: Prepare | Gather etcd node state and membership
|
||||
block:
|
||||
- name: Prepare | Identify nodes with installed or missing etcd
|
||||
- name: Prepare | Detect installed, to-install, and to-remove etcd nodes
|
||||
run_once: true
|
||||
add_hostvars:
|
||||
hosts: etcd
|
||||
vars:
|
||||
installed_etcd: >-
|
||||
{{- $needInstalled := list -}}
|
||||
{{- $installed := list -}}
|
||||
{{- range .groups.etcd -}}
|
||||
{{- if (index $.hostvars . "etcd_install_LoadState" "stdout") | eq "loaded" -}}
|
||||
{{- $needInstalled = append $needInstalled . -}}
|
||||
{{- if and ((index $.hostvars . "etcd_install_LoadState" "stdout") | eq "loaded") ($.delete_nodes | default list | has . | not) -}}
|
||||
{{- $installed = append $installed . -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{ $needInstalled | first | default "" }}
|
||||
{{ $installed | first | default "" }}
|
||||
need_installed_etcd: >-
|
||||
{{- $needInstalled := list -}}
|
||||
{{- range .groups.etcd -}}
|
||||
{{- if (index $.hostvars . "etcd_install_LoadState" "stdout") | eq "not-found" -}}
|
||||
{{- if and ((index $.hostvars . "etcd_install_LoadState" "stdout") | eq "not-found") ($.delete_nodes | default list | has . | not) -}}
|
||||
{{- $needInstalled = append $needInstalled . -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{ $needInstalled | toJson }}
|
||||
need_uninstall_etcd: >-
|
||||
{{- $needUnInstalled := list -}}
|
||||
{{- range .groups.etcd -}}
|
||||
{{- if $.delete_nodes | default list | has . -}}
|
||||
{{- $needUnInstalled = append $needUnInstalled . -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{ $needUnInstalled | toJson }}
|
||||
|
||||
- name: Prepare | Check installed etcd version
|
||||
- name: Prepare | Validate installed etcd version
|
||||
when: .etcd_install_LoadState.stdout | eq "loaded"
|
||||
block:
|
||||
- name: Prepare | Ensure target etcd version is not lower than installed version
|
||||
|
|
@ -40,29 +48,32 @@
|
|||
fail_msg: >-
|
||||
Installed etcd version: {{ index .etcd_install_version "stdout" "etcd Version" }} is lower than target etcd version: {{ .etcd.etcd_version }}
|
||||
|
||||
- name: Prepare | Synchronize etcd package to node if new install or upgrade
|
||||
when:
|
||||
- or (.etcd_install_version.error | empty | not) (.etcd.etcd_version | semverCompare (printf ">v%s" (index .etcd_install_version "stdout" "etcd Version")))
|
||||
- name: Prepare | Distribute etcd package for install or upgrade
|
||||
when: >-
|
||||
or
|
||||
(.etcd_install_version.error | empty | not)
|
||||
(.etcd.etcd_version | semverCompare (printf ">v%s" (index .etcd_install_version "stdout" "etcd Version")))
|
||||
block:
|
||||
- name: Prepare | Copy etcd binary package to remote node
|
||||
- name: Prepare | Copy etcd binary package to node
|
||||
copy:
|
||||
src: >-
|
||||
{{ .binary_dir }}/etcd/{{ .etcd.etcd_version }}/{{ .binary_type }}/etcd-{{ .etcd.etcd_version }}-linux-{{ .binary_type }}.tar.gz
|
||||
dest: >-
|
||||
{{ .tmp_dir }}/etcd-{{ .etcd.etcd_version }}-linux-{{ .binary_type }}.tar.gz
|
||||
- name: Prepare | Extract etcd binary package to /usr/local/bin/
|
||||
- name: Prepare | Extract etcd binaries to /usr/local/bin/
|
||||
command: |
|
||||
tar --strip-components=1 -C /usr/local/bin/ -xvf {{ .tmp_dir }}/etcd-{{ .etcd.etcd_version }}-linux-{{ .binary_type }}.tar.gz \
|
||||
--wildcards 'etcd-{{ .etcd.etcd_version }}-linux-{{ .binary_type }}/etcd*'
|
||||
|
||||
- name: Prepare | Synchronize certificates to node for new install or expansion
|
||||
- name: Prepare | Synchronize certificates and etcd.env when changed
|
||||
when: >-
|
||||
or
|
||||
(.etcd_install_version.error | empty | not)
|
||||
(.etcd_install_LoadState.stdout | eq "not-found")
|
||||
(and
|
||||
(.installed_etcd | empty | not)
|
||||
(.need_installed_etcd | fromJson | empty | not)
|
||||
)
|
||||
(.need_uninstall_etcd | fromJson | empty | not)
|
||||
block:
|
||||
- name: Prepare | Copy CA certificate to etcd node
|
||||
copy:
|
||||
|
|
@ -79,3 +90,7 @@
|
|||
src: >-
|
||||
{{ .etcd.key_file }}
|
||||
dest: /etc/ssl/etcd/ssl/server.key
|
||||
- name: Prepare | Render /etc/etcd.env configuration file
|
||||
template:
|
||||
src: etcd.env
|
||||
dest: /etc/etcd.env
|
||||
|
|
@ -1,15 +1,17 @@
|
|||
{{- $ips := list -}}
|
||||
{{- $state := "new" -}}
|
||||
{{- range .groups.etcd | default list -}}
|
||||
{{- $internalIPv4 := index $.hostvars . "internal_ipv4" | default "" -}}
|
||||
{{- $internalIPv6 := index $.hostvars . "internal_ipv6" | default "" -}}
|
||||
{{- if $internalIPv4 | empty | not -}}
|
||||
{{- $ips = append $ips (printf "%s=https://%s:2380" (index $.hostvars . "hostname") $internalIPv4) -}}
|
||||
{{- else if $internalIPv6 | empty | not }}
|
||||
{{- $ips = append $ips (printf "%s=https://%s:2380" (index $.hostvars . "hostname") $internalIPv6) -}}
|
||||
{{- end -}}
|
||||
{{ if index $.hostvars . "etcd_install_LoadState" "stdout" | eq "loaded" -}}
|
||||
{{- $state := "existing" -}}
|
||||
{{- if $.need_uninstall_etcd | fromJson | default list | has . | not -}}
|
||||
{{- $internalIPv4 := index $.hostvars . "internal_ipv4" | default "" -}}
|
||||
{{- $internalIPv6 := index $.hostvars . "internal_ipv6" | default "" -}}
|
||||
{{- if $internalIPv4 | empty | not -}}
|
||||
{{- $ips = append $ips (printf "%s=https://%s:2380" (index $.hostvars . "hostname") $internalIPv4) -}}
|
||||
{{- else if $internalIPv6 | empty | not }}
|
||||
{{- $ips = append $ips (printf "%s=https://%s:2380" (index $.hostvars . "hostname") $internalIPv6) -}}
|
||||
{{- end -}}
|
||||
{{ if index $.hostvars . "etcd_install_LoadState" "stdout" | eq "loaded" -}}
|
||||
{{- $state = "existing" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
ETCD_DATA_DIR={{ .etcd.env.data_dir }}
|
||||
|
|
@ -0,0 +1,165 @@
|
|||
---
|
||||
- name: ScalingDown | Execute actions on etcd nodes scheduled for removal
|
||||
block:
|
||||
- name: ScalingDown | Remove etcd member from cluster
|
||||
run_once: true
|
||||
delegate_to: "{{ .installed_etcd }}"
|
||||
when:
|
||||
- .need_uninstall_etcd | default "[]" | fromJson | empty | not
|
||||
command: |
|
||||
{{- $need_uninstall_etcd_hosts := list -}}
|
||||
{{- range (.need_uninstall_etcd | default "[]" | fromJson) -}}
|
||||
{{- $need_uninstall_etcd_hosts = append $need_uninstall_etcd_hosts (index $.hostvars . "hostname") -}}
|
||||
{{- end -}}
|
||||
for hostname in {{ $need_uninstall_etcd_hosts | join " " }};do
|
||||
# Get the member ID of the node to be removed
|
||||
MEMBER_ID=$(ETCDCTL_API=3 etcdctl \
|
||||
--endpoints=https://localhost:2379 \
|
||||
--cacert=/etc/ssl/etcd/ssl/ca.crt \
|
||||
--cert=/etc/ssl/etcd/ssl/server.crt \
|
||||
--key=/etc/ssl/etcd/ssl/server.key \
|
||||
member list | grep $hostname | awk -F',' '{print $1}')
|
||||
|
||||
if [ -z "$MEMBER_ID" ]; then
|
||||
echo "Member does not exist, skipping removal."
|
||||
exit 0
|
||||
fi
|
||||
echo "Removing member $MEMBER_ID"
|
||||
|
||||
# Remove the member from the etcd cluster
|
||||
ETCDCTL_API=3 etcdctl \
|
||||
--endpoints=https://localhost:2379 \
|
||||
--cacert=/etc/ssl/etcd/ssl/ca.crt \
|
||||
--cert=/etc/ssl/etcd/ssl/server.crt \
|
||||
--key=/etc/ssl/etcd/ssl/server.key \
|
||||
member remove "$MEMBER_ID"
|
||||
|
||||
############################################
|
||||
# Wait for the member ID to disappear from the list (ensure removal has been committed)
|
||||
############################################
|
||||
echo "Waiting for member $MEMBER_ID to disappear from the cluster..."
|
||||
|
||||
for i in $(seq 1 60); do
|
||||
STILL_PRESENT=$(ETCDCTL_API=3 etcdctl \
|
||||
--endpoints=https://localhost:2379 \
|
||||
--cacert=/etc/ssl/etcd/ssl/ca.crt \
|
||||
--cert=/etc/ssl/etcd/ssl/server.crt \
|
||||
--key=/etc/ssl/etcd/ssl/server.key \
|
||||
member list | awk -F',' '{print $1}' | grep -w "$MEMBER_ID" || true)
|
||||
|
||||
if [ -z "$STILL_PRESENT" ]; then
|
||||
echo "Member $MEMBER_ID successfully removed from the cluster."
|
||||
break
|
||||
fi
|
||||
|
||||
sleep 2
|
||||
done
|
||||
|
||||
if [ -n "$STILL_PRESENT" ]; then
|
||||
echo "ERROR: Timeout waiting for member $MEMBER_ID to be removed."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
############################################
|
||||
# Wait for an etcd leader to exist (ensure quorum has recovered)
|
||||
############################################
|
||||
echo "Waiting for etcd leader to be present..."
|
||||
ALL_ENDPOINTS=$(ETCDCTL_API=3 etcdctl \
|
||||
--endpoints=https://localhost:2379 \
|
||||
--cacert=/etc/ssl/etcd/ssl/ca.crt \
|
||||
--cert=/etc/ssl/etcd/ssl/server.crt \
|
||||
--key=/etc/ssl/etcd/ssl/server.key \
|
||||
member list | awk -F',' '{gsub(/^ +| +$/,"",$5); print $5}' | tr '\n' ',' | sed 's/,$//')
|
||||
|
||||
if [ -z "$ALL_ENDPOINTS" ]; then
|
||||
echo "ERROR: Cannot get endpoints from etcd member list"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for i in $(seq 1 60); do
|
||||
# endpoint status text format fields:
|
||||
# endpoint, ID, DB SIZE, IS LEADER, LEADER ID
|
||||
LEADER_LINE=$(ETCDCTL_API=3 etcdctl \
|
||||
--endpoints="$ALL_ENDPOINTS" \
|
||||
--cacert=/etc/ssl/etcd/ssl/ca.crt \
|
||||
--cert=/etc/ssl/etcd/ssl/server.crt \
|
||||
--key=/etc/ssl/etcd/ssl/server.key \
|
||||
endpoint status | awk -F',' '{print $8}' | grep -v "^$" | head -n1)
|
||||
|
||||
if [ -n "$LEADER" ] && [ "$LEADER" != "0" ]; then
|
||||
echo "Leader is present: $LEADER"
|
||||
break
|
||||
fi
|
||||
|
||||
sleep 2
|
||||
done
|
||||
|
||||
if [ -z "$LEADER" ] || [ "$LEADER" = "0" ]; then
|
||||
echo "ERROR: No leader found after member removal."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "ETCD member $MEMBER_ID removed and quorum is stable."
|
||||
done
|
||||
- name: ScalingDown | delete etcd
|
||||
# If need_uninstall_etcd is empty, remove the entire cluster.
|
||||
# If need_uninstall_etcd is not empty, remove only the specified node.
|
||||
when: >-
|
||||
or
|
||||
(.need_uninstall_etcd | default "[]" | fromJson | empty)
|
||||
(.need_uninstall_etcd | default "[]" | fromJson | has .inventory_hostname)
|
||||
block:
|
||||
- name: ScalingDown | Stop and disable the etcd systemd service
|
||||
ignore_errors: true
|
||||
command: |
|
||||
systemctl stop etcd.service
|
||||
systemctl disable etcd.service
|
||||
rm -rf /etc/systemd/system/etcd.service*
|
||||
systemctl daemon-reload
|
||||
systemctl reset-failed etcd.service
|
||||
- name: ScalingDown | Remove traffic prioritization rules for etcd ports
|
||||
when: .etcd.traffic_priority
|
||||
command: |
|
||||
tc filter del dev eth0 parent 1: protocol ip prio 1 u32 match ip sport 2379 0xffff
|
||||
tc filter del dev eth0 parent 1: protocol ip prio 1 u32 match ip sport 2380 0xffff
|
||||
- name: ScalingDown | Delete all etcd data, configuration, and binaries
|
||||
command: |
|
||||
rm -rf {{ .etcd.env.data_dir }}
|
||||
rm -rf /etc/ssl/etcd/
|
||||
rm -rf /etc/etcd.env
|
||||
rm -rf /usr/local/bin/etcd*
|
||||
rm -rf /var/lib/etcd*
|
||||
- name: ScalingDown | Remove backup-etcd timer, service, and backup scripts
|
||||
ignore_errors: true
|
||||
command: |
|
||||
systemctl disable --now backup-etcd.timer
|
||||
rm /etc/systemd/system/backup-etcd.timer
|
||||
rm -rf /etc/systemd/system/backup-etcd.service*
|
||||
rm /usr/local/bin/kube-scripts/backup_etcd.sh
|
||||
systemctl daemon-reexec && systemctl daemon-reload
|
||||
|
||||
- name: ScalingDown | Restart etcd cluster on remaining members
|
||||
when:
|
||||
- .need_uninstall_etcd | default "[]" | fromJson | empty | not
|
||||
- .etcd_install_LoadState.stdout | eq "loaded"
|
||||
- .need_uninstall_etcd | default "[]" | fromJson | has .inventory_hostname | not
|
||||
block:
|
||||
- name: ScalingDown | Restart the etcd service
|
||||
command: |
|
||||
systemctl restart etcd.service
|
||||
- name: ScalingDown | Wait for etcd service to become healthy
|
||||
command: |
|
||||
for ((i=1; i<=12; i++)); do
|
||||
if ETCDCTL_API=3 etcdctl \
|
||||
--endpoints=https://localhost:2379 \
|
||||
--cacert=/etc/ssl/etcd/ssl/ca.crt \
|
||||
--cert=/etc/ssl/etcd/ssl/server.crt \
|
||||
--key=/etc/ssl/etcd/ssl/server.key \
|
||||
endpoint health >/dev/null 2>&1; then
|
||||
echo "✅ etcd is healthy"
|
||||
exit 0
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
echo "❌ etcd did not become healthy within 1 minute"
|
||||
exit 1
|
||||
|
|
@ -1,14 +1,11 @@
|
|||
- name: Expansion | Expand cluster on existing etcd nodes
|
||||
when: .etcd_install_LoadState.stdout | eq "loaded"
|
||||
- name: ScalingUp | Restart etcd cluster on existing members
|
||||
when:
|
||||
- .etcd_install_LoadState.stdout | eq "loaded"
|
||||
block:
|
||||
- name: Expansion | Update /etc/etcd.env configuration file
|
||||
template:
|
||||
src: etcd.env
|
||||
dest: /etc/etcd.env
|
||||
- name: Expansion | Restart etcd service
|
||||
- name: ScalingUp | Restart the etcd service
|
||||
command: |
|
||||
systemctl restart etcd.service
|
||||
- name: Expansion | Verify etcd service becomes healthy within 1 minute
|
||||
- name: ScalingUp | Ensure etcd service becomes healthy
|
||||
command: |
|
||||
for ((i=1; i<=12; i++)); do
|
||||
if ETCDCTL_API=3 etcdctl \
|
||||
|
|
@ -17,16 +14,18 @@
|
|||
--cert=/etc/ssl/etcd/ssl/server.crt \
|
||||
--key=/etc/ssl/etcd/ssl/server.key \
|
||||
endpoint health >/dev/null 2>&1; then
|
||||
echo "✅ etcd is health"
|
||||
echo "✅ etcd is healthy"
|
||||
exit 0
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
echo "❌ etcd etcd is not health in 1 minute"
|
||||
echo "❌ etcd did not become healthy within 1 minute"
|
||||
exit 1
|
||||
|
||||
- name: Expansion | Add new etcd member from non-installed node
|
||||
when: .etcd_install_LoadState.stdout | eq "not-found"
|
||||
- name: ScalingUp | Add new etcd member from a node where etcd is not yet installed
|
||||
when:
|
||||
- .etcd_install_LoadState.stdout | eq "not-found"
|
||||
- .need_installed_etcd | fromJson | has .inventory_hostname
|
||||
delegate_to: "{{ .installed_etcd }}"
|
||||
command: |
|
||||
ETCDCTL_API=3 etcdctl \
|
||||
|
|
@ -34,4 +33,4 @@
|
|||
--cacert=/etc/ssl/etcd/ssl/ca.crt \
|
||||
--cert=/etc/ssl/etcd/ssl/server.crt \
|
||||
--key=/etc/ssl/etcd/ssl/server.key \
|
||||
member add {{ .inventory_hostname }}
|
||||
member add {{ .hostname }}
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
---
|
||||
- include_tasks: prepare.yaml
|
||||
|
||||
- name: ETCD | Upgrade etcd if a newer version is available
|
||||
when:
|
||||
- .etcd_install_LoadState.stdout | eq "loaded"
|
||||
- .etcd.etcd_version | semverCompare (printf ">v%s" (index .etcd_install_version "stdout" "etcd Version"))
|
||||
include_tasks: upgrade.yaml
|
||||
|
||||
- name: ETCD | Expand the etcd cluster by adding new nodes if required
|
||||
when:
|
||||
- .installed_etcd | empty | not
|
||||
- .need_installed_etcd | fromJson | empty | not
|
||||
include_tasks: expansion.yaml
|
||||
|
||||
- name: ETCD | Install etcd and set up the backup service if not already present
|
||||
when: .etcd_install_LoadState.stdout | eq "not-found"
|
||||
block:
|
||||
- include_tasks: install.yaml
|
||||
- include_tasks: backup_service.yaml
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
---
|
||||
- name: ETCD | Completely uninstall the etcd service and remove all related files
|
||||
block:
|
||||
- name: ETCD | Stop and disable the etcd systemd service
|
||||
ignore_errors: true
|
||||
command: |
|
||||
systemctl stop etcd.service
|
||||
systemctl disable etcd.service
|
||||
rm -rf /etc/systemd/system/etcd.service*
|
||||
systemctl daemon-reload
|
||||
systemctl reset-failed etcd.service
|
||||
- name: ETCD | Remove traffic priority rules for etcd ports
|
||||
command: |
|
||||
tc filter del dev eth0 parent 1: protocol ip prio 1 u32 match ip sport 2379 0xffff
|
||||
tc filter del dev eth0 parent 1: protocol ip prio 1 u32 match ip sport 2380 0xffff
|
||||
when: .etcd.traffic_priority
|
||||
- name: ETCD | Delete all etcd data, configuration, and binaries
|
||||
command: |
|
||||
rm -rf {{ .etcd.env.data_dir }}
|
||||
rm -rf /etc/ssl/etcd/
|
||||
rm -rf /etc/etcd.env
|
||||
rm -rf /usr/local/bin/etcd*
|
||||
|
||||
- name: ETCD | Uninstall backup-etcd timer and service, and remove backup scripts
|
||||
ignore_errors: true
|
||||
command: |
|
||||
systemctl disable --now backup-etcd.timer
|
||||
rm /etc/systemd/system/backup-etcd.timer
|
||||
rm -rf /etc/systemd/system/backup-etcd.service*
|
||||
rm /usr/local/bin/kube-scripts/backup_etcd.sh
|
||||
systemctl daemon-reexec && systemctl daemon-reload
|
||||
|
|
@ -2,7 +2,8 @@
|
|||
- name: Kubernetes | Completely reset the node using kubeadm
|
||||
ignore_errors: true
|
||||
command: |
|
||||
kubeadm reset -f
|
||||
# After Kubernetes v1.27.0, the remove-etcd-member phase will automatically clean up /var/lib/etcd
|
||||
kubeadm reset -f {{ if .etcd.deployment_type | eq "external" }}--skip-phases remove-etcd-member{{ end }}
|
||||
|
||||
- name: Kubernetes | Gracefully stop and disable the kubelet service
|
||||
ignore_errors: true
|
||||
|
|
@ -23,5 +24,4 @@
|
|||
rm -rf /var/log/pods/
|
||||
rm -rf /etc/kubernetes/
|
||||
rm -rf ~/.kube/config
|
||||
rm -rf /root/.kube/config
|
||||
rm -rf /var/lib/etcd
|
||||
rm -rf /root/.kube/config
|
||||
Loading…
Reference in New Issue