diff --git a/builtin/core/defaults/config/v1.23.yaml b/builtin/core/defaults/config/v1.23.yaml index 18b24611..441980a4 100644 --- a/builtin/core/defaults/config/v1.23.yaml +++ b/builtin/core/defaults/config/v1.23.yaml @@ -14,7 +14,7 @@ spec: etcd_version: v3.5.4 # ========== image registry ========== # keepalived image tag. Used for load balancing when there are multiple image registry nodes. - # keepalived_version: stable + # keepalived_version: 2.0.20 # ========== image registry: harbor ========== # harbor image tag # harbor_version: v2.6.3 diff --git a/builtin/core/defaults/config/v1.24.yaml b/builtin/core/defaults/config/v1.24.yaml index ddfab449..c80e7fa1 100644 --- a/builtin/core/defaults/config/v1.24.yaml +++ b/builtin/core/defaults/config/v1.24.yaml @@ -14,7 +14,7 @@ spec: etcd_version: v3.5.6 # ========== image registry ========== # keepalived image tag. Used for load balancing when there are multiple image registry nodes. - # keepalived_version: stable + # keepalived_version: v2.0.20 # ========== image registry: harbor ========== # harbor image tag # harbor_version: v2.7.1 diff --git a/builtin/core/defaults/config/v1.25.yaml b/builtin/core/defaults/config/v1.25.yaml index 77501b75..69ebc085 100644 --- a/builtin/core/defaults/config/v1.25.yaml +++ b/builtin/core/defaults/config/v1.25.yaml @@ -14,7 +14,7 @@ spec: etcd_version: v3.5.7 # ========== image registry ========== # keepalived image tag. Used for load balancing when there are multiple image registry nodes. - # keepalived_version: stable + # keepalived_version: 2.0.20 # ========== image registry: harbor ========== # harbor image tag # harbor_version: v2.8.1 diff --git a/builtin/core/defaults/config/v1.26.yaml b/builtin/core/defaults/config/v1.26.yaml index 5a3e227a..8ef52a3a 100644 --- a/builtin/core/defaults/config/v1.26.yaml +++ b/builtin/core/defaults/config/v1.26.yaml @@ -14,7 +14,7 @@ spec: etcd_version: v3.5.8 # ========== image registry ========== # keepalived image tag. Used for load balancing when there are multiple image registry nodes. - # keepalived_version: stable + # keepalived_version: 2.0.20 # ========== image registry: harbor ========== # harbor image tag # harbor_version: v2.9.1 diff --git a/builtin/core/defaults/config/v1.27.yaml b/builtin/core/defaults/config/v1.27.yaml index 321f6f66..be1a385f 100644 --- a/builtin/core/defaults/config/v1.27.yaml +++ b/builtin/core/defaults/config/v1.27.yaml @@ -14,7 +14,7 @@ spec: etcd_version: v3.5.9 # ========== image registry ========== # keepalived image tag. Used for load balancing when there are multiple image registry nodes. - # keepalived_version: stable + # keepalived_version: 2.0.20 # ========== image registry: harbor ========== # harbor image tag # harbor_version: v2.10.1 diff --git a/builtin/core/defaults/config/v1.28.yaml b/builtin/core/defaults/config/v1.28.yaml index cb9abbbc..aadf3439 100644 --- a/builtin/core/defaults/config/v1.28.yaml +++ b/builtin/core/defaults/config/v1.28.yaml @@ -14,7 +14,7 @@ spec: etcd_version: v3.5.9 # ========== image registry ========== # keepalived image tag. Used for load balancing when there are multiple image registry nodes. - # keepalived_version: stable + # keepalived_version: 2.0.20 # ========== image registry: harbor ========== # harbor image tag # harbor_version: v2.10.1 diff --git a/builtin/core/defaults/config/v1.29.yaml b/builtin/core/defaults/config/v1.29.yaml index be8eea3f..bc625415 100644 --- a/builtin/core/defaults/config/v1.29.yaml +++ b/builtin/core/defaults/config/v1.29.yaml @@ -14,7 +14,7 @@ spec: etcd_version: v3.5.10 # ========== image registry ========== # keepalived image tag. Used for load balancing when there are multiple image registry nodes. - # keepalived_version: stable + # keepalived_version: 2.0.20 # ========== image registry: harbor ========== # harbor image tag # harbor_version: v2.10.1 diff --git a/builtin/core/defaults/config/v1.30.yaml b/builtin/core/defaults/config/v1.30.yaml index 453f6345..ea32d24c 100644 --- a/builtin/core/defaults/config/v1.30.yaml +++ b/builtin/core/defaults/config/v1.30.yaml @@ -14,7 +14,7 @@ spec: etcd_version: v3.5.10 # ========== image registry ========== # keepalived image tag. Used for load balancing when there are multiple image registry nodes. - # keepalived_version: stable + # keepalived_version: 2.0.20 # ========== image registry: harbor ========== # harbor image tag # harbor_version: v2.10.1 diff --git a/builtin/core/defaults/config/v1.31.yaml b/builtin/core/defaults/config/v1.31.yaml index 1ed5a61e..fd54e6f3 100644 --- a/builtin/core/defaults/config/v1.31.yaml +++ b/builtin/core/defaults/config/v1.31.yaml @@ -14,7 +14,7 @@ spec: etcd_version: v3.5.11 # ========== image registry ========== # keepalived image tag. Used for load balancing when there are multiple image registry nodes. - # keepalived_version: stable + # keepalived_version: 2.0.20 # ========== image registry: harbor ========== # harbor image tag # harbor_version: v2.10.1 diff --git a/builtin/core/defaults/config/v1.32.yaml b/builtin/core/defaults/config/v1.32.yaml index 3f931275..66161f66 100644 --- a/builtin/core/defaults/config/v1.32.yaml +++ b/builtin/core/defaults/config/v1.32.yaml @@ -14,7 +14,7 @@ spec: etcd_version: v3.5.11 # ========== image registry ========== # keepalived image tag. Used for load balancing when there are multiple image registry nodes. - # keepalived_version: stable + # keepalived_version: 2.0.20 # ========== image registry: harbor ========== # harbor image tag # harbor_version: v2.10.1 diff --git a/builtin/core/defaults/config/v1.33.yaml b/builtin/core/defaults/config/v1.33.yaml index 400c1a16..221babc9 100644 --- a/builtin/core/defaults/config/v1.33.yaml +++ b/builtin/core/defaults/config/v1.33.yaml @@ -14,7 +14,7 @@ spec: etcd_version: v3.5.11 # ========== image registry ========== # keepalived image tag. Used for load balancing when there are multiple image registry nodes. - # keepalived_version: stable + # keepalived_version: 2.0.20 # ========== image registry: harbor ========== # harbor image tag # harbor_version: v2.10.1 diff --git a/builtin/core/playbooks/init_registry.yaml b/builtin/core/playbooks/init_registry.yaml index 63fa56ca..f28b1390 100644 --- a/builtin/core/playbooks/init_registry.yaml +++ b/builtin/core/playbooks/init_registry.yaml @@ -2,10 +2,10 @@ - import_playbook: hook/pre_install.yaml - hosts: - - localhost + - localhost roles: - - init/init-artifact - - init/init-cert + - init/init-artifact + - init/init-cert - hosts: - image_registry diff --git a/builtin/core/roles/init/init-cert/tasks/main.yaml b/builtin/core/roles/init/init-cert/tasks/main.yaml index ffae1a55..5456e1bf 100644 --- a/builtin/core/roles/init/init-cert/tasks/main.yaml +++ b/builtin/core/roles/init/init-cert/tasks/main.yaml @@ -21,10 +21,10 @@ {{- range .groups.etcd | default list -}} {{- $internalIPv4 := index $.hostvars . "internal_ipv4" | default "" -}} {{- $internalIPv6 := index $.hostvars . "internal_ipv6" | default "" -}} - {{- if ne $internalIPv4 "" -}} + {{- if $internalIPv4 | empty | not -}} {{- $ips = append $ips $internalIPv4 -}} {{- end -}} - {{- if ne $internalIPv6 "" -}} + {{- if $internalIPv6 | empty | not -}} {{- $ips = append $ips $internalIPv6 -}} {{- end -}} {{- end -}} @@ -46,13 +46,16 @@ cn: image_registry sans: >- {{- $ips := list -}} + {{- if .image_registry.ha_vip | empty | not -}} + {{- $ips = append $ips .image_registry.ha_vip -}} + {{- end -}} {{- range .groups.image_registry | default list -}} {{- $internalIPv4 := index $.hostvars . "internal_ipv4" | default "" -}} {{- $internalIPv6 := index $.hostvars . "internal_ipv6" | default "" -}} - {{- if ne $internalIPv4 "" -}} + {{- if $internalIPv4 | empty | not -}} {{- $ips = append $ips $internalIPv4 -}} {{- end -}} - {{- if ne $internalIPv6 "" -}} + {{- if $internalIPv6 | empty | not -}} {{- $ips = append $ips $internalIPv6 -}} {{- end -}} {{- end -}} diff --git a/builtin/core/roles/install/image-registry/defaults/main.yaml b/builtin/core/roles/install/image-registry/defaults/main.yaml index 3f26e774..2bbdb1c4 100644 --- a/builtin/core/roles/install/image-registry/defaults/main.yaml +++ b/builtin/core/roles/install/image-registry/defaults/main.yaml @@ -30,7 +30,7 @@ image_registry: nfs_dir: /share/registry storage: filesystem: - rootdir: /var/lib/registry + rootdir: /opt/registry/data # nfs_mount: /repository/registry # if set. will mount rootdirectory to nfs server in nfs_mount. # azure: # accountname: accountname diff --git a/builtin/core/roles/install/image-registry/files/keepalived/healthcheck.sh b/builtin/core/roles/install/image-registry/files/keepalived/healthcheck.sh new file mode 100644 index 00000000..43bd99d6 --- /dev/null +++ b/builtin/core/roles/install/image-registry/files/keepalived/healthcheck.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +nc -zv -w 2 localhost 443 > /dev/null 2>&1 + +if [ $? -eq 0 ]; then + exit 0 +else + exit 1 +fi diff --git a/builtin/core/roles/install/image-registry/tasks/install_docker.yaml b/builtin/core/roles/install/image-registry/tasks/install_docker.yaml index 174fcb95..f54f5582 100644 --- a/builtin/core/roles/install/image-registry/tasks/install_docker.yaml +++ b/builtin/core/roles/install/image-registry/tasks/install_docker.yaml @@ -5,7 +5,7 @@ register: docker_install_version - name: Install docker - when: .docker_install_version.stderr | empty | not) (.docker_install_version.stdout | hasPrefix (printf "Docker version %s," .docker_version) | not) + when: or (.docker_install_version.stderr | empty | not) (.docker_install_version.stdout | hasPrefix (printf "Docker version %s," .docker_version) | not) block: - name: Sync docker binary to remote copy: diff --git a/builtin/core/roles/install/image-registry/tasks/install_harbor.yaml b/builtin/core/roles/install/image-registry/tasks/install_harbor.yaml index 562b2ab0..2b3a7f25 100644 --- a/builtin/core/roles/install/image-registry/tasks/install_harbor.yaml +++ b/builtin/core/roles/install/image-registry/tasks/install_harbor.yaml @@ -2,7 +2,7 @@ - name: Sync harbor package to remote copy: src: >- - {{ .binary_dir }}/image-registry/harbor/{{ .harbor_version }}/{{ .binary_type.stdout }}/harbor-offline-installer-{{ .harbor_version }}.tgz + {{ .binary_dir }}/image-registry/harbor/{{ .harbor_version }}/{{ .binary_type }}/harbor-offline-installer-{{ .harbor_version }}.tgz dest: >- /opt/harbor/{{ .harbor_version }}/harbor-offline-installer-{{ .harbor_version }}.tgz @@ -30,15 +30,6 @@ dest: >- /opt/harbor/{{ .harbor_version }}/harbor/harbor.yml -- name: Generate keepalived docker compose - template: - src: harbor_keepalived.docker-compose - dest: >- - /opt/harbor/{{ .harbor_version }}/harbor/docker-compose-keepalived.yml - when: - - .image_registry.ha_vip | empty | not - - .image_registry_service.stderr | empty | not - - name: Install harbor command: | cd /opt/harbor/{{ .harbor_version }}/harbor && /bin/bash install.sh @@ -50,3 +41,62 @@ - name: Start harbor service command: systemctl daemon-reload && systemctl start harbor.service && systemctl enable harbor.service + +- name: HA harbor sync images + when: + - .image_registry.ha_vip | empty | not + - .groups.image_registry | len | lt 1 + block: + - name: add keepalived service to docker-compose + command: | + KEEPALIVED_SERVICE='# keepalived is generated by kubekey. + keepalived: + image: osixia/keepalived:{{ .keepalived_version }} + container_name: keepalived + command: + - --copy-service + network_mode: "host" + restart: always + dns_search: . + cap_drop: + - ALL + cap_add: + - NET_ADMIN + - NET_BROADCAST + - NET_RAW + - CHOWN + - DAC_OVERRIDE + depends_on: + - proxy + volumes: + - type: bind + source: /opt/keepalived/{{ .keepalived_version }}/keepalived.conf + target: /container/service/keepalived/assets/keepalived.conf + - type: bind + source: /opt/keepalived/{{ .keepalived_version }}/healthcheck.sh + target: /etc/keepalived/healthcheck.sh' + TARGET_FILE="/opt/harbor/{{ .harbor_version }}/harbor/docker-compose.yml" + TMP_FILE="/opt/harbor/{{ .harbor_version }}/harbor/docker-compose.yml.tmp" + awk -v service="$KEEPALIVED_SERVICE" ' + /^services:/ { + print + print service + next + } + { print } + ' "$TARGET_FILE" > "$TMP_FILE" && mv "$TMP_FILE" "$TARGET_FILE" + systemctl restart harbor.service + - name: wait harbor service ready + command: | + if ! timeout 300 bash -c 'while ! nc -zv localhost 443; do sleep 2; done'; then + echo "ERROR: Harbor did not start within 5 minutes!" + exit 1 + fi + - name: sync harbor-replications scripts to remote + template: + src: harbor-replications.sh + dest: /opt/harbor/scripts/harbor-replications.sh + mode: 0755 + - name: execute harbor-replications.sh + command: bash /opt/harbor/scripts/harbor-replications.sh + \ No newline at end of file diff --git a/builtin/core/roles/install/image-registry/tasks/install_keepalived.yaml b/builtin/core/roles/install/image-registry/tasks/install_keepalived.yaml index 74b10aa0..27220825 100644 --- a/builtin/core/roles/install/image-registry/tasks/install_keepalived.yaml +++ b/builtin/core/roles/install/image-registry/tasks/install_keepalived.yaml @@ -1,23 +1,65 @@ --- +- name: Get interface from ha_vip + block: + - name: Get all interface with cidr + command: | + ip -o addr show | awk ' + BEGIN { + printf "[\n"; + first = 1; + } + /inet / && !/ lo|docker|br-|veth/ { + if (!first) { + printf ",\n"; + } + first = 0; + printf " {\n"; + printf " \"interface\": \"%s\",\n", $2; + printf " \"cidr\": \"%s\"\n", $4; + printf " }"; + } + END { + printf "\n]\n"; + } + ' + register: interface + register_type: json + - name: filter interface by ha_vip + set_fact: + ha_vip_interface: >- + {{- $interface := "" }} + {{- range .interface.stdout | default list -}} + {{- if .cidr | ipInCIDR | has $.image_registry.ha_vip -}} + {{- $interface = .interface -}} + {{- end -}} + {{- end -}} + {{ $interface }} + - name: Check if network is exist + assert: + that: .kube_vip_interface | empty + fail_msg: "cannot find network interface to match ha_vip" + - name: Sync keepalived image to remote copy: src: >- - {{ .binary_dir }}/image-registry/keepalived/{{ .keepalived_version }}/{{ .binary_type.stdout }}/keepalived-{{ .keepalived_version }}-linux-{{ .binary_type.stdout }}.tgz + {{ .binary_dir }}/image-registry/keepalived/{{ .keepalived_version }}/{{ .binary_type }}/keepalived-{{ .keepalived_version }}-linux-{{ .binary_type }}.tgz dest: >- - /opt/keepalived/{{ .keepalived_version }}/keepalived-{{ .keepalived_version }}-linux-{{ .binary_type.stdout }}.tgz + /opt/keepalived/{{ .keepalived_version }}/keepalived-{{ .keepalived_version }}-linux-{{ .binary_type }}.tgz - name: Load keeplived image command: | - docker load -i /opt/keepalived/{{ .keepalived_version }}/keepalived-{{ .keepalived_version }}-linux-{{ .binary_type.stdout }}.tgz + docker load -i /opt/keepalived/{{ .keepalived_version }}/keepalived-{{ .keepalived_version }}-linux-{{ .binary_type }}.tgz - name: Sync keeplived config to remote template: - src: keeplived.config + src: keepalived.conf dest: >- - /opt/keeplived/{{ .keepalived_version }}/keepalived.conf + /opt/keepalived/{{ .keepalived_version }}/keepalived.conf + mode: 0664 - name: Sync healthcheck shell to remote - template: - src: keepalived.healthcheck + copy: + src: keepalived/healthcheck.sh dest: >- - /opt/keeplived/{{ .keepalived_version }}/healthcheck.sh + /opt/keepalived/{{ .keepalived_version }}/healthcheck.sh + mode: 0755 diff --git a/builtin/core/roles/install/image-registry/tasks/install_registry.yaml b/builtin/core/roles/install/image-registry/tasks/install_registry.yaml index 08f4986c..a42b23f3 100644 --- a/builtin/core/roles/install/image-registry/tasks/install_registry.yaml +++ b/builtin/core/roles/install/image-registry/tasks/install_registry.yaml @@ -23,9 +23,8 @@ mount -t nfs {{ $internalIPv6 }}:{{ .image_registry.registry.storage.filesystem.nfs_mount }} {{ .image_registry.registry.storage.filesystem.rootdir }} {{- end }} when: - - and .image_registry.registry.storage.filesystem.nfs_mount (ne .image_registry.registry.storage.filesystem.nfs_mount "") + - .image_registry.registry.storage.filesystem.nfs_mount | empty | not - .groups.nfs | default list | len | eq 1 - - .image_registry_service.stderr | empty | not - name: Load registry image command: | @@ -58,9 +57,16 @@ /opt/registry/{{ .registry_version }}/config.yml - name: Register registry service - copy: + template: src: registry.service dest: /etc/systemd/system/registry.service - name: Start registry service command: systemctl daemon-reload && systemctl start registry.service && systemctl enable registry.service + +- name: wait registry service ready + command: | + if ! timeout 300 bash -c 'while ! nc -zv localhost 443; do sleep 2; done'; then + echo "ERROR: Harbor did not start within 5 minutes!" + exit 1 + fi diff --git a/builtin/core/roles/install/image-registry/tasks/load_images.yaml b/builtin/core/roles/install/image-registry/tasks/load_images.yaml index a07cf4ce..8e1ffac8 100644 --- a/builtin/core/roles/install/image-registry/tasks/load_images.yaml +++ b/builtin/core/roles/install/image-registry/tasks/load_images.yaml @@ -36,7 +36,7 @@ done when: .image_registry.type | eq "harbor" -- name: Sync images package to harbor +- name: Sync images package to image_registry tags: ["only_image"] image: push: diff --git a/builtin/core/roles/install/image-registry/tasks/main.yaml b/builtin/core/roles/install/image-registry/tasks/main.yaml index b794737a..3eb426fb 100644 --- a/builtin/core/roles/install/image-registry/tasks/main.yaml +++ b/builtin/core/roles/install/image-registry/tasks/main.yaml @@ -4,7 +4,9 @@ - include_tasks: install_docker_compose.yaml - include_tasks: install_keepalived.yaml - when: .image_registry.ha_vip | empty | not + when: + - .image_registry.ha_vip | empty | not + - .groups.image_registry | len | lt 1 - name: Install harbor when: .image_registry.type | eq "harbor" diff --git a/builtin/core/roles/install/image-registry/templates/harbor-replications.sh b/builtin/core/roles/install/image-registry/templates/harbor-replications.sh new file mode 100644 index 00000000..5afa78d2 --- /dev/null +++ b/builtin/core/roles/install/image-registry/templates/harbor-replications.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +function createRegistries() { +{{- range .groups.image_registry | default list }} + {{- if ne . $.inventory_hostname }} + curl -k -u '{{ printf "%s:%s" $.image_registry.auth.username $.image_registry.auth.password }}' -X POST -H "Content-Type: application/json" "https://{{ $.inventory_hostname }}/api/v2.0/registries" -d "{\"name\": \"{{ . }}\", \"type\": \"harbor\", \"url\":\"https://{{ . }}:7443\", \"credential\": {\"access_key\": \"{{ $.image_registry.auth.username }}\", \"access_secret\": \"{{ $.image_registry.auth.password }}\"}, \"insecure\": true}" + {{- end }} +{{- end }} +} + +function createReplication() { +{{- range $index, $host := .groups.image_registry | default list }} + {{- if ne $host $.inventory_hostname }} + curl -k -u '{{ printf "%s:%s" $.image_registry.auth.username $.image_registry.auth.password }}' -X POST -H "Content-Type: application/json" "https://{{ $.inventory_hostname }}/api/v2.0/replication/policies" -d "{\"name\": \"{{ printf "%s_%s" $.inventory_hostname $host }}\", \"enabled\": true, \"deletion\":true, \"override\":true, \"replicate_deletion\":true, \"dest_registry\":{ \"id\": {{ $index }}, \"name\": \"{{ $host }}\"}, \"trigger\": {\"type\": \"event_based\"}, \"dest_namespace_replace_count\":1 }" + {{- end }} +{{- end }} +} + +createRegistries +createReplication diff --git a/builtin/core/roles/install/image-registry/templates/harbor.service b/builtin/core/roles/install/image-registry/templates/harbor.service index 9219b1e6..60df0119 100644 --- a/builtin/core/roles/install/image-registry/templates/harbor.service +++ b/builtin/core/roles/install/image-registry/templates/harbor.service @@ -5,7 +5,7 @@ Requires=docker.service [Service] Type=simple -ExecStart=/usr/local/bin/docker-compose -p harbor -f /opt/harbor/{{ .harbor_version }}/harbor/docker-compose.yml up{{ if and .image_registry.ha_vip (ne .image_registry.ha_vip "") }} && /usr/local/bin/docker-compose -p harbor -f /opt/harbor/{{ .harbor_version }}/harbor/docker-compose-keepalived.yml up{{ end }} +ExecStart=/usr/local/bin/docker-compose -p harbor -f /opt/harbor/{{ .harbor_version }}/harbor/docker-compose.yml up ExecStop=/usr/local/bin/docker-compose -p harbor down Restart=on-failure [Install] diff --git a/builtin/core/roles/install/image-registry/templates/harbor_keepalived.docker-compose b/builtin/core/roles/install/image-registry/templates/harbor_keepalived.docker-compose deleted file mode 100644 index 49e86c23..00000000 --- a/builtin/core/roles/install/image-registry/templates/harbor_keepalived.docker-compose +++ /dev/null @@ -1,26 +0,0 @@ ---- -version: '2.3' -services: - keepalived: - image: osixia/keepalived: {{ .keepalived_version }} - container_name: keepalived - restart: always - dns_search: . - cap_drop: - - ALL - cap_add: - - CHOWN - - DAC_OVERRIDE - - SETGID - - SETUID - depends_on: - - proxy - volumes: - - type: bind - source: /opt/keeplived/{{ .keepalived_version }}/keepalived.conf - target: /container/service/keepalived/assets/keepalived.conf - - type: bind - source: /opt/keeplived/{{ .keepalived_version }}/healthcheck.sh - target: /etc/keepalived/healthcheck.sh - networks: - - harbor diff --git a/builtin/core/roles/install/image-registry/templates/keepalived.conf b/builtin/core/roles/install/image-registry/templates/keepalived.conf new file mode 100644 index 00000000..e160c620 --- /dev/null +++ b/builtin/core/roles/install/image-registry/templates/keepalived.conf @@ -0,0 +1,30 @@ +vrrp_script healthcheck { + script "/etc/keepalived/healthcheck.sh" + interval 10 + fall 2 + rise 2 + timeout 5 + init_fail +} + global_defs { + script_user root + router_id harbor-ha + enable_script_security +} + vrrp_instance VI_1 { + state {{ if .groups.image_registry | first | eq .inventory_hostname }}MASTER{{ else }}BACKUP{{ end }} + interface {{ .ha_vip_interface }} + virtual_router_id 31 + priority 50 + advert_int 1 + authentication { + auth_type PASS + auth_pass k8s-test + } + virtual_ipaddress { + {{ .image_registry.ha_vip }} + } + track_script { + healthcheck + } + } diff --git a/builtin/core/roles/install/image-registry/templates/keepalived.config b/builtin/core/roles/install/image-registry/templates/keepalived.config deleted file mode 100644 index 36c11c2f..00000000 --- a/builtin/core/roles/install/image-registry/templates/keepalived.config +++ /dev/null @@ -1,31 +0,0 @@ -vrrp_script healthcheck { - script "/etc/keepalived/healthcheck.sh" - interval 10 - fall 2 - rise 2 - timeout 5 - init_fail - } - global_defs { - script_user root - router_id harbor-ha - enable_script_security - lvs_sync_daemon ens3 VI_1 - } - vrrp_instance VI_1 { - state BACKUP - interface ens3 - virtual_router_id 31 - priority 50 - advert_int 1 - authentication { - auth_type PASS - auth_pass k8s-test - } - virtual_ipaddress { - {{ .image_registry.ha_vip }} - } - track_script { - healthcheck - } - } diff --git a/builtin/core/roles/install/image-registry/templates/keepalived.healthcheck b/builtin/core/roles/install/image-registry/templates/keepalived.healthcheck deleted file mode 100644 index c517da8e..00000000 --- a/builtin/core/roles/install/image-registry/templates/keepalived.healthcheck +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -{{- if .image_registry.type | eq "registry" }} -# registry service -service=registry:5000 -{{- else }} -# harbor service -service=harbor:80 -{{- end }} - -nc -zv -w 2 $service > /dev/null 2>&1 - -if [ $? -eq 0 ]; then - exit 0 -else - exit 1 -fi diff --git a/builtin/core/roles/install/image-registry/templates/registry.config b/builtin/core/roles/install/image-registry/templates/registry.config index 64bdf289..a964621a 100644 --- a/builtin/core/roles/install/image-registry/templates/registry.config +++ b/builtin/core/roles/install/image-registry/templates/registry.config @@ -22,9 +22,9 @@ log: # to: # - errors@example.com storage: -{{- if .image_registry.registry.storage.filesystem.rootdirectory | empty | not }} +{{- if .image_registry.registry.storage.filesystem.rootdir | empty | not }} filesystem: - rootdirectory: {{ .image_registry.registry.storage.filesystem.rootdirectory }} + rootdirectory: {{ .image_registry.registry.storage.filesystem.rootdir }} maxthreads: 100 {{- end }} {{- if .image_registry.registry.storage.azure }} @@ -71,13 +71,13 @@ storage: usedualstack: false loglevel: debug {{- end }} - inmemory: # This driver takes no parameters +# inmemory: # This driver takes no parameters delete: enabled: false redirect: disable: false cache: - blobdescriptor: redis + blobdescriptor: inmemory blobdescriptorsize: 10000 maintenance: uploadpurging: @@ -124,7 +124,7 @@ storage: # options: # baseurl: https://example.com/ http: - addr: localhost:5000 + addr: 0.0.0.0:5000 # prefix: /my/nested/registry/ # host: https://myregistryaddress.org:5000 secret: asecretforlocaldevelopment diff --git a/builtin/core/roles/install/image-registry/templates/registry.docker-compose b/builtin/core/roles/install/image-registry/templates/registry.docker-compose index 1d76d926..b6746e77 100644 --- a/builtin/core/roles/install/image-registry/templates/registry.docker-compose +++ b/builtin/core/roles/install/image-registry/templates/registry.docker-compose @@ -20,7 +20,7 @@ services: - type: bind source: /opt/registry/{{ .registry_version }}/config.yml target: /etc/docker/registry/config.yml - port: + ports: - 443:5000 networks: - registry @@ -28,26 +28,28 @@ services: keepalived: image: osixia/keepalived:{{ .keepalived_version }} container_name: keepalived + command: + - --copy-service + network_mode: "host" restart: always dns_search: . cap_drop: - ALL cap_add: + - NET_ADMIN + - NET_BROADCAST + - NET_RAW - CHOWN - DAC_OVERRIDE - - SETGID - - SETUID depends_on: - registry volumes: - type: bind - source: /opt/keeplived/{{ .keepalived_version }}/keepalived.conf + source: /opt/keepalived/{{ .keepalived_version }}/keepalived.conf target: /container/service/keepalived/assets/keepalived.conf - type: bind - source: /opt/keeplived/{{ .keepalived_version }}/healthcheck.sh + source: /opt/keepalived/{{ .keepalived_version }}/healthcheck.sh target: /etc/keepalived/healthcheck.sh - networks: - - registry {{- end }} networks: registry: diff --git a/builtin/core/roles/install/image-registry/templates/registry.service b/builtin/core/roles/install/image-registry/templates/registry.service index f6e7f56c..27061352 100644 --- a/builtin/core/roles/install/image-registry/templates/registry.service +++ b/builtin/core/roles/install/image-registry/templates/registry.service @@ -1,5 +1,5 @@ [Unit] -Description=harbor +Description=registry After=docker.service systemd-networkd.service systemd-resolved.service Requires=docker.service diff --git a/builtin/core/roles/kubernetes/pre-kubernetes/tasks/high-availability/kube_vip.yaml b/builtin/core/roles/kubernetes/pre-kubernetes/tasks/high-availability/kube_vip.yaml index 72c0d5fa..19a3ddbf 100644 --- a/builtin/core/roles/kubernetes/pre-kubernetes/tasks/high-availability/kube_vip.yaml +++ b/builtin/core/roles/kubernetes/pre-kubernetes/tasks/high-availability/kube_vip.yaml @@ -1,22 +1,43 @@ --- -- name: Get network interface for kube_vip - command: | - {{- if .kubernetes.control_plane_endpoint.kube_vip.address | ipFamily | eq "IPv4" }} - ip route | grep '{{ .internal_ipv4 }}' | grep 'proto kernel scope link src' | awk '{print $3}' - {{- else if .kubernetes.control_plane_endpoint.host | ipFamily | eq "IPv6" }} - ip route | grep '{{ .internal_ipv6 }}' | grep 'proto kernel scope link src' | awk '{print $3}' - {{- else }} - echo "kubernetes.control_plane_endpoint.kube_vip.address" should be ipv4 or ipv6 - exit 1 - {{- end }} - register: interface - -- name: Check if network is exist - assert: - that: - - .interface.stderr | empty - - .interface.stdout | empty | not - fail_msg: "cannot find network interface to match kube_vip" +- name: Get interface from kube_vip + block: + - name: Get all interface with cidr + command: | + ip -o addr show | awk ' + BEGIN { + printf "[\n"; + first = 1; + } + /inet / && !/ lo|docker|br-|veth/ { + if (!first) { + printf ",\n"; + } + first = 0; + printf " {\n"; + printf " \"interface\": \"%s\",\n", $2; + printf " \"cidr\": \"%s\"\n", $4; + printf " }"; + } + END { + printf "\n]\n"; + } + ' + register: interface + register_type: json + - name: filter interface by ha_vip + set_fact: + kube_vip_interface: >- + {{- $interface := "" }} + {{- range .interface.stdout | default list -}} + {{- if .cidr | ipInCIDR | has $.kubernetes.control_plane_endpoint.kube_vip.address -}} + {{- $interface = .interface -}} + {{- end -}} + {{- end -}} + {{ $interface }} + - name: Check if network is exist + assert: + that: .kube_vip_interface | empty + fail_msg: "cannot find network interface to match kube_vip" - name: Generate kube_vip manifest template: diff --git a/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubevip/kubevip.ARP b/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubevip/kubevip.ARP index 05979968..5b7fb6ef 100644 --- a/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubevip/kubevip.ARP +++ b/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubevip/kubevip.ARP @@ -14,7 +14,7 @@ spec: - name: port value: "6443" - name: vip_interface - value: {{ .interface.stdout }} + value: {{ .kube_vip_interface }} - name: vip_cidr value: "32" - name: cp_enable diff --git a/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubevip/kubevip.BGP b/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubevip/kubevip.BGP index afece06d..b4be82ba 100644 --- a/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubevip/kubevip.BGP +++ b/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubevip/kubevip.BGP @@ -14,7 +14,7 @@ spec: - name: port value: "6443" - name: vip_interface - value: {{ .interface.stdout }} + value: {{ .kube_vip_interface }} - name: vip_cidr value: "32" - name: cp_enable diff --git a/builtin/core/roles/precheck/env_check/tasks/image_registry.yaml b/builtin/core/roles/precheck/env_check/tasks/image_registry.yaml new file mode 100644 index 00000000..0958e943 --- /dev/null +++ b/builtin/core/roles/precheck/env_check/tasks/image_registry.yaml @@ -0,0 +1,18 @@ +# image_registry is installed by docker_compose +- name: docker_version and dockercompose_version should not be empty + when: .groups.image_registry | empty | not + assert: + that: + - .docker_version | empty | not + - .dockercompose_version | empty | not + msg: >- + "docker_version" and "dockercompose_version" should not be empty + +- name: keepalived_version should not be empty when image_registry is high availability + when: + - .image_registry.ha_vip | empty | not + - .groups.image_registry | len | lt 1 + assert: + that: .keepalived_version | empty | not + msg: >- + "keepalived_version" should not be empty when image_registry is high availability \ No newline at end of file diff --git a/builtin/core/roles/uninstall/image_registry/tasks/keepalived.yaml b/builtin/core/roles/uninstall/image_registry/tasks/keepalived.yaml index 367d4ca2..dcbfa7f3 100644 --- a/builtin/core/roles/uninstall/image_registry/tasks/keepalived.yaml +++ b/builtin/core/roles/uninstall/image_registry/tasks/keepalived.yaml @@ -1,3 +1,8 @@ +- name: Delete arp by kube-vip + command: | + ip neigh show | grep {{ .image_registry.ha_vip }} | awk '{print $1 " dev " $3}' | xargs -r -L1 ip neigh delete + ip -o addr show | grep {{ .image_registry.ha_vip }} | awk '{system("ip addr del "$4" dev "$2)}' + - name: Delete residue keepalived files command: | rm -rf /opt/keepalived/ \ No newline at end of file diff --git a/docs/images/ha-harbor.excalidraw b/docs/images/ha-harbor.excalidraw new file mode 100644 index 00000000..0320872a --- /dev/null +++ b/docs/images/ha-harbor.excalidraw @@ -0,0 +1,784 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://excalidraw.com", + "elements": [ + { + "id": "o93_Uv5XcZbMQM0H30Zkq", + "type": "rectangle", + "x": 868, + "y": 241, + "width": 244, + "height": 59.99999999999999, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7F", + "roundness": { + "type": 3 + }, + "seed": 865016729, + "version": 290, + "versionNonce": 326982455, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "Q-e9pr0XedDsE8hLhtxpo" + }, + { + "id": "_oQQMRTunRyo9Zq43ZnJU", + "type": "arrow" + }, + { + "id": "3wbQCkbDLOFDINm5zzTS-", + "type": "arrow" + }, + { + "id": "LWBWVZxRHzET8YEmPb-f3", + "type": "arrow" + }, + { + "id": "DQu2mOOAST7Qj33GexWWU", + "type": "arrow" + } + ], + "updated": 1750302611172, + "link": null, + "locked": false + }, + { + "id": "Q-e9pr0XedDsE8hLhtxpo", + "type": "text", + "x": 926.0099983215332, + "y": 258.5, + "width": 127.9800033569336, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7G", + "roundness": null, + "seed": 1864202359, + "version": 274, + "versionNonce": 2119579319, + "isDeleted": false, + "boundElements": null, + "updated": 1750302514626, + "link": null, + "locked": false, + "text": "harbor service", + "fontSize": 20, + "fontFamily": 6, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "o93_Uv5XcZbMQM0H30Zkq", + "originalText": "harbor service", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "KcVtlaVjZmNTnyoeRJBfq", + "type": "rectangle", + "x": 486, + "y": 245.5, + "width": 244.00000000000003, + "height": 60.000000000000014, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7H", + "roundness": { + "type": 3 + }, + "seed": 1227906551, + "version": 335, + "versionNonce": 1110807543, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "o1sccqOk7_faggHLFNheO" + }, + { + "id": "L8_xG3dqum2_YKMebNu2u", + "type": "arrow" + }, + { + "id": "3wbQCkbDLOFDINm5zzTS-", + "type": "arrow" + }, + { + "id": "LWBWVZxRHzET8YEmPb-f3", + "type": "arrow" + }, + { + "id": "cO9D1owMBo5ciGKj9M8W7", + "type": "arrow" + } + ], + "updated": 1750302607030, + "link": null, + "locked": false + }, + { + "id": "o1sccqOk7_faggHLFNheO", + "type": "text", + "x": 544.0099983215332, + "y": 263, + "width": 127.9800033569336, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7I", + "roundness": null, + "seed": 1568638743, + "version": 318, + "versionNonce": 1300714231, + "isDeleted": false, + "boundElements": [], + "updated": 1750302560040, + "link": null, + "locked": false, + "text": "harbor service", + "fontSize": 20, + "fontFamily": 6, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "KcVtlaVjZmNTnyoeRJBfq", + "originalText": "harbor service", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "53tMOC8ghJAglooQgD-kW", + "type": "rectangle", + "x": 484, + "y": 126, + "width": 624, + "height": 78, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7J", + "roundness": { + "type": 3 + }, + "seed": 981932439, + "version": 81, + "versionNonce": 428276025, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "neXgNEUOhJo1yjimmCiTH" + }, + { + "id": "L8_xG3dqum2_YKMebNu2u", + "type": "arrow" + }, + { + "id": "_oQQMRTunRyo9Zq43ZnJU", + "type": "arrow" + } + ], + "updated": 1750302564064, + "link": null, + "locked": false + }, + { + "id": "neXgNEUOhJo1yjimmCiTH", + "type": "text", + "x": 731.4399948120117, + "y": 152.5, + "width": 129.12001037597656, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7K", + "roundness": null, + "seed": 1816398713, + "version": 50, + "versionNonce": 212167193, + "isDeleted": false, + "boundElements": null, + "updated": 1750302564064, + "link": null, + "locked": false, + "text": "Load Blanacer", + "fontSize": 20, + "fontFamily": 6, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "53tMOC8ghJAglooQgD-kW", + "originalText": "Load Blanacer", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "edP1PJ_ABStEloX-Syel7", + "type": "rectangle", + "x": 488, + "y": 337, + "width": 244.00000000000003, + "height": 60.000000000000014, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7N", + "roundness": { + "type": 3 + }, + "seed": 271586231, + "version": 393, + "versionNonce": 1107051063, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "WaP95sXUDIWaHJ8dD3VlR" + }, + { + "id": "cO9D1owMBo5ciGKj9M8W7", + "type": "arrow" + } + ], + "updated": 1750302607030, + "link": null, + "locked": false + }, + { + "id": "WaP95sXUDIWaHJ8dD3VlR", + "type": "text", + "x": 541.75, + "y": 354.5, + "width": 136.5, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7O", + "roundness": null, + "seed": 511870167, + "version": 388, + "versionNonce": 1016794583, + "isDeleted": false, + "boundElements": [], + "updated": 1750302602385, + "link": null, + "locked": false, + "text": "storage service", + "fontSize": 20, + "fontFamily": 6, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "edP1PJ_ABStEloX-Syel7", + "originalText": "storage service", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "QnD-8-ujPJygThWv4oRRT", + "type": "rectangle", + "x": 872, + "y": 336, + "width": 244.00000000000003, + "height": 60.000000000000014, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7P", + "roundness": { + "type": 3 + }, + "seed": 303174871, + "version": 384, + "versionNonce": 384514423, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "zvSBobCjWHUzU-fOWcJLi" + }, + { + "id": "DQu2mOOAST7Qj33GexWWU", + "type": "arrow" + } + ], + "updated": 1750302611172, + "link": null, + "locked": false + }, + { + "id": "zvSBobCjWHUzU-fOWcJLi", + "type": "text", + "x": 925.75, + "y": 353.5, + "width": 136.5, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7Q", + "roundness": null, + "seed": 1388798455, + "version": 379, + "versionNonce": 1293090647, + "isDeleted": false, + "boundElements": [], + "updated": 1750302598218, + "link": null, + "locked": false, + "text": "storage service", + "fontSize": 20, + "fontFamily": 6, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "QnD-8-ujPJygThWv4oRRT", + "originalText": "storage service", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "L8_xG3dqum2_YKMebNu2u", + "type": "arrow", + "x": 789.8272747713355, + "y": 204.9990705847769, + "width": 130.79362370312788, + "height": 39.02567346207175, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7R", + "roundness": { + "type": 2 + }, + "seed": 383988983, + "version": 166, + "versionNonce": 2021834489, + "isDeleted": false, + "boundElements": null, + "updated": 1750302564064, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + -130.79362370312788, + 39.02567346207175 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "53tMOC8ghJAglooQgD-kW", + "focus": -0.2888910803214833, + "gap": 7 + }, + "endBinding": { + "elementId": "KcVtlaVjZmNTnyoeRJBfq", + "focus": -0.2441231343283578, + "gap": 5.5 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "_oQQMRTunRyo9Zq43ZnJU", + "type": "arrow", + "x": 862.5126006477127, + "y": 204.99907058477686, + "width": 142.14928077039178, + "height": 33.100855462394435, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7S", + "roundness": { + "type": 2 + }, + "seed": 72016441, + "version": 108, + "versionNonce": 732067801, + "isDeleted": false, + "boundElements": null, + "updated": 1750302564064, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 142.14928077039178, + 33.100855462394435 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "53tMOC8ghJAglooQgD-kW", + "focus": 0.2194976788477558, + "gap": 7 + }, + "endBinding": { + "elementId": "o93_Uv5XcZbMQM0H30Zkq", + "focus": 0.6222604211431018, + "gap": 10 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "3wbQCkbDLOFDINm5zzTS-", + "type": "arrow", + "x": 739.5702440443347, + "y": 258.96307786317976, + "width": 118.85951191133074, + "height": 0.014658463536704858, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7T", + "roundness": { + "type": 2 + }, + "seed": 1405197335, + "version": 135, + "versionNonce": 1501321783, + "isDeleted": false, + "boundElements": null, + "updated": 1750302560193, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 118.85951191133074, + 0.014658463536704858 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "KcVtlaVjZmNTnyoeRJBfq", + "focus": -0.5514950166112959, + "gap": 10 + }, + "endBinding": { + "elementId": "o93_Uv5XcZbMQM0H30Zkq", + "focus": 0.3999999999999996, + "gap": 10 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "LWBWVZxRHzET8YEmPb-f3", + "type": "arrow", + "x": 858.4297559556653, + "y": 279.2973724995128, + "width": 120.65385031474193, + "height": 1.916023220647503, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7U", + "roundness": { + "type": 2 + }, + "seed": 67343543, + "version": 140, + "versionNonce": 1144521719, + "isDeleted": false, + "boundElements": null, + "updated": 1750302574470, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + -120.65385031474193, + 1.916023220647503 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "o93_Uv5XcZbMQM0H30Zkq", + "focus": -0.156993339676499, + "gap": 10 + }, + "endBinding": { + "elementId": "KcVtlaVjZmNTnyoeRJBfq", + "focus": 0.24342244771700097, + "gap": 7.7759056409233835 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "dBCR-0VHhAZrQsuybKfxf", + "type": "text", + "x": 746, + "y": 290, + "width": 110.52000427246094, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7V", + "roundness": null, + "seed": 1033720663, + "version": 68, + "versionNonce": 931739607, + "isDeleted": false, + "boundElements": null, + "updated": 1750302594968, + "link": null, + "locked": false, + "text": "sync images", + "fontSize": 20, + "fontFamily": 6, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "sync images", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "cO9D1owMBo5ciGKj9M8W7", + "type": "arrow", + "x": 610, + "y": 313, + "width": 0, + "height": 15, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7W", + "roundness": { + "type": 2 + }, + "seed": 1108486455, + "version": 11, + "versionNonce": 745405719, + "isDeleted": false, + "boundElements": null, + "updated": 1750302607030, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 15 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "KcVtlaVjZmNTnyoeRJBfq", + "focus": -0.016393442622950786, + "gap": 7.5 + }, + "endBinding": { + "elementId": "edP1PJ_ABStEloX-Syel7", + "focus": 0, + "gap": 9 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "DQu2mOOAST7Qj33GexWWU", + "type": "arrow", + "x": 1004, + "y": 311, + "width": 0, + "height": 17, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7X", + "roundness": { + "type": 2 + }, + "seed": 572285047, + "version": 11, + "versionNonce": 1740299351, + "isDeleted": false, + "boundElements": null, + "updated": 1750302611172, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 17 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "o93_Uv5XcZbMQM0H30Zkq", + "focus": -0.11475409836065573, + "gap": 10 + }, + "endBinding": { + "elementId": "QnD-8-ujPJygThWv4oRRT", + "focus": 0.08196721311475404, + "gap": 8 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + } + ], + "appState": { + "gridSize": 20, + "gridStep": 5, + "gridModeEnabled": false, + "viewBackgroundColor": "#ffffff", + "lockedMultiSelections": {} + }, + "files": {} +} \ No newline at end of file diff --git a/docs/images/ha-harbor.png b/docs/images/ha-harbor.png new file mode 100644 index 00000000..740d530e Binary files /dev/null and b/docs/images/ha-harbor.png differ diff --git a/docs/images/ha-registry.excalidraw b/docs/images/ha-registry.excalidraw new file mode 100644 index 00000000..4a015256 --- /dev/null +++ b/docs/images/ha-registry.excalidraw @@ -0,0 +1,551 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://excalidraw.com", + "elements": [ + { + "id": "o93_Uv5XcZbMQM0H30Zkq", + "type": "rectangle", + "x": 868, + "y": 241, + "width": 244, + "height": 59.99999999999999, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7F", + "roundness": { + "type": 3 + }, + "seed": 865016729, + "version": 292, + "versionNonce": 497225262, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "Q-e9pr0XedDsE8hLhtxpo" + }, + { + "id": "_oQQMRTunRyo9Zq43ZnJU", + "type": "arrow" + }, + { + "id": "DQu2mOOAST7Qj33GexWWU", + "type": "arrow" + } + ], + "updated": 1750924036450, + "link": null, + "locked": false + }, + { + "id": "Q-e9pr0XedDsE8hLhtxpo", + "type": "text", + "x": 921.1600036621094, + "y": 258.5, + "width": 137.67999267578125, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7G", + "roundness": null, + "seed": 1864202359, + "version": 282, + "versionNonce": 697610606, + "isDeleted": false, + "boundElements": null, + "updated": 1750924050371, + "link": null, + "locked": false, + "text": "registry service", + "fontSize": 20, + "fontFamily": 6, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "o93_Uv5XcZbMQM0H30Zkq", + "originalText": "registry service", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "KcVtlaVjZmNTnyoeRJBfq", + "type": "rectangle", + "x": 486, + "y": 245.5, + "width": 244.00000000000003, + "height": 60.000000000000014, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7H", + "roundness": { + "type": 3 + }, + "seed": 1227906551, + "version": 337, + "versionNonce": 1529332210, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "o1sccqOk7_faggHLFNheO" + }, + { + "id": "L8_xG3dqum2_YKMebNu2u", + "type": "arrow" + }, + { + "id": "cO9D1owMBo5ciGKj9M8W7", + "type": "arrow" + } + ], + "updated": 1750924036450, + "link": null, + "locked": false + }, + { + "id": "o1sccqOk7_faggHLFNheO", + "type": "text", + "x": 539.1600036621094, + "y": 263, + "width": 137.67999267578125, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7I", + "roundness": null, + "seed": 1568638743, + "version": 328, + "versionNonce": 511754546, + "isDeleted": false, + "boundElements": [], + "updated": 1750924033996, + "link": null, + "locked": false, + "text": "registry service", + "fontSize": 20, + "fontFamily": 6, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "KcVtlaVjZmNTnyoeRJBfq", + "originalText": "registry service", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "53tMOC8ghJAglooQgD-kW", + "type": "rectangle", + "x": 484, + "y": 126, + "width": 624, + "height": 78, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7J", + "roundness": { + "type": 3 + }, + "seed": 981932439, + "version": 81, + "versionNonce": 428276025, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "neXgNEUOhJo1yjimmCiTH" + }, + { + "id": "L8_xG3dqum2_YKMebNu2u", + "type": "arrow" + }, + { + "id": "_oQQMRTunRyo9Zq43ZnJU", + "type": "arrow" + } + ], + "updated": 1750302564064, + "link": null, + "locked": false + }, + { + "id": "neXgNEUOhJo1yjimmCiTH", + "type": "text", + "x": 731.4399948120117, + "y": 152.5, + "width": 129.12001037597656, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7K", + "roundness": null, + "seed": 1816398713, + "version": 50, + "versionNonce": 212167193, + "isDeleted": false, + "boundElements": null, + "updated": 1750302564064, + "link": null, + "locked": false, + "text": "Load Blanacer", + "fontSize": 20, + "fontFamily": 6, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "53tMOC8ghJAglooQgD-kW", + "originalText": "Load Blanacer", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "edP1PJ_ABStEloX-Syel7", + "type": "rectangle", + "x": 488, + "y": 383, + "width": 628, + "height": 60.000000000000014, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7N", + "roundness": { + "type": 3 + }, + "seed": 271586231, + "version": 458, + "versionNonce": 1441604658, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "WaP95sXUDIWaHJ8dD3VlR" + }, + { + "id": "cO9D1owMBo5ciGKj9M8W7", + "type": "arrow" + }, + { + "id": "DQu2mOOAST7Qj33GexWWU", + "type": "arrow" + } + ], + "updated": 1750924054994, + "link": null, + "locked": false + }, + { + "id": "WaP95sXUDIWaHJ8dD3VlR", + "type": "text", + "x": 733.75, + "y": 400.5, + "width": 136.5, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7O", + "roundness": null, + "seed": 511870167, + "version": 452, + "versionNonce": 81645042, + "isDeleted": false, + "boundElements": [], + "updated": 1750924054994, + "link": null, + "locked": false, + "text": "storage service", + "fontSize": 20, + "fontFamily": 6, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "edP1PJ_ABStEloX-Syel7", + "originalText": "storage service", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "L8_xG3dqum2_YKMebNu2u", + "type": "arrow", + "x": 789.8336419760351, + "y": 204.99907058477686, + "width": 130.72171598152477, + "height": 39.02567346207178, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7R", + "roundness": { + "type": 2 + }, + "seed": 383988983, + "version": 167, + "versionNonce": 798245042, + "isDeleted": false, + "boundElements": null, + "updated": 1750924028531, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + -130.72171598152477, + 39.02567346207178 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "53tMOC8ghJAglooQgD-kW", + "focus": -0.2888910803214833, + "gap": 7 + }, + "endBinding": { + "elementId": "KcVtlaVjZmNTnyoeRJBfq", + "focus": -0.2441231343283578, + "gap": 5.5 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "_oQQMRTunRyo9Zq43ZnJU", + "type": "arrow", + "x": 862.5126006477127, + "y": 204.99907058477686, + "width": 142.14928077039178, + "height": 33.100855462394435, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7S", + "roundness": { + "type": 2 + }, + "seed": 72016441, + "version": 108, + "versionNonce": 732067801, + "isDeleted": false, + "boundElements": null, + "updated": 1750302564064, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 142.14928077039178, + 33.100855462394435 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "53tMOC8ghJAglooQgD-kW", + "focus": 0.2194976788477558, + "gap": 7 + }, + "endBinding": { + "elementId": "o93_Uv5XcZbMQM0H30Zkq", + "focus": 0.6222604211431018, + "gap": 10 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "cO9D1owMBo5ciGKj9M8W7", + "type": "arrow", + "x": 655.6624775795378, + "y": 307.58611094053396, + "width": 103.19328237376556, + "height": 74.33503752082942, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7W", + "roundness": { + "type": 2 + }, + "seed": 1108486455, + "version": 120, + "versionNonce": 1804812210, + "isDeleted": false, + "boundElements": null, + "updated": 1750924054995, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 103.19328237376556, + 74.33503752082942 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "KcVtlaVjZmNTnyoeRJBfq", + "focus": -0.016393442622950786, + "gap": 7.5 + }, + "endBinding": { + "elementId": "edP1PJ_ABStEloX-Syel7", + "focus": 0, + "gap": 9 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "DQu2mOOAST7Qj33GexWWU", + "type": "arrow", + "x": 966.2341709724777, + "y": 303.90007395282873, + "width": 99.30046503552558, + "height": 78.16233299063248, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b7X", + "roundness": { + "type": 2 + }, + "seed": 572285047, + "version": 91, + "versionNonce": 1177844082, + "isDeleted": false, + "boundElements": null, + "updated": 1750924054995, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + -99.30046503552558, + 78.16233299063248 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "o93_Uv5XcZbMQM0H30Zkq", + "focus": -0.11475409836065573, + "gap": 10 + }, + "endBinding": { + "elementId": "edP1PJ_ABStEloX-Syel7", + "focus": 0.07278735375547067, + "gap": 8 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + } + ], + "appState": { + "gridSize": 20, + "gridStep": 5, + "gridModeEnabled": false, + "viewBackgroundColor": "#ffffff", + "lockedMultiSelections": {} + }, + "files": {} +} \ No newline at end of file diff --git a/docs/images/ha-registry.png b/docs/images/ha-registry.png new file mode 100644 index 00000000..5eae8262 Binary files /dev/null and b/docs/images/ha-registry.png differ diff --git a/docs/zh/core/architecture.md b/docs/zh/core/architecture.md index 083e1a1f..6a68f597 100644 --- a/docs/zh/core/architecture.md +++ b/docs/zh/core/architecture.md @@ -47,6 +47,8 @@ precheck 集群安装前，对集群节点进行检查是否满足集群安装 - **containerd 版本检查**: 当使用 containerd 作为容器管理器时，验证 containerd 版本是否满足最低版本要求 **nfs_precheck**: NFS 存储检查，包括： - **NFS 服务器数量检查**: 验证集群中只能有一个 NFS 服务器节点，确保 NFS 服务部署的唯一性 +**image_registry_precheck**: 镜像仓库检查，包括： +- **镜像仓库必要软件检查**: 需检查 `docker_version` 和 `dockercompose_version` 均已配置且不为空。镜像仓库通过 docker_compose 进行安装，缺少必要软件会导致安装失败。 ## init @@ -61,11 +63,11 @@ init 阶段负责准备和构建集群安装所需的所有资源，包括： install 阶段是 KubeKey 的核心安装阶段，负责在集群节点上实际部署和配置 Kubernetes 集群，包括： -**install nfs**: 为 `nfs` 组中的节点安装nfs服务。 -**install image_registry**: 为 `image_registry` 组中的节点安装镜像仓库。目前支持两种类型的镜像仓库：harbor，registry。 -**install etcd**: 为 `etcd` 组中的节点安装etcd。 -**install cri**: 为 `k8s_cluster` 组中的节点安装cri。目前支持两种CRI：docker，containerd。 -**kubernetes_install**: 为 `k8s_cluster` 组中的节点安装kubernetes。 +**install nfs**: 为 `nfs` 组中的节点安装nfs服务。 +**install image_registry**: 为 `image_registry` 组中的节点安装镜像仓库。目前支持两种类型的镜像仓库：harbor，registry。 +**install etcd**: 为 `etcd` 组中的节点安装etcd。 +**install cri**: 为 `k8s_cluster` 组中的节点安装cri。目前支持两种CRI：docker，containerd。 +**kubernetes_install**: 为 `k8s_cluster` 组中的节点安装kubernetes。 **install helm**: 为已安装好的kubernetes集群安装额外的helm 应用。包含：CNI（calico，cilium，flannel，hybridnet，kubeovn，multus） @@ -78,6 +80,6 @@ post_hook 阶段在集群安装完成后执行，负责集群的最终配置和 2. 设置脚本文件权限为 0755 3. 遍历每个远程节点上 `/etc/kubekey/scripts/` 目录下所有 `post_install_*.sh` 文件，并执行该脚本文件 - > work_dir: 工作目录，默认当前命令执行目录。 - > inventory_hostname: Inventory.yaml 文件中定义的host对应的名称。 + > **work_dir**: 工作目录，默认当前命令执行目录。 + > **inventory_hostname**: Inventory.yaml 文件中定义的host对应的名称。 diff --git a/docs/zh/core/image_registry.md b/docs/zh/core/image_registry.md new file mode 100644 index 00000000..93aabcfc --- /dev/null +++ b/docs/zh/core/image_registry.md @@ -0,0 +1,245 @@ +# image_registry + +image_registry允许用户安装镜像仓库。支持harbor和registry两种镜像仓库 + +## requirement + +- 一台或多台运行兼容 deb/rpm 的 Linux 操作系统的计算机；例如：Ubuntu 或 CentOS。 +- 每台机器 8 GB 以上的内存，内存不足时应用会受限制。 +- 用作控制平面节点的计算机上至少有 4 个 CPU。 +- 集群中所有计算机之间具有完全的网络连接。你可以使用公共网络或专用网络 +- 使用本地存储时。计算机需要100G高速存储的磁盘空间。 + +## 安装harbor + +### 构建Inventory +```yaml +apiVersion: kubekey.kubesphere.io/v1 +kind: Inventory +metadata: + name: default +spec: + hosts: # your can set all nodes here. or set nodes on special groups. +# node1: +# connector: +# type: ssh +# host: node1 +# port: 22 +# user: root +# password: 123456 + groups: + # all kubernetes nodes. + k8s_cluster: + groups: + - kube_control_plane + - kube_worker + # control_plane nodes + kube_control_plane: + hosts: + - localhost + # worker nodes + kube_worker: + hosts: + - localhost + # etcd nodes when etcd_deployment_type is external + etcd: + hosts: + - localhost + image_registry: + hosts: + - localhost + # nfs nodes for registry storage. and kubernetes nfs storage +# nfs: +# hosts: +# - localhost + +``` +需设置 `image_registry` 组 + +### 安装 +harbor是默认安装的镜像仓库 +1. 安装前检查 + ```shell + kk precheck image_registry -i inventory.yaml --set harbor_version=v2.10.1,docker_version=24.0.7,dockercompose_version=v2.20.3 + ``` +2. 安装 +- 单独安装 +`image_registry` 可以脱离集群单独进行安装。 + ```shell + kk init registry -i inventory.yaml --set harbor_version=v2.10.1,docker_version=24.0.7,dockercompose_version=v2.20.3 + ``` + +- 在创建集群时，自动安装 +在创建集群时，会检测 `image_registry` 节点是否安装了harbor, 没有安装时会自动根据配置安装harbor。 + ```shell + kk create cluster -i inventory.yaml --set harbor_version=v2.10.1,docker_version=24.0.7, dockercompose_version=v2.20.3 + ``` + +### harbor高可用 + +harbor高可用有两种实现方式。 + +1. 每个harbor共享同一个存储服务。 +官方做法，适用于在kubernetes集群中安装。需要独立部署PostgreSQL 和 Redis 服务 +参考：https://goharbor.io/docs/edge/install-config/harbor-ha-helm/ + +2. 每个harbor有单独的存储服务。 +kubekey的做法，适用于在服务器上安装。 +![ha-harbor](../../images/ha-harbor.png) +- load balancer: 通过docker compose部署keepalived服务实现。 +- harbor service: 通过docker compose部署harbor实现。 +- sync images: 通过harbor的复制管理功能实现。 + +安装示例: +```shell +./kk init registry -i inventory.yaml --set image_registry.ha_vip=xx.xx.xx.xx --set harbor_version=v2.10.1,docker_version=24.0.7,dockercompose_version=v2.20.3 --set keepalived_version=2.0.20,artifact.artifact_url.keepalived.amd64=keepalived-2.0.20-linux-amd64.tgz +``` +1. 在inventory中的image_registry 组中设置多个节点 + +2. 设置变量`image_registry.ha_vip` ha_vip 是负载均衡的入口 + +3. 设置变量 `keepalived_version` 和 `artifact.artifact_url.keepalived.amd64` keepalived 是用于负载均衡的镜像。目前kubekey并未提供下载地址，可通过手动打包的方式来实现。 + ```shell + # download keepalived images + docker pull osixia/keepalived:{{ .keepalived_version }} + # package image + docker save -o keepalived-{{ .keepalived_version }}-linux-{{ .binary_type }}.tgz osixia/ keepalived:{{ .keepalived_version }} + # move image to workdir + mv keepalived-{{ .keepalived_version }}-linux-{{ .binary_type }}.tgz {{ .binary_dir }}/ image-registry/keepalived/{{ .keepalived_version }}/{{ .binary_type }}/ + ``` + `binary_type`: 是机器的架构（目前支持amd64和arm64，可通过 `gather_fact` 自动获取） + `binary_dir`: 软件包存放地址，通常为: `{{ .work_dir}}/kubekey` + +4. 设置变量 `harbor_version`, `docker_version` 和 `dockercompose_version`。harbor通过docker-compose进行安装。 + + +## 安装registry + +### 构建Inventory +```yaml +apiVersion: kubekey.kubesphere.io/v1 +kind: Inventory +metadata: + name: default +spec: + hosts: # your can set all nodes here. or set nodes on special groups. +# node1: +# connector: +# type: ssh +# host: node1 +# port: 22 +# user: root +# password: 123456 + groups: + # all kubernetes nodes. + k8s_cluster: + groups: + - kube_control_plane + - kube_worker + # control_plane nodes + kube_control_plane: + hosts: + - localhost + # worker nodes + kube_worker: + hosts: + - localhost + # etcd nodes when etcd_deployment_type is external + etcd: + hosts: + - localhost + image_registry: + hosts: + - localhost + # nfs nodes for registry storage. and kubernetes nfs storage +# nfs: +# hosts: +# - localhost + +``` +### 构建 registry 镜像包 +kubekey暂未提供registry的离线镜像包地址，需通过手动打包的方式来实现。 + ```shell + # download registry images + docker pull registry:{{ .registry_version }} + # package image + docker save -o registry-{{ .registry_version }}-linux-{{ .binary_type }}.tgz registry:{{ .registry_version }} + # move image to workdir + mv registry-{{ .registry_version }}-linux-{{ .binary_type }}.tgz {{ .binary_dir }}/ image-registry/registry/{{ .registry_version }}/{{ .binary_type }}/ + ``` + `binary_type`: 是机器的架构（目前支持amd64和arm64，可通过 `gather_fact` 自动获取） + `binary_dir`: 软件包存放地址，通常为: `{{ .work_dir}}/kubekey` + +### 安装 +安装registry需要设置`image_registry.type`值为`registry` +1. 安装前检查 + ```shell + kk precheck image_registry -i inventory.yaml --set image_registry.type=registry --set registry_version=2.8.3,docker_version=24.0.7,dockercompose_version=v2.20.3 + ``` +2. 安装 +- 单独安装 +`image_registry` 可以脱离集群单独进行安装。 + ```shell + kk init registry -i inventory.yaml --set image_registry.type=registry --set registry_version=2.8.3,docker_version=24.0.7,dockercompose_version=v2.20.3 --set artifact.artifact_url.registry.amd64=registry-2.8.3-linux.amd64.tgz + ``` + +- 在创建集群时，自动安装 +在创建集群时，会检测 `image_registry` 节点是否安装了harbor, 没有安装时会自动根据配置安装harbor。 + ```shell + kk create cluster -i inventory.yaml --set image_registry.type=registry --set registry_version=2.8.3,docker_version=24.0.7,dockercompose_version=v2.20.3 --set artifact.artifact_url.registry.amd64=registry-2.8.3-linux.amd64.tgz + ``` + +### registry高可用 + +![ha-registry](../../images/ha-registry.png) +- load balancer: 通过docker compose部署keepalived服务实现。 +- registry service: 通过docker compose部署registry实现。 +- storage service: registry 高可用可通过共享存储的方式来实现。registry 支持多种存储后端，常见的有： + - **filesystem**: 本地存储。默认情况下，registry 使用本地磁盘存储镜像数据。如果需要实现高可用，可以将本地存储目 录挂载到 NFS 等共享存储上。配置示例： + ```yaml + image_registry: + registry: + storage: + filesystem: + rootdir: /opt/registry/data + nfs_mount: /repository/registry # 可选，将 rootdir 挂载到 NFS 服务器 + ``` + 需要在 `nfs` 节点配置和挂载好共享目录，保证所有 registry 实例的数据一致性。 + + - **azure**: 使用 Azure Blob Storage 作为后端存储。适用于部署在 Azure 云环境下的场景。配置示例： + ```yaml + image_registry: + registry: + storage: + azure: + accountname: + accountkey: + container: + ``` + + - **gcs**: 使用 Google Cloud Storage 作为后端存储。适用于部署在 GCP 云环境下的场景。配置示例： + ```yaml + image_registry: + registry: + storage: + gcs: + bucket: + keyfile: /path/to/keyfile.json + ``` + + - **s3**: 使用 Amazon S3 或兼容 S3 协议的对象存储作为后端存储。适用于 AWS 或支持 S3 协议的私有云。配置示例： + ```yaml + image_registry: + registry: + storage: + s3: + accesskey: + secretkey: + region: + bucket: + ``` + +> **注意：** +> 1. 使用共享存储（如 NFS、S3、GCS、Azure Blob）时，建议至少部署 2 个及以上 registry 实例，并通过负载均衡（如 keepalived+nginx）实现高可用访问。 +> 2. 配置共享存储时，需保证各 registry 节点对存储的读写权限和网络连通性。 + \ No newline at end of file diff --git a/scripts/harborCreateRegistriesAndReplications.sh b/scripts/harborCreateRegistriesAndReplications.sh deleted file mode 100644 index 86375c48..00000000 --- a/scripts/harborCreateRegistriesAndReplications.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - - -function createRegistries() { - - # create registry - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master1_Address}/api/v2.0/registries" -d "{\"name\": \"master1_2_master2\", \"type\": \"harbor\", \"url\":\"https://${master2_Address}:7443\", \"credential\": {\"access_key\": \"${Harbor_User}\", \"access_secret\": \"${Harbor_Passwd}\"}, \"insecure\": true}" - # create registry - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master1_Address}/api/v2.0/registries" -d "{\"name\": \"master1_2_master3\", \"type\": \"harbor\", \"url\":\"https://${master3_Address}:7443\", \"credential\": {\"access_key\": \"${Harbor_User}\", \"access_secret\": \"${Harbor_Passwd}\"}, \"insecure\": true}" - - # create registry - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master2_Address}/api/v2.0/registries" -d "{\"name\": \"master2_2_master1\", \"type\": \"harbor\", \"url\":\"https://${master1_Address}:7443\", \"credential\": {\"access_key\": \"${Harbor_User}\", \"access_secret\": \"${Harbor_Passwd}\"}, \"insecure\": true}" - # create registry - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master2_Address}/api/v2.0/registries" -d "{\"name\": \"master2_2_master3\", \"type\": \"harbor\", \"url\":\"https://${master3_Address}:7443\", \"credential\": {\"access_key\": \"${Harbor_User}\", \"access_secret\": \"${Harbor_Passwd}\"}, \"insecure\": true}" - - # create registry - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master3_Address}/api/v2.0/registries" -d "{\"name\": \"master3_2_master1\", \"type\": \"harbor\", \"url\":\"https://${master1_Address}:7443\", \"credential\": {\"access_key\": \"${Harbor_User}\", \"access_secret\": \"${Harbor_Passwd}\"}, \"insecure\": true}" - # create registry - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master3_Address}/api/v2.0/registries" -d "{\"name\": \"master3_2_master2\", \"type\": \"harbor\", \"url\":\"https://${master2_Address}:7443\", \"credential\": {\"access_key\": \"${Harbor_User}\", \"access_secret\": \"${Harbor_Passwd}\"}, \"insecure\": true}" - -} - -function listRegistries() { - curl -k -u $Harbor_UserPwd -X GET -H "Content-Type: application/json" "https://${Harbor_master1_Address}/api/v2.0/registries" - curl -k -u $Harbor_UserPwd -X GET -H "Content-Type: application/json" "https://${Harbor_master2_Address}/api/v2.0/registries" - curl -k -u $Harbor_UserPwd -X GET -H "Content-Type: application/json" "https://${Harbor_master3_Address}/api/v2.0/registries" - -} - -function createReplication() { - - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master1_Address}/api/v2.0/replication/policies" -d "{\"name\": \"master1_2_master2\", \"enabled\": true, \"deletion\":true, \"override\":true, \"replicate_deletion\":true, \"dest_registry\":{ \"id\": 1, \"name\": \"master1_2_master2\"}, \"trigger\": {\"type\": \"event_based\"}, \"dest_namespace_replace_count\":1 }" - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master1_Address}/api/v2.0/replication/policies" -d "{\"name\": \"master1_2_master3\", \"enabled\": true, \"deletion\":true, \"override\":true, \"replicate_deletion\":true, \"dest_registry\":{ \"id\": 2, \"name\": \"master1_2_master3\"}, \"trigger\": {\"type\": \"event_based\"}, \"dest_namespace_replace_count\":1 }" - - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master2_Address}/api/v2.0/replication/policies" -d "{\"name\": \"master2_2_master1\", \"enabled\": true, \"deletion\":true, \"override\":true, \"replicate_deletion\":true, \"dest_registry\":{ \"id\": 1, \"name\": \"master2_2_master1\"}, \"trigger\": {\"type\": \"event_based\"}, \"dest_namespace_replace_count\":1 }" - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master2_Address}/api/v2.0/replication/policies" -d "{\"name\": \"master2_2_master3\", \"enabled\": true, \"deletion\":true, \"override\":true, \"replicate_deletion\":true, \"dest_registry\":{ \"id\": 2, \"name\": \"master2_2_master3\"}, \"trigger\": {\"type\": \"event_based\"}, \"dest_namespace_replace_count\":1 }" - - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master3_Address}/api/v2.0/replication/policies" -d "{\"name\": \"master3_2_master1\", \"enabled\": true, \"deletion\":true, \"override\":true, \"replicate_deletion\":true, \"dest_registry\":{ \"id\": 1, \"name\": \"master3_2_master1\"}, \"trigger\": {\"type\": \"event_based\"}, \"dest_namespace_replace_count\":1 }" - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master3_Address}/api/v2.0/replication/policies" -d "{\"name\": \"master3_2_master2\", \"enabled\": true, \"deletion\":true, \"override\":true, \"replicate_deletion\":true, \"dest_registry\":{ \"id\": 2, \"name\": \"master3_2_master2\"}, \"trigger\": {\"type\": \"event_based\"}, \"dest_namespace_replace_count\":1 }" -} - -function listReplications() { - - curl -k -u $Harbor_UserPwd -X GET -H "Content-Type: application/json" "https://${Harbor_master1_Address}/api/v2.0/replication/policies" - curl -k -u $Harbor_UserPwd -X GET -H "Content-Type: application/json" "https://${Harbor_master2_Address}/api/v2.0/replication/policies" - curl -k -u $Harbor_UserPwd -X GET -H "Content-Type: application/json" "https://${Harbor_master3_Address}/api/v2.0/replication/policies" -} - -#### main ###### -Harbor_master1_Address=master1:7443 -master1_Address=192.168.122.61 -Harbor_master2_Address=master2:7443 -master2_Address=192.168.122.62 -Harbor_master3_Address=master3:7443 -master3_Address=192.168.122.63 -Harbor_User=admin #登录Harbor的用户 -Harbor_Passwd="Harbor12345" #登录Harbor的用户密码 -Harbor_UserPwd="$Harbor_User:$Harbor_Passwd" - - -createRegistries -listRegistries -createReplication -listReplications