Optimize the process of build repository iso

1. Use a configuration file to manage the required packages instead of in Dockerfile, this way is more flexible and easier to maintain. 2. Build repository iso support debian9/debian10 and arm64 architecture. 3. Add github action workflow of auto build repository iso artifact. Signed-off-by: muzi502 <muzi502.li@gmail.com>
2025-12-26 01:22:51 +00:00 · 2022-04-09 01:10:02 +08:00 · 2022-04-09 01:10:02 +08:00 · 86e1a6fb37
parent f751e1a79a
commit 86e1a6fb37
9 changed files with 339 additions and 51 deletions
--- a/.github/workflows/gen-repository-iso.yaml
+++ b/.github/workflows/gen-repository-iso.yaml
@ -0,0 +1,72 @@
+---
+name: gen-repository-iso
+on:
+  push:
+    tag:
+      - 'v*'
+  workflow_dispatch:
+
+jobs:
+  build:
+    if: startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: centos7-rpms
+            dockerfile: dockerfile.centos7
+          - name: debian9-debs
+            dockerfile: dockerfile.debian9
+          - name: debian10-debs
+            dockerfile: dockerfile.debian10
+          - name: ubuntu-16.04-debs
+            dockerfile: dockerfile.ubuntu1604
+          - name: ubuntu-18.04-debs
+            dockerfile: dockerfile.ubuntu1804
+          - name: ubuntu-20.04-debs
+            dockerfile: dockerfile.ubuntu2004
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build iso image to local
+        uses: docker/build-push-action@v2
+        with:
+          context: hack/gen-repository-iso
+          file: hack/gen-repository-iso/${{ matrix.dockerfile }}
+          platforms: linux/amd64,linux/arm64
+          outputs: type=local,dest=./output
+
+      - name: Prepare for upload iso
+        shell: bash
+        run: |
+          mv ./output/linux_amd64/*.iso ${{ matrix.name }}-amd64.iso
+          mv ./output/linux_arm64/*.iso ${{ matrix.name }}-arm64.iso
+          sha256sum *.iso >  ${{ matrix.name }}.iso.sha256sum.txt
+
+      - name: Wait for release workflow to finish
+        uses: lewagon/wait-on-check-action@v1.0.0
+        with:
+          ref: ${{ github.ref }}
+          check-name: 'goreleaser'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          wait-interval: 10
+          allowed-conclusions: success
+
+      - name: Release and upload packages
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: softprops/action-gh-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          files: |
+            ${{ matrix.name }}.iso.sha256sum.txt
+            ${{ matrix.name }}-amd64.iso
+            ${{ matrix.name }}-arm64.iso
--- a/19
+++ b/19
@ -256,4 +256,21 @@ kk: fmt vet
 	-o bin/$(GOOS)/$(GOARCH)/kk ./cmd/main.go;

 go-releaser-test:
-	goreleaser release --rm-dist --skip-publish --snapshot
+	goreleaser release --rm-dist --skip-publish --snapshot
+
+# build the artifact of repository iso
+ISO_ARCH ?= amd64
+ISO_OUTPUT_DIR ?= ./output
+ISO_BUILD_WORKDIR := hack/gen-repository-iso
+ISO_OS_NAMES := centos7 debian9 debian10 ubuntu1604 ubuntu1804 ubuntu2004
+ISO_BUILD_NAMES := $(addprefix build-iso-,$(ISO_OS_NAMES))
+build-iso-all: $(ISO_BUILD_NAMES)
+.PHONY: $(ISO_BUILD_NAMES)
+$(ISO_BUILD_NAMES):
+	@export DOCKER_BUILDKIT=1
+	docker build \
+		--platform linux/$(ISO_ARCH) \
+		--build-arg TARGETARCH=$(ISO_ARCH) \
+		-o type=local,dest=$(ISO_OUTPUT_DIR) \
+		-f $(ISO_BUILD_WORKDIR)/dockerfile.$(subst build-iso-,,$@) \
+		$(ISO_BUILD_WORKDIR)
--- a/hack/gen-repository-iso/dockerfile.centos7
+++ b/hack/gen-repository-iso/dockerfile.centos7
@ -1,15 +1,22 @@
-FROM centos:centos7
+FROM centos:7 as centos7
+ARG TARGETARCH
+ENV OS=centos
+ENV OS_VERSION=7
+ARG BUILD_TOOLS="yum-utils createrepo mkisofs epel-release"
+ARG DIR=${OS}${OS_VERSION}-${TARGETARCH}-rpms
+
+RUN yum install -q -y ${BUILD_TOOLS} \
+    && yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo \
+    && yum makecache

 WORKDIR package
+COPY packages.yaml .
+COPY --from=mikefarah/yq:4.11.1 /usr/bin/yq /usr/bin/yq
+RUN yq eval ".common[],.rpms[],.${OS}[],.${OS}${OS_VERSION}" packages.yaml > packages.list

-ENV DIR=centos-7-amd64-rpms
-ENV DOCKERVERSION=20.10.8
+RUN sort -u packages.list | xargs repotrack -p ${DIR} \
+    && createrepo -d ${DIR} \
+    && mkisofs -r -o ${DIR}.iso ${DIR}

-RUN mkdir ${DIR} && cd ${DIR} && \
-    yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo && \
-    yumdownloader --resolve --destdir ./ docker-ce-${DOCKERVERSION} docker-ce-cli-${DOCKERVERSION} containerd.io \
-        nss nss-sysinit nss-tools lz4 chrony bind-utils nfs-utils net-tools ipvsadm ceph-common glusterfs-fuse yum-utils \
-        curl openssl socat conntrack ebtables ipset telnet lvm2 keepalived haproxy psmisc tcpdump && \
-        yum install createrepo mkisofs -y && createrepo ./
-
-RUN  mkisofs -r -o ${DIR}.iso ${DIR}
+FROM scratch
+COPY --from=centos7 /package/*.iso /
--- a/hack/gen-repository-iso/dockerfile.debian10
+++ b/hack/gen-repository-iso/dockerfile.debian10
@ -0,0 +1,38 @@
+FROM debian:10 as debian10
+ARG TARGETARCH
+ARG OS_RELEASE=buster
+ARG OS_VERSION=10
+ARG DIR=debian-10-${TARGETARCH}-debs
+ARG PKGS=.common[],.debs[],.debian[],.debian10[]
+ARG BUILD_TOOLS="apt-transport-https software-properties-common ca-certificates curl wget gnupg dpkg-dev genisoimage dirmngr"
+ENV DEBIAN_FRONTEND=noninteractive
+
+# dump system package list
+RUN dpkg --get-selections | grep -v deinstall | cut -f1 | cut -d ':' -f1 > packages.list
+RUN ARCH=$(dpkg --print-architecture) \
+    && apt update -qq \
+    && apt install -y --no-install-recommends $BUILD_TOOLS \
+    && if [ "$TARGETARCH" = "amd64" ]; then \
+        curl -fsSL https://download.gluster.org/pub/gluster/glusterfs/7/rsa.pub | apt-key add - ; \
+        echo deb https://download.gluster.org/pub/gluster/glusterfs/7/LATEST/Debian/${OS_VERSION}/amd64/apt ${OS_RELEASE} main > /etc/apt/sources.list.d/gluster.list ; \
+       fi \
+    && curl -fsSL "https://download.docker.com/linux/debian/gpg" | apt-key add -qq - \
+    && echo "deb [arch=$TARGETARCH] https://download.docker.com/linux/debian ${OS_RELEASE} stable" > /etc/apt/sources.list.d/docker.list \
+    && apt update -qq
+
+WORKDIR /package
+COPY packages.yaml .
+
+COPY --from=mikefarah/yq:4.11.1 /usr/bin/yq /usr/bin/yq
+RUN yq eval "${PKGS}" packages.yaml >> packages.list \
+    && sort -u packages.list | xargs apt-get install --yes --reinstall --print-uris | awk -F "'" '{print $2}' | grep -v '^$' | sort -u > packages.urls
+
+RUN mkdir -p ${DIR} \
+    && wget -q -x -P ${DIR} -i packages.urls \
+    && cd ${DIR} \
+    && dpkg-scanpackages ./ /dev/null | gzip -9c > ./Packages.gz
+
+RUN genisoimage -r -o ${DIR}.iso ${DIR}
+
+FROM scratch
+COPY --from=debian10 /package/*.iso /
--- a/hack/gen-repository-iso/dockerfile.debian9
+++ b/hack/gen-repository-iso/dockerfile.debian9
@ -0,0 +1,37 @@
+FROM debian:9 as debian9
+ARG TARGETARCH
+ARG OS_RELEASE=stretch
+ARG OS_VERSION=9
+ARG DIR=debian-9-${TARGETARCH}-debs
+ARG PKGS=.common[],.debs[],.debian[],.debian9[]
+ARG BUILD_TOOLS="apt-transport-https software-properties-common ca-certificates curl wget gnupg dpkg-dev genisoimage dirmngr"
+ENV DEBIAN_FRONTEND=noninteractive
+
+# dump system package list
+RUN dpkg --get-selections | grep -v deinstall | cut -f1 | cut -d ':' -f1 > packages.list
+RUN apt update -qq \
+    && apt install -y --no-install-recommends $BUILD_TOOLS \
+    && if [ "$TARGETARCH" = "amd64" ]; then \
+        curl -fsSL https://download.gluster.org/pub/gluster/glusterfs/7/rsa.pub | apt-key add - ; \
+        echo deb https://download.gluster.org/pub/gluster/glusterfs/7/LATEST/Debian/${OS_VERSION}/amd64/apt ${OS_RELEASE} main > /etc/apt/sources.list.d/gluster.list ; \
+       fi \
+    && curl -fsSL "https://download.docker.com/linux/debian/gpg" | apt-key add -qq - \
+    && echo "deb [arch=$TARGETARCH] https://download.docker.com/linux/debian ${OS_RELEASE} stable" > /etc/apt/sources.list.d/docker.list \
+    && apt update -qq
+
+WORKDIR /package
+COPY packages.yaml .
+
+COPY --from=mikefarah/yq:4.11.1 /usr/bin/yq /usr/bin/yq
+RUN yq eval "${PKGS}" packages.yaml >> packages.list \
+    && sort -u packages.list | xargs apt-get install --yes --reinstall --print-uris | awk -F "'" '{print $2}' | grep -v '^$' | sort -u > packages.urls
+
+RUN mkdir -p ${DIR} \
+    && wget -q -x -P ${DIR} -i packages.urls \
+    && cd ${DIR} \
+    && dpkg-scanpackages ./ /dev/null | gzip -9c > ./Packages.gz
+
+RUN genisoimage -r -o ${DIR}.iso ${DIR}
+
+FROM scratch
+COPY --from=debian9 /package/*.iso /
--- a/hack/gen-repository-iso/dockerfile.ubuntu1604
+++ b/hack/gen-repository-iso/dockerfile.ubuntu1604
@ -1,19 +1,33 @@
-FROM ubuntu:16.04
+FROM ubuntu:16.04 as ubuntu1604
+ARG TARGETARCH
+ARG OS_RELEASE=xenial
+ARG DIR=ubuntu-16.04-${TARGETARCH}-debs
+ARG PKGS=.common[],.debs[],.ubuntu[],.ubuntu1604[]
+ARG BUILD_TOOLS="apt-transport-https software-properties-common ca-certificates curl wget gnupg dpkg-dev genisoimage"
+ENV DEBIAN_FRONTEND=noninteractive

-WORKDIR package
+# dump system package list
+RUN dpkg --get-selections | grep -v deinstall | cut -f1 | cut -d ':' -f1 > packages.list
+RUN apt update -qq \
+    && apt install -y --no-install-recommends $BUILD_TOOLS \
+    && add-apt-repository ppa:gluster/glusterfs-7 -y \
+    && curl -fsSL "https://download.docker.com/linux/ubuntu/gpg" | apt-key add -qq - \
+    && echo "deb [arch=$TARGETARCH] https://download.docker.com/linux/ubuntu ${OS_RELEASE} stable" > /etc/apt/sources.list.d/docker.list\
+    && apt update -qq

-ENV DIR=ubuntu-16.04-amd64-debs
-ENV DOCKERVERSION=5:20.10.7~3-0~ubuntu-xenial
-ENV PACKAGES="apt-transport-https ca-certificates curl git gnupg-agent software-properties-common sudo openssl openssh-server socat conntrack ebtables ipset docker-ce=${DOCKERVERSION} docker-ce-cli=${DOCKERVERSION} containerd.io chrony ipvsadm nfs-common ceph-common glusterfs-client dnsutils telnet lvm2 keepalived haproxy psmisc tcpdump"
+WORKDIR /package
+COPY packages.yaml .

-RUN mkdir ${DIR} && apt update && apt install curl sudo software-properties-common apt-transport-https -y && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - && \
-    add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable" && \
-    add-apt-repository ppa:gluster/glusterfs-7 -y && apt update
+COPY --from=mikefarah/yq:4.11.1 /usr/bin/yq /usr/bin/yq
+RUN yq eval "${PKGS}" packages.yaml >> packages.list \
+    && sort -u packages.list | xargs apt-get install --yes --reinstall --print-uris | awk -F "'" '{print $2}' | grep -v '^$' | sort -u > packages.urls

-ADD download-pkgs.sh ./${DIR}/
+RUN mkdir -p ${DIR} \
+    && wget -q -x -P ${DIR} -i packages.urls \
+    && cd ${DIR} \
+    && dpkg-scanpackages ./ /dev/null | gzip -9c > ./Packages.gz

-RUN cd ${DIR} && sh download-pkgs.sh && rm -rf download-pkgs.sh && rm -rf
+RUN genisoimage -r -o ${DIR}.iso ${DIR}

-RUN apt install apt-utils dpkg-dev -y && cd ${DIR} && dpkg-scanpackages ./ /dev/null |gzip > ./Packages.gz
-
-RUN apt-get install genisoimage -y &&  mkisofs -r -o ${DIR}.iso ${DIR}
+FROM scratch
+COPY --from=ubuntu1604 /package/*.iso /
--- a/hack/gen-repository-iso/dockerfile.ubuntu1804
+++ b/hack/gen-repository-iso/dockerfile.ubuntu1804
@ -1,19 +1,34 @@
-FROM ubuntu:18.04
+FROM ubuntu:18.04 as ubuntu1804
+ARG TARGETARCH
+ARG OS_RELEASE=bionic
+ARG DIR=ubuntu-18.04-${TARGETARCH}-debs
+ARG PKGS=.common[],.debs[],.ubuntu[],.ubuntu1804[]
+ARG BUILD_TOOLS="apt-transport-https software-properties-common ca-certificates curl wget gnupg dpkg-dev genisoimage"
+ENV DEBIAN_FRONTEND=noninteractive

-WORKDIR package
+# dump system package list
+RUN dpkg --get-selections | grep -v deinstall | cut -f1 | cut -d ':' -f1 > packages.list
+RUN apt update -qq \
+    && apt install -y --no-install-recommends $BUILD_TOOLS \
+    && add-apt-repository ppa:gluster/glusterfs-7 -y \
+    && curl -fsSL "https://download.docker.com/linux/ubuntu/gpg" | apt-key add -qq - \
+    && echo "deb [arch=$TARGETARCH] https://download.docker.com/linux/ubuntu ${OS_RELEASE} stable" > /etc/apt/sources.list.d/docker.list\
+    && apt update -qq

-ENV DIR=ubuntu-18.04-amd64-debs
-ENV DOCKERVERSION=5:20.10.8~3-0~ubuntu-bionic
-ENV PACKAGES="apt-transport-https ca-certificates curl git gnupg-agent software-properties-common sudo openssl openssh-server socat conntrack ebtables ipset docker-ce=${DOCKERVERSION} docker-ce-cli=${DOCKERVERSION} containerd.io chrony ipvsadm nfs-common ceph-common glusterfs-client dnsutils telnet lvm2 keepalived haproxy psmisc tcpdump"
+WORKDIR /package
+COPY packages.yaml .

-RUN mkdir ${DIR} && apt update && apt install curl sudo software-properties-common apt-transport-https -y && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - && \
-    add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable" && \
-    add-apt-repository ppa:gluster/glusterfs-7 -y && apt update
+COPY --from=mikefarah/yq:4.11.1 /usr/bin/yq /usr/bin/yq
+RUN yq eval "${PKGS}" packages.yaml >> packages.list \
+    && dpkg --get-selections | grep -v deinstall | cut -f1 | cut -d ':' -f1 >> packages.list \
+    && sort -u packages.list | xargs apt-get install --yes --reinstall --print-uris | awk -F "'" '{print $2}' | grep -v '^$' | sort -u > packages.urls

-ADD download-pkgs.sh ./${DIR}/
+RUN mkdir -p ${DIR} \
+    && wget -q -x -P ${DIR} -i packages.urls \
+    && cd ${DIR} \
+    && dpkg-scanpackages ./ /dev/null | gzip -9c > ./Packages.gz

-RUN cd ${DIR} && sh download-pkgs.sh && rm -rf download-pkgs.sh && rm -rf errors.txt
+RUN genisoimage -r -o ${DIR}.iso ${DIR}

-RUN apt install apt-utils dpkg-dev -y && cd ${DIR} && dpkg-scanpackages ./ /dev/null |gzip > ./Packages.gz
-
-RUN apt-get install genisoimage -y &&  mkisofs -r -o ${DIR}.iso ${DIR}
+FROM scratch
+COPY --from=ubuntu1804 /package/*.iso /
--- a/hack/gen-repository-iso/dockerfile.ubuntu2004
+++ b/hack/gen-repository-iso/dockerfile.ubuntu2004
@ -1,19 +1,33 @@
-FROM ubuntu:20.04
+FROM ubuntu:20.04 as ubuntu2004
+ARG TARGETARCH
+ARG OS_RELEASE=focal
+ARG DIR=ubuntu-20.04-${TARGETARCH}-debs
+ARG PKGS=.common[],.debs[],.ubuntu[],.ubuntu2004[]
+ARG BUILD_TOOLS="apt-transport-https software-properties-common ca-certificates curl wget gnupg dpkg-dev genisoimage"
+ENV DEBIAN_FRONTEND=noninteractive

-WORKDIR package
+# dump system package list
+RUN dpkg --get-selections | grep -v deinstall | cut -f1 | cut -d ':' -f1 > packages.list
+RUN apt update -qq \
+    && apt install -y --no-install-recommends $BUILD_TOOLS \
+    && add-apt-repository ppa:gluster/glusterfs-7 -y \
+    && curl -fsSL "https://download.docker.com/linux/ubuntu/gpg" | apt-key add -qq - \
+    && echo "deb [arch=$TARGETARCH] https://download.docker.com/linux/ubuntu ${OS_RELEASE} stable" > /etc/apt/sources.list.d/docker.list\
+    && apt update -qq

-ENV DIR=ubuntu-20.04-amd64-debs
-ENV DOCKERVERSION=5:20.10.8~3-0~ubuntu-focal
-ENV PACKAGES="apt-transport-https ca-certificates curl git gnupg-agent software-properties-common sudo openssl openssh-server socat conntrack ebtables ipset docker-ce=${DOCKERVERSION} docker-ce-cli=${DOCKERVERSION} containerd.io chrony ipvsadm nfs-common ceph-common glusterfs-client dnsutils telnet lvm2 keepalived haproxy psmisc tcpdump"
+WORKDIR /package
+COPY packages.yaml .

-RUN mkdir ${DIR} && apt update && apt install curl sudo software-properties-common apt-transport-https -y && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - && \
-    add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable" && \
-    add-apt-repository ppa:gluster/glusterfs-7 -y && apt update
+COPY --from=mikefarah/yq:4.11.1 /usr/bin/yq /usr/bin/yq
+RUN yq eval "${PKGS}" packages.yaml >> packages.list \
+    && sort -u packages.list | xargs apt-get install --yes --reinstall --print-uris | awk -F "'" '{print $2}' | grep -v '^$' | sort -u > packages.urls

-ADD download-pkgs.sh ./${DIR}/
+RUN mkdir -p ${DIR} \
+    && wget -q -x -P ${DIR} -i packages.urls \
+    && cd ${DIR} \
+    && dpkg-scanpackages ./ /dev/null | gzip -9c > ./Packages.gz

-RUN cd ${DIR} && sh download-pkgs.sh && rm -rf download-pkgs.sh && rm -rf errors.txt
+RUN genisoimage -r -o ${DIR}.iso ${DIR}

-RUN apt install apt-utils dpkg-dev -y && cd ${DIR} && dpkg-scanpackages ./ /dev/null |gzip > ./Packages.gz
-
-RUN apt-get install genisoimage -y &&  mkisofs -r -o ${DIR}.iso ${DIR}
+FROM scratch
+COPY --from=ubuntu2004 /package/*.iso /
--- a/hack/gen-repository-iso/packages.yaml
+++ b/hack/gen-repository-iso/packages.yaml
@ -0,0 +1,74 @@
+---
+common:
+  - curl
+  - ceph-common
+  - net-tools
+  - lvm2
+  - telnet
+  - tcpdump
+  - socat
+  - openssl
+  - chrony
+  - conntrack
+  - curl
+  - ipvsadm
+  - ipset
+  - psmisc
+  - bash-completion
+  - ebtables
+  - haproxy
+  - keepalived
+rpms:
+  - nfs-utils
+  - yum-utils
+  - bind-utils
+  - glusterfs-fuse
+  - lz4
+  - nss
+  - nss-sysinit
+  - nss-tools
+debs:
+  - apt-transport-https
+  - ca-certificates
+  - dnsutils
+  - git
+  - glusterfs-client
+  - gnupg-agent
+  - nfs-common
+  - openssh-server
+  - software-properties-common
+  - sudo
+
+centos:
+  - containerd.io
+
+centos7:
+  - libselinux-python
+  - docker-ce-20.10.8
+  - docker-ce-cli-20.10.8
+
+debian:
+  - containerd.io
+
+debian9:
+  - docker-ce=5:19.03.15~3-0~debian-stretch
+  - docker-ce-cli=5:19.03.15~3-0~debian-stretch
+
+debian10:
+  - docker-ce=5:20.10.7~3-0~debian-buster
+  - docker-ce-cli=5:20.10.7~3-0~debian-buster
+
+ubuntu:
+  - containerd.io
+
+ubuntu1604:
+  - docker-ce=5:20.10.7~3-0~ubuntu-xenial
+  - docker-ce-cli=5:20.10.7~3-0~ubuntu-xenial
+
+ubuntu1804:
+  - docker-ce=5:20.10.8~3-0~ubuntu-bionic
+  - docker-ce-cli=5:20.10.7~3-0~ubuntu-bionic
+
+ubuntu2004:
+  - docker-ce=5:20.10.8~3-0~ubuntu-focal
+  - docker-ce-cli=5:20.10.7~3-0~ubuntu-focal