From 7a7541c8df0805850dddcb97c9cda073cad60868 Mon Sep 17 00:00:00 2001
From: liqiang-fit2cloud <liqiang@fit2cloud.com>
Date: Fri, 15 Mar 2024 13:31:31 +0800
Subject: [PATCH] =?UTF-8?q?build:=20=E5=88=86=E5=BC=80=E9=95=9C=E5=83=8F?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../workflows/build-and-push-python-pg.yml    |  55 ++++++
 .../workflows/build-and-push-vector-model.yml |  55 ++++++
 .github/workflows/build-and-push.yml          |   2 +-
 Dockerfile => installer/Dockerfile            |  32 +---
 installer/Dockerfile-python-pg                | 176 ++++++++++++++++++
 installer/Dockerfile-vector-model             |   6 +
 installer/install-python.sh                   | 101 ----------
 7 files changed, 295 insertions(+), 132 deletions(-)
 create mode 100644 .github/workflows/build-and-push-python-pg.yml
 create mode 100644 .github/workflows/build-and-push-vector-model.yml
 rename Dockerfile => installer/Dockerfile (57%)
 create mode 100644 installer/Dockerfile-python-pg
 create mode 100644 installer/Dockerfile-vector-model
 delete mode 100644 installer/install-python.sh

diff --git a/.github/workflows/build-and-push-python-pg.yml b/.github/workflows/build-and-push-python-pg.yml
new file mode 100644
index 000000000..d3fe1c31d
--- /dev/null
+++ b/.github/workflows/build-and-push-python-pg.yml
@@ -0,0 +1,55 @@
+name: build-and-push-python-pg
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build-and-push-python-pg-to-ghcr:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check Disk Space
+        run: df -h
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: true
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: true
+          swap-storage: true
+      - name: Check Disk Space
+        run: df -h
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: main
+      - name: Prepare
+        id: prepare
+        run: |
+          DOCKER_IMAGE=ghcr.io/maxkb-dev/python-pg
+          DOCKER_PLATFORMS=linux/amd64,linux/arm64
+          TAG_NAME=python3.11.8-pg15.6
+          DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:latest"
+          echo ::set-output name=docker_image::${DOCKER_IMAGE}
+          echo ::set-output name=version::${TAG_NAME}
+          echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \
+            --build-arg VERSION=${TAG_NAME} \
+            --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \
+            --build-arg --no-cache \
+            --build-arg VCS_REF=${GITHUB_SHA::8} \
+            ${DOCKER_IMAGE_TAGS} .
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GH_TOKEN }}
+      - name: Docker Buildx (build-and-push)
+        run: |
+          docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} -f installer/Dockerfile-python-pg
\ No newline at end of file
diff --git a/.github/workflows/build-and-push-vector-model.yml b/.github/workflows/build-and-push-vector-model.yml
new file mode 100644
index 000000000..f83e5192f
--- /dev/null
+++ b/.github/workflows/build-and-push-vector-model.yml
@@ -0,0 +1,55 @@
+name: build-and-push-vector-model
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build-and-push-vector-model-to-ghcr:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check Disk Space
+        run: df -h
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: true
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: true
+          swap-storage: true
+      - name: Check Disk Space
+        run: df -h
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: main
+      - name: Prepare
+        id: prepare
+        run: |
+          DOCKER_IMAGE=ghcr.io/maxkb-dev/vector-model
+          DOCKER_PLATFORMS=linux/amd64,linux/arm64
+          TAG_NAME=v1.0.0
+          DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:latest"
+          echo ::set-output name=docker_image::${DOCKER_IMAGE}
+          echo ::set-output name=version::${TAG_NAME}
+          echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \
+            --build-arg VERSION=${TAG_NAME} \
+            --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \
+            --build-arg --no-cache \
+            --build-arg VCS_REF=${GITHUB_SHA::8} \
+            ${DOCKER_IMAGE_TAGS} .
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GH_TOKEN }}
+      - name: Docker Buildx (build-and-push)
+        run: |
+          docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} -f installer/Dockerfile-vector-model
\ No newline at end of file
diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml
index cd3cd274a..011c02aa1 100644
--- a/.github/workflows/build-and-push.yml
+++ b/.github/workflows/build-and-push.yml
@@ -107,4 +107,4 @@ jobs:
           password: ${{ secrets.GH_TOKEN }}
       - name: Docker Buildx (build-and-push)
         run: |
-          docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }}
\ No newline at end of file
+          docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} -f installer/Dockerfile
\ No newline at end of file
diff --git a/Dockerfile b/installer/Dockerfile
similarity index 57%
rename from Dockerfile
rename to installer/Dockerfile
index 746d033e0..7a26a0376 100644
--- a/Dockerfile
+++ b/installer/Dockerfile
@@ -1,9 +1,4 @@
-FROM python:3.11-slim as vector-model-build
-COPY installer/install_model.py install_model.py
-RUN pip3 install --upgrade pip setuptools && \
-    pip install pycrawlers && \
-    pip install transformers && \
-    python3 install_model.py
+FROM ghcr.io/maxkb-dev/python-pg:v1.0.0 as vector-model
 
 FROM node:18-alpine3.18 as web-build
 COPY ui ui
@@ -12,30 +7,7 @@ RUN cd ui && \
     npm run build && \
     rm -rf ./node_modules
 
-FROM postgres:15.6-bookworm
-
-ENV LANG=C.UTF-8
-
-RUN apt-get update
-
-RUN apt-get install postgresql-15-pgvector
-
-COPY installer/init.sql /docker-entrypoint-initdb.d
-
-RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo "Asia/Shanghai" > /etc/timezone
-
-ENV POSTGRES_USER root
-
-ENV POSTGRES_PASSWORD Password123@postgres
-
-
-# ---- prepare python env --- #
-ENV PATH /usr/local/bin:$PATH
-COPY installer/install-python.sh /install-python.sh
-RUN chmod 755 /install-python.sh; bash -c "/install-python.sh > /dev/null 2>&1" ; rm -f /install-python.sh
-ENV PYTHON_VERSION 3.11.8
-
-# ---- build maxkb --- #
+FROM ghcr.io/maxkb-dev/python-pg:python3.11.8-pg15.6
 # 创建工作目录
 RUN mkdir -p /opt/maxkb/app && mkdir -p /opt/maxkb/model && mkdir -p /opt/maxkb/conf
 VOLUME /opt/maxkb
diff --git a/installer/Dockerfile-python-pg b/installer/Dockerfile-python-pg
new file mode 100644
index 000000000..0032f9da0
--- /dev/null
+++ b/installer/Dockerfile-python-pg
@@ -0,0 +1,176 @@
+FROM postgres:15.6-bookworm
+
+ENV LANG=C.UTF-8
+
+RUN apt-get update
+
+RUN apt-get install postgresql-15-pgvector
+
+COPY installer/init.sql /docker-entrypoint-initdb.d
+
+RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo "Asia/Shanghai" > /etc/timezone
+
+ENV POSTGRES_USER root
+
+ENV POSTGRES_PASSWORD Password123@postgres
+
+ENV PATH /usr/local/bin:$PATH
+
+# http://bugs.python.org/issue19846
+# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK.
+ENV LANG C.UTF-8
+
+# runtime dependencies
+RUN set -eux; \
+	apt-get update; \
+	apt-get install -y --no-install-recommends \
+		ca-certificates \
+		netbase \
+		tzdata \
+	; \
+	rm -rf /var/lib/apt/lists/*
+
+ENV GPG_KEY A035C8C19219BA821ECEA86B64E628F8D684696D
+ENV PYTHON_VERSION 3.11.8
+
+RUN set -eux; \
+	\
+	savedAptMark="$(apt-mark showmanual)"; \
+	apt-get update; \
+	apt-get install -y --no-install-recommends \
+		dpkg-dev \
+		gcc \
+		gnupg \
+		libbluetooth-dev \
+		libbz2-dev \
+		libc6-dev \
+		libdb-dev \
+		libexpat1-dev \
+		libffi-dev \
+		libgdbm-dev \
+		liblzma-dev \
+		libncursesw5-dev \
+		libreadline-dev \
+		libsqlite3-dev \
+		libssl-dev \
+		make \
+		tk-dev \
+		uuid-dev \
+		wget \
+		xz-utils \
+		zlib1g-dev \
+	; \
+	\
+	wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz"; \
+	wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc"; \
+	GNUPGHOME="$(mktemp -d)"; export GNUPGHOME; \
+	gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$GPG_KEY"; \
+	gpg --batch --verify python.tar.xz.asc python.tar.xz; \
+	gpgconf --kill all; \
+	rm -rf "$GNUPGHOME" python.tar.xz.asc; \
+	mkdir -p /usr/src/python; \
+	tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz; \
+	rm python.tar.xz; \
+	\
+	cd /usr/src/python; \
+	gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \
+	./configure \
+		--build="$gnuArch" \
+		--enable-loadable-sqlite-extensions \
+		--enable-optimizations \
+		--enable-option-checking=fatal \
+		--enable-shared \
+		--with-lto \
+		--with-system-expat \
+		--without-ensurepip \
+	; \
+	nproc="$(nproc)"; \
+	EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)"; \
+	LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"; \
+	LDFLAGS="${LDFLAGS:--Wl},--strip-all"; \
+	make -j "$nproc" \
+		"EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \
+		"LDFLAGS=${LDFLAGS:-}" \
+		"PROFILE_TASK=${PROFILE_TASK:-}" \
+	; \
+# https://github.com/docker-library/python/issues/784
+# prevent accidental usage of a system installed libpython of the same version
+	rm python; \
+	make -j "$nproc" \
+		"EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \
+		"LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" \
+		"PROFILE_TASK=${PROFILE_TASK:-}" \
+		python \
+	; \
+	make install; \
+	\
+	cd /; \
+	rm -rf /usr/src/python; \
+	\
+	find /usr/local -depth \
+		\( \
+			\( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \
+			-o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \) \) \
+		\) -exec rm -rf '{}' + \
+	; \
+	\
+	ldconfig; \
+	\
+	apt-mark auto '.*' > /dev/null; \
+	apt-mark manual $savedAptMark; \
+	find /usr/local -type f -executable -not \( -name '*tkinter*' \) -exec ldd '{}' ';' \
+		| awk '/=>/ { so = $(NF-1); if (index(so, "/usr/local/") == 1) { next }; gsub("^/(usr/)?", "", so); printf "*%s\n", so }' \
+		| sort -u \
+		| xargs -r dpkg-query --search \
+		| cut -d: -f1 \
+		| sort -u \
+		| xargs -r apt-mark manual \
+	; \
+	apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \
+	rm -rf /var/lib/apt/lists/*; \
+	\
+	python3 --version
+
+# make some useful symlinks that are expected to exist ("/usr/local/bin/python" and friends)
+RUN set -eux; \
+	for src in idle3 pydoc3 python3 python3-config; do \
+		dst="$(echo "$src" | tr -d 3)"; \
+		[ -s "/usr/local/bin/$src" ]; \
+		[ ! -e "/usr/local/bin/$dst" ]; \
+		ln -svT "$src" "/usr/local/bin/$dst"; \
+	done
+
+# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
+ENV PYTHON_PIP_VERSION 24.0
+# https://github.com/docker-library/python/issues/365
+ENV PYTHON_SETUPTOOLS_VERSION 65.5.1
+# https://github.com/pypa/get-pip
+ENV PYTHON_GET_PIP_URL https://github.com/pypa/get-pip/raw/dbf0c85f76fb6e1ab42aa672ffca6f0a675d9ee4/public/get-pip.py
+ENV PYTHON_GET_PIP_SHA256 dfe9fd5c28dc98b5ac17979a953ea550cec37ae1b47a5116007395bfacff2ab9
+
+RUN set -eux; \
+	\
+	savedAptMark="$(apt-mark showmanual)"; \
+	apt-get update; \
+	apt-get install -y --no-install-recommends wget; \
+	\
+	wget -O get-pip.py "$PYTHON_GET_PIP_URL"; \
+	echo "$PYTHON_GET_PIP_SHA256 *get-pip.py" | sha256sum -c -; \
+	\
+	apt-mark auto '.*' > /dev/null; \
+	[ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \
+	apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \
+	rm -rf /var/lib/apt/lists/*; \
+	\
+	export PYTHONDONTWRITEBYTECODE=1; \
+	\
+	python get-pip.py \
+		--disable-pip-version-check \
+		--no-cache-dir \
+		--no-compile \
+		"pip==$PYTHON_PIP_VERSION" \
+		"setuptools==$PYTHON_SETUPTOOLS_VERSION" \
+	; \
+	rm -f get-pip.py; \
+	\
+	pip --version
diff --git a/installer/Dockerfile-vector-model b/installer/Dockerfile-vector-model
new file mode 100644
index 000000000..97e35b4e4
--- /dev/null
+++ b/installer/Dockerfile-vector-model
@@ -0,0 +1,6 @@
+FROM python:3.11-slim
+COPY installer/install_model.py install_model.py
+RUN pip3 install --upgrade pip setuptools && \
+    pip install pycrawlers && \
+    pip install transformers && \
+    python3 install_model.py
\ No newline at end of file
diff --git a/installer/install-python.sh b/installer/install-python.sh
deleted file mode 100644
index 0835f28cc..000000000
--- a/installer/install-python.sh
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/bin/bash
-
-# ---- prepare python env --- #
-# ---- copy from https://github.com/docker-library/python/blob/master/3.11/slim-bookworm/Dockerfile --- #
-
-export PATH=/usr/local/bin:$PATH
-
-export LANG=C.UTF-8
-
-apt-get update;
-apt-get install -y --no-install-recommends ca-certificates netbase tzdata
-rm -rf /var/lib/apt/lists/*
-
-export GPG_KEY=A035C8C19219BA821ECEA86B64E628F8D684696D
-export PYTHON_VERSION=3.11.8
-
-savedAptMark="$(apt-mark showmanual)"
-apt-get update
-apt-get install -y --no-install-recommends dpkg-dev gcc gnupg libbluetooth-dev libbz2-dev libc6-dev libdb-dev libexpat1-dev libffi-dev libgdbm-dev liblzma-dev libncursesw5-dev libreadline-dev libsqlite3-dev libssl-dev make \
-		tk-dev uuid-dev wget xz-utils zlib1g-dev
-wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz"
-wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc"
-GNUPGHOME="$(mktemp -d)"
-export GNUPGHOME
-gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$GPG_KEY"
-gpg --batch --verify python.tar.xz.asc python.tar.xz
-gpgconf --kill all
-rm -rf "$GNUPGHOME" python.tar.xz.asc
-mkdir -p /usr/src/python
-tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz
-rm python.tar.xz;
-cd /usr/src/python
-gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"
-./configure --build="$gnuArch" --enable-loadable-sqlite-extensions --enable-optimizations --enable-option-checking=fatal --enable-shared --with-lto --with-system-expat --without-ensurepip \
-nproc="$(nproc)"
-EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)"
-LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"
-LDFLAGS="${LDFLAGS:--Wl},--strip-all"
-make -j "$nproc" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" "LDFLAGS=${LDFLAGS:-}" "PROFILE_TASK=${PROFILE_TASK:-}"
-
-rm python
-make -j "$nproc" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" "LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" "PROFILE_TASK=${PROFILE_TASK:-}" python
-make install
-cd /
-rm -rf /usr/src/python
-
-find /usr/local -depth \
-		\( \
-			\( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \
-			-o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \) \) \
-		\) -exec rm -rf '{}'
-ldconfig
-apt-mark auto '.*' > /dev/null
-apt-mark manual $savedAptMark
-find /usr/local -type f -executable -not \( -name '*tkinter*' \) -exec ldd '{}' ';' \
-		| awk '/=>/ { so = $(NF-1); if (index(so, "/usr/local/") == 1) { next }; gsub("^/(usr/)?", "", so); printf "*%s\n", so }' \
-		| sort -u \
-		| xargs -r dpkg-query --search \
-		| cut -d: -f1 \
-		| sort -u \
-		| xargs -r apt-mark manual
-apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false
-rm -rf /var/lib/apt/lists/*
-python3 --version
-
-# make some useful symlinks that are expected to exist ("/usr/local/bin/python" and friends)
-
-for src in idle3 pydoc3 python3 python3-config; do \
-		dst="$(echo "$src" | tr -d 3)"; \
-		[ -s "/usr/local/bin/$src" ]; \
-		[ ! -e "/usr/local/bin/$dst" ]; \
-		ln -svT "$src" "/usr/local/bin/$dst"; \
-done
-
-# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-export PYTHON_PIP_VERSION=24.0
-# https://github.com/docker-library/python/issues/365
-export PYTHON_SETUPTOOLS_VERSION=65.5.1
-# https://github.com/pypa/get-pip
-export PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/dbf0c85f76fb6e1ab42aa672ffca6f0a675d9ee4/public/get-pip.py
-export PYTHON_GET_PIP_SHA256=dfe9fd5c28dc98b5ac17979a953ea550cec37ae1b47a5116007395bfacff2ab9
-
-
-savedAptMark="$(apt-mark showmanual)"
-apt-get update
-apt-get install -y --no-install-recommends wget
-
-wget -O get-pip.py "$PYTHON_GET_PIP_URL"
-echo "$PYTHON_GET_PIP_SHA256 *get-pip.py" | sha256sum -c -
-apt-mark auto '.*' > /dev/null
-[ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null
-apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false
-rm -rf /var/lib/apt/lists/*
-
-export PYTHONDONTWRITEBYTECODE=1
-
-python get-pip.py --disable-pip-version-check --no-cache-dir --no-compile "pip==$PYTHON_PIP_VERSION" "setuptools==$PYTHON_SETUPTOOLS_VERSION"
-
-rm -f get-pip.py
-
-pip --version
\ No newline at end of file