Merge branch 'main' of github.com:maxkb-dev/maxkb

2025-12-28 14:52:58 +00:00 · 2024-03-15 14:04:33 +08:00 · 2024-03-15 14:04:33 +08:00 · 57cb283a51
parent 357be32df1 fed53c455e
commit 57cb283a51
8 changed files with 172 additions and 71 deletions
--- a/.github/workflows/build-and-push-python-pg.yml
+++ b/.github/workflows/build-and-push-python-pg.yml
@ -0,0 +1,41 @@
+name: build-and-push-python-pg
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build-and-push-python-pg-to-ghcr:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: main
+      - name: Prepare
+        id: prepare
+        run: |
+          DOCKER_IMAGE=ghcr.io/maxkb-dev/python-pg
+          DOCKER_PLATFORMS=linux/amd64,linux/arm64
+          TAG_NAME=python3.11.8-pg15.6
+          DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:latest"
+          echo ::set-output name=docker_image::${DOCKER_IMAGE}
+          echo ::set-output name=version::${TAG_NAME}
+          echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \
+            --build-arg VERSION=${TAG_NAME} \
+            --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \
+            --build-arg --no-cache \
+            --build-arg VCS_REF=${GITHUB_SHA::8} \
+            ${DOCKER_IMAGE_TAGS} .
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GH_TOKEN }}
+      - name: Docker Buildx (build-and-push)
+        run: |
+          docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} -f installer/Dockerfile-python-pg
--- a/.github/workflows/build-and-push-vector-model.yml
+++ b/.github/workflows/build-and-push-vector-model.yml
@ -0,0 +1,55 @@
+name: build-and-push-vector-model
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build-and-push-vector-model-to-ghcr:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check Disk Space
+        run: df -h
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: true
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: true
+          swap-storage: true
+      - name: Check Disk Space
+        run: df -h
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: main
+      - name: Prepare
+        id: prepare
+        run: |
+          DOCKER_IMAGE=ghcr.io/maxkb-dev/vector-model
+          DOCKER_PLATFORMS=linux/amd64,linux/arm64
+          TAG_NAME=v1.0.0
+          DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:latest"
+          echo ::set-output name=docker_image::${DOCKER_IMAGE}
+          echo ::set-output name=version::${TAG_NAME}
+          echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \
+            --build-arg VERSION=${TAG_NAME} \
+            --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \
+            --build-arg --no-cache \
+            --build-arg VCS_REF=${GITHUB_SHA::8} \
+            ${DOCKER_IMAGE_TAGS} .
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GH_TOKEN }}
+      - name: Docker Buildx (build-and-push)
+        run: |
+          docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} -f installer/Dockerfile-vector-model
--- a/.github/workflows/build-and-push.yml
+++ b/.github/workflows/build-and-push.yml
@ -12,15 +12,20 @@ jobs:
  build-and-push-fit2cloud-registry:
    runs-on: ubuntu-latest
    steps:
-      - name: Maximize build space
-        uses: easimon/maximize-build-space@master
+      - name: Check Disk Space
+        run: df -h
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
        with:
-          root-reserve-mb: 35840
-          remove-dotnet: 'true'
-          remove-android: 'true'
-          remove-haskell: 'true'
-          remove-codeql: 'true'
-          remove-docker-images: 'true'
+          tool-cache: true
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: true
+          swap-storage: true
+      - name: Check Disk Space
+        run: df -h
      - name: Checkout
        uses: actions/checkout@v4
        with:
@ -52,20 +57,25 @@ jobs:
          password: ${{ secrets.FIT2CLOUD_REGISTRY_PASSWORD }}
      - name: Docker Buildx (build-and-push)
        run: |
-          docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }}
+          docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} -f installer/Dockerfile

  build-and-push-to-ghcr:
    runs-on: ubuntu-latest
    steps:
-      - name: Maximize build space
-        uses: easimon/maximize-build-space@master
+      - name: Check Disk Space
+        run: df -h
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
        with:
-          root-reserve-mb: 35840
-          remove-dotnet: 'true'
-          remove-android: 'true'
-          remove-haskell: 'true'
-          remove-codeql: 'true'
-          remove-docker-images: 'true'
+          tool-cache: true
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: true
+          swap-storage: true
+      - name: Check Disk Space
+        run: df -h
      - name: Checkout
        uses: actions/checkout@v4
        with:
@ -97,4 +107,4 @@ jobs:
          password: ${{ secrets.GH_TOKEN }}
      - name: Docker Buildx (build-and-push)
        run: |
-          docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }}
+          docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} -f installer/Dockerfile
--- a/.gitignore
+++ b/.gitignore
@ -8,7 +8,6 @@ __pycache__/

 # Distribution / packaging
 .Python
-build/
 develop-eggs/
 dist/
 downloads/
--- a/installer/Dockerfile
+++ b/installer/Dockerfile
@ -0,0 +1,42 @@
+FROM ghcr.io/maxkb-dev/python-pg:v1.0.0 as vector-model
+
+FROM node:18-alpine3.18 as web-build
+COPY ui ui
+RUN cd ui && \
+    npm install && \
+    npm run build && \
+    rm -rf ./node_modules
+
+FROM ghcr.io/maxkb-dev/python-pg:python3.11.8-pg15.6
+# 创建工作目录
+RUN mkdir -p /opt/maxkb/app && mkdir -p /opt/maxkb/model && mkdir -p /opt/maxkb/conf
+VOLUME /opt/maxkb
+# 拷贝项目
+COPY . /opt/maxkb/app
+COPY installer/config.yaml /opt/maxkb/conf
+RUN rm -rf /opt/maxkb/app/ui /opt/maxkb/app/build
+COPY --from=vector-model model /opt/maxkb/app/model
+COPY --from=web-build ui /opt/maxkb/app/ui
+RUN ls -la /opt/maxkb/app
+# 复制模型
+RUN mv /opt/maxkb/app/model/* /opt/maxkb/model
+RUN ls /opt/maxkb/model
+RUN cp -r /opt/maxkb/model/base/hub /opt/maxkb/model/tokenizer
+# 设置工作目录
+WORKDIR /opt/maxkb/app
+# 更新apt-get包管理器
+RUN apt-get update&&apt-get install -y  curl
+# 更新pip
+RUN pip3 install --upgrade pip
+# 安装 poetry包管理器
+RUN pip3 install poetry
+# 导出依赖
+RUN poetry export -f requirements.txt --output requirements.txt --without-hashes
+# 下载python依赖
+RUN pip3 install --no-cache-dir -r requirements.txt
+# 启动命令
+EXPOSE 8000
+COPY installer/run-maxkb.sh /usr/bin/
+RUN chmod 755 /usr/bin/run-maxkb.sh
+ENTRYPOINT ["bash", "-c"]
+CMD [ "/usr/bin/run-maxkb.sh" ]
--- a/installer/Dockerfile-python-pg
+++ b/installer/Dockerfile-python-pg
@ -1,17 +1,3 @@
-FROM python:3.11-slim as vector-model-build
-COPY install_model.py install_model.py
-RUN pip3 install --upgrade pip setuptools && \
-    pip install pycrawlers && \
-    pip install transformers && \
-    python3 install_model.py
-
-FROM node:18-alpine3.18 as web-build
-COPY ui ui
-RUN cd ui && \
-    npm install && \
-    npm run build && \
-    rm -rf ./node_modules
-
 FROM postgres:15.6-bookworm

 ENV LANG=C.UTF-8
@ -28,9 +14,6 @@ ENV POSTGRES_USER root

 ENV POSTGRES_PASSWORD Password123@postgres

-# ---- prepare python env --- #
-# ---- copy from https://github.com/docker-library/python/blob/master/3.11/slim-bookworm/Dockerfile --- #
-# ensure local python is preferred over distribution python
 ENV PATH /usr/local/bin:$PATH

 # http://bugs.python.org/issue19846
@ -191,38 +174,3 @@ RUN set -eux; \
 	rm -f get-pip.py; \
 	\
 	pip --version
-
-# ---- build maxkb --- #
-
-# 创建工作目录
-RUN mkdir -p /opt/maxkb/app && mkdir -p /opt/maxkb/model && mkdir -p /opt/maxkb/conf
-VOLUME /opt/maxkb
-# 拷贝项目
-COPY . /opt/maxkb/app
-COPY installer/config.yaml /opt/maxkb/conf
-RUN rm -rf /opt/maxkb/app/ui /opt/maxkb/app/installer
-COPY --from=vector-model-build model /opt/maxkb/app/model
-COPY --from=web-build ui /opt/maxkb/app/ui
-RUN ls -la /opt/maxkb/app
-# 复制模型
-RUN mv /opt/maxkb/app/model/* /opt/maxkb/model
-RUN ls /opt/maxkb/model
-RUN cp -r /opt/maxkb/model/base/hub /opt/maxkb/model/tokenizer
-# 设置工作目录
-WORKDIR /opt/maxkb/app
-# 更新apt-get包管理器
-RUN apt-get update&&apt-get install -y  curl
-# 更新pip
-RUN pip3 install --upgrade pip
-# 安装 poetry包管理器
-RUN pip3 install poetry
-# 导出依赖
-RUN poetry export -f requirements.txt --output requirements.txt --without-hashes
-# 下载python依赖
-RUN pip3 install --no-cache-dir -r requirements.txt
-EXPOSE 8000
-# 启动命令
-COPY installer/run-maxkb.sh /usr/bin/
-RUN chmod 755 /usr/bin/run-maxkb.sh
-ENTRYPOINT ["bash", "-c"]
-CMD [ "/usr/bin/run-maxkb.sh" ]
--- a/installer/Dockerfile-vector-model
+++ b/installer/Dockerfile-vector-model
@ -0,0 +1,6 @@
+FROM python:3.11-slim
+COPY installer/install_model.py install_model.py
+RUN pip3 install --upgrade pip setuptools && \
+    pip install pycrawlers && \
+    pip install transformers && \
+    python3 install_model.py
--- a/installer/install_model.py
+++ b/installer/install_model.py