diff --git a/.github/workflows/build-and-push-python-pg.yml b/.github/workflows/build-and-push-python-pg.yml new file mode 100644 index 000000000..478fdbab4 --- /dev/null +++ b/.github/workflows/build-and-push-python-pg.yml @@ -0,0 +1,41 @@ +name: build-and-push-python-pg + +on: + workflow_dispatch: + +jobs: + build-and-push-python-pg-to-ghcr: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: main + - name: Prepare + id: prepare + run: | + DOCKER_IMAGE=ghcr.io/maxkb-dev/python-pg + DOCKER_PLATFORMS=linux/amd64,linux/arm64 + TAG_NAME=python3.11.8-pg15.6 + DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:latest" + echo ::set-output name=docker_image::${DOCKER_IMAGE} + echo ::set-output name=version::${TAG_NAME} + echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \ + --build-arg VERSION=${TAG_NAME} \ + --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ + --build-arg --no-cache \ + --build-arg VCS_REF=${GITHUB_SHA::8} \ + ${DOCKER_IMAGE_TAGS} . + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GH_TOKEN }} + - name: Docker Buildx (build-and-push) + run: | + docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} -f installer/Dockerfile-python-pg \ No newline at end of file diff --git a/.github/workflows/build-and-push-vector-model.yml b/.github/workflows/build-and-push-vector-model.yml new file mode 100644 index 000000000..f83e5192f --- /dev/null +++ b/.github/workflows/build-and-push-vector-model.yml @@ -0,0 +1,55 @@ +name: build-and-push-vector-model + +on: + workflow_dispatch: + +jobs: + build-and-push-vector-model-to-ghcr: + runs-on: ubuntu-latest + steps: + - name: Check Disk Space + run: df -h + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + - name: Check Disk Space + run: df -h + - name: Checkout + uses: actions/checkout@v4 + with: + ref: main + - name: Prepare + id: prepare + run: | + DOCKER_IMAGE=ghcr.io/maxkb-dev/vector-model + DOCKER_PLATFORMS=linux/amd64,linux/arm64 + TAG_NAME=v1.0.0 + DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:latest" + echo ::set-output name=docker_image::${DOCKER_IMAGE} + echo ::set-output name=version::${TAG_NAME} + echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \ + --build-arg VERSION=${TAG_NAME} \ + --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ + --build-arg --no-cache \ + --build-arg VCS_REF=${GITHUB_SHA::8} \ + ${DOCKER_IMAGE_TAGS} . + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GH_TOKEN }} + - name: Docker Buildx (build-and-push) + run: | + docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} -f installer/Dockerfile-vector-model \ No newline at end of file diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 635cba113..2afe5e026 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -12,15 +12,20 @@ jobs: build-and-push-fit2cloud-registry: runs-on: ubuntu-latest steps: - - name: Maximize build space - uses: easimon/maximize-build-space@master + - name: Check Disk Space + run: df -h + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main with: - root-reserve-mb: 35840 - remove-dotnet: 'true' - remove-android: 'true' - remove-haskell: 'true' - remove-codeql: 'true' - remove-docker-images: 'true' + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + - name: Check Disk Space + run: df -h - name: Checkout uses: actions/checkout@v4 with: @@ -52,20 +57,25 @@ jobs: password: ${{ secrets.FIT2CLOUD_REGISTRY_PASSWORD }} - name: Docker Buildx (build-and-push) run: | - docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} + docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} -f installer/Dockerfile build-and-push-to-ghcr: runs-on: ubuntu-latest steps: - - name: Maximize build space - uses: easimon/maximize-build-space@master + - name: Check Disk Space + run: df -h + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main with: - root-reserve-mb: 35840 - remove-dotnet: 'true' - remove-android: 'true' - remove-haskell: 'true' - remove-codeql: 'true' - remove-docker-images: 'true' + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + - name: Check Disk Space + run: df -h - name: Checkout uses: actions/checkout@v4 with: @@ -97,4 +107,4 @@ jobs: password: ${{ secrets.GH_TOKEN }} - name: Docker Buildx (build-and-push) run: | - docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} \ No newline at end of file + docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} -f installer/Dockerfile \ No newline at end of file diff --git a/.gitignore b/.gitignore index fb5ff526a..7147555f8 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,6 @@ __pycache__/ # Distribution / packaging .Python -build/ develop-eggs/ dist/ downloads/ diff --git a/installer/Dockerfile b/installer/Dockerfile new file mode 100644 index 000000000..1d50b06e5 --- /dev/null +++ b/installer/Dockerfile @@ -0,0 +1,42 @@ +FROM ghcr.io/maxkb-dev/python-pg:v1.0.0 as vector-model + +FROM node:18-alpine3.18 as web-build +COPY ui ui +RUN cd ui && \ + npm install && \ + npm run build && \ + rm -rf ./node_modules + +FROM ghcr.io/maxkb-dev/python-pg:python3.11.8-pg15.6 +# 创建工作目录 +RUN mkdir -p /opt/maxkb/app && mkdir -p /opt/maxkb/model && mkdir -p /opt/maxkb/conf +VOLUME /opt/maxkb +# 拷贝项目 +COPY . /opt/maxkb/app +COPY installer/config.yaml /opt/maxkb/conf +RUN rm -rf /opt/maxkb/app/ui /opt/maxkb/app/build +COPY --from=vector-model model /opt/maxkb/app/model +COPY --from=web-build ui /opt/maxkb/app/ui +RUN ls -la /opt/maxkb/app +# 复制模型 +RUN mv /opt/maxkb/app/model/* /opt/maxkb/model +RUN ls /opt/maxkb/model +RUN cp -r /opt/maxkb/model/base/hub /opt/maxkb/model/tokenizer +# 设置工作目录 +WORKDIR /opt/maxkb/app +# 更新apt-get包管理器 +RUN apt-get update&&apt-get install -y curl +# 更新pip +RUN pip3 install --upgrade pip +# 安装 poetry包管理器 +RUN pip3 install poetry +# 导出依赖 +RUN poetry export -f requirements.txt --output requirements.txt --without-hashes +# 下载python依赖 +RUN pip3 install --no-cache-dir -r requirements.txt +# 启动命令 +EXPOSE 8000 +COPY installer/run-maxkb.sh /usr/bin/ +RUN chmod 755 /usr/bin/run-maxkb.sh +ENTRYPOINT ["bash", "-c"] +CMD [ "/usr/bin/run-maxkb.sh" ] \ No newline at end of file diff --git a/Dockerfile b/installer/Dockerfile-python-pg similarity index 75% rename from Dockerfile rename to installer/Dockerfile-python-pg index c7cab0d44..0032f9da0 100644 --- a/Dockerfile +++ b/installer/Dockerfile-python-pg @@ -1,17 +1,3 @@ -FROM python:3.11-slim as vector-model-build -COPY install_model.py install_model.py -RUN pip3 install --upgrade pip setuptools && \ - pip install pycrawlers && \ - pip install transformers && \ - python3 install_model.py - -FROM node:18-alpine3.18 as web-build -COPY ui ui -RUN cd ui && \ - npm install && \ - npm run build && \ - rm -rf ./node_modules - FROM postgres:15.6-bookworm ENV LANG=C.UTF-8 @@ -28,9 +14,6 @@ ENV POSTGRES_USER root ENV POSTGRES_PASSWORD Password123@postgres -# ---- prepare python env --- # -# ---- copy from https://github.com/docker-library/python/blob/master/3.11/slim-bookworm/Dockerfile --- # -# ensure local python is preferred over distribution python ENV PATH /usr/local/bin:$PATH # http://bugs.python.org/issue19846 @@ -191,38 +174,3 @@ RUN set -eux; \ rm -f get-pip.py; \ \ pip --version - -# ---- build maxkb --- # - -# 创建工作目录 -RUN mkdir -p /opt/maxkb/app && mkdir -p /opt/maxkb/model && mkdir -p /opt/maxkb/conf -VOLUME /opt/maxkb -# 拷贝项目 -COPY . /opt/maxkb/app -COPY installer/config.yaml /opt/maxkb/conf -RUN rm -rf /opt/maxkb/app/ui /opt/maxkb/app/installer -COPY --from=vector-model-build model /opt/maxkb/app/model -COPY --from=web-build ui /opt/maxkb/app/ui -RUN ls -la /opt/maxkb/app -# 复制模型 -RUN mv /opt/maxkb/app/model/* /opt/maxkb/model -RUN ls /opt/maxkb/model -RUN cp -r /opt/maxkb/model/base/hub /opt/maxkb/model/tokenizer -# 设置工作目录 -WORKDIR /opt/maxkb/app -# 更新apt-get包管理器 -RUN apt-get update&&apt-get install -y curl -# 更新pip -RUN pip3 install --upgrade pip -# 安装 poetry包管理器 -RUN pip3 install poetry -# 导出依赖 -RUN poetry export -f requirements.txt --output requirements.txt --without-hashes -# 下载python依赖 -RUN pip3 install --no-cache-dir -r requirements.txt -EXPOSE 8000 -# 启动命令 -COPY installer/run-maxkb.sh /usr/bin/ -RUN chmod 755 /usr/bin/run-maxkb.sh -ENTRYPOINT ["bash", "-c"] -CMD [ "/usr/bin/run-maxkb.sh" ] \ No newline at end of file diff --git a/installer/Dockerfile-vector-model b/installer/Dockerfile-vector-model new file mode 100644 index 000000000..97e35b4e4 --- /dev/null +++ b/installer/Dockerfile-vector-model @@ -0,0 +1,6 @@ +FROM python:3.11-slim +COPY installer/install_model.py install_model.py +RUN pip3 install --upgrade pip setuptools && \ + pip install pycrawlers && \ + pip install transformers && \ + python3 install_model.py \ No newline at end of file diff --git a/install_model.py b/installer/install_model.py similarity index 100% rename from install_model.py rename to installer/install_model.py