From 35d20714986b7407b36f1126a482ab95a7fd3a13 Mon Sep 17 00:00:00 2001 From: liqiang-fit2cloud Date: Fri, 15 Mar 2024 10:26:41 +0800 Subject: [PATCH] =?UTF-8?q?build:=20=E5=B0=9D=E8=AF=95=E6=89=93=E5=8C=85al?= =?UTF-8?q?linone?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build-and-push-allinone.yml | 54 +++++ Dockerfile-allinone | 228 ++++++++++++++++++ installer/config.yaml | 20 ++ installer/init.sql | 5 + installer/run-maxkb.sh | 16 ++ 5 files changed, 323 insertions(+) create mode 100644 .github/workflows/build-and-push-allinone.yml create mode 100644 Dockerfile-allinone create mode 100644 installer/config.yaml create mode 100644 installer/init.sql create mode 100644 installer/run-maxkb.sh diff --git a/.github/workflows/build-and-push-allinone.yml b/.github/workflows/build-and-push-allinone.yml new file mode 100644 index 000000000..9854b6cb7 --- /dev/null +++ b/.github/workflows/build-and-push-allinone.yml @@ -0,0 +1,54 @@ +name: build-and-push + +on: + workflow_dispatch: + inputs: + registryAddress: + description: 'Registry Address' + default: 'registry-hkproxy.fit2cloud.com' + required: true + dockerImageName: + description: 'Docker Image Name' + default: 'maxkb/maxkb' + required: true + dockerImageTag: + description: 'Docker Image Tag' + default: 'v1.0.0' + required: true + +jobs: + build-and-push-to-ghcr: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: main + - name: Prepare + id: prepare + run: | + DOCKER_IMAGE=ghcr.io/maxkb-dev/maxkb + DOCKER_PLATFORMS=linux/amd64 + TAG_NAME=${{ github.event.inputs.dockerImageTag }} + DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:latest" + echo ::set-output name=docker_image::${DOCKER_IMAGE} + echo ::set-output name=version::${TAG_NAME} + echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \ + --build-arg VERSION=${TAG_NAME} \ + --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ + --build-arg --no-cache \ + --build-arg VCS_REF=${GITHUB_SHA::8} \ + ${DOCKER_IMAGE_TAGS} . + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GH_TOKEN }} + - name: Docker Buildx (build-and-push) + run: | + docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} -f Dockerfile-allinone diff --git a/Dockerfile-allinone b/Dockerfile-allinone new file mode 100644 index 000000000..1cfbb70dc --- /dev/null +++ b/Dockerfile-allinone @@ -0,0 +1,228 @@ +FROM python:3.11-slim as vector-model-build +COPY install_model.py install_model.py +RUN pip3 install --upgrade pip setuptools && \ + pip install pycrawlers && \ + pip install transformers && \ + python3 install_model.py + +FROM node:18-alpine3.18 as web-build +COPY ui ui +RUN cd ui && \ + npm install && \ + npm run build && \ + rm -rf ./node_modules + +FROM postgres:15.6-bookworm + +ENV LANG=C.UTF-8 + +RUN apt-get update + +RUN apt-get install postgresql-15-pgvector + +COPY installer/init.sql /docker-entrypoint-initdb.d + +RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo "Asia/Shanghai" > /etc/timezone + +ENV POSTGRES_USER root + +ENV POSTGRES_PASSWORD Password123@mysql + +# ---- prepare python env --- # +# ---- copy from https://github.com/docker-library/python/blob/master/3.11/slim-bookworm/Dockerfile --- # +# ensure local python is preferred over distribution python +ENV PATH /usr/local/bin:$PATH + +# http://bugs.python.org/issue19846 +# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. +ENV LANG C.UTF-8 + +# runtime dependencies +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + netbase \ + tzdata \ + ; \ + rm -rf /var/lib/apt/lists/* + +ENV GPG_KEY A035C8C19219BA821ECEA86B64E628F8D684696D +ENV PYTHON_VERSION 3.11.8 + +RUN set -eux; \ + \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + dpkg-dev \ + gcc \ + gnupg \ + libbluetooth-dev \ + libbz2-dev \ + libc6-dev \ + libdb-dev \ + libexpat1-dev \ + libffi-dev \ + libgdbm-dev \ + liblzma-dev \ + libncursesw5-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + make \ + tk-dev \ + uuid-dev \ + wget \ + xz-utils \ + zlib1g-dev \ + ; \ + \ + wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz"; \ + wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc"; \ + GNUPGHOME="$(mktemp -d)"; export GNUPGHOME; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$GPG_KEY"; \ + gpg --batch --verify python.tar.xz.asc python.tar.xz; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" python.tar.xz.asc; \ + mkdir -p /usr/src/python; \ + tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz; \ + rm python.tar.xz; \ + \ + cd /usr/src/python; \ + gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \ + ./configure \ + --build="$gnuArch" \ + --enable-loadable-sqlite-extensions \ + --enable-optimizations \ + --enable-option-checking=fatal \ + --enable-shared \ + --with-lto \ + --with-system-expat \ + --without-ensurepip \ + ; \ + nproc="$(nproc)"; \ + EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)"; \ + LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"; \ + LDFLAGS="${LDFLAGS:--Wl},--strip-all"; \ + make -j "$nproc" \ + "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ + "LDFLAGS=${LDFLAGS:-}" \ + "PROFILE_TASK=${PROFILE_TASK:-}" \ + ; \ +# https://github.com/docker-library/python/issues/784 +# prevent accidental usage of a system installed libpython of the same version + rm python; \ + make -j "$nproc" \ + "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ + "LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" \ + "PROFILE_TASK=${PROFILE_TASK:-}" \ + python \ + ; \ + make install; \ + \ + cd /; \ + rm -rf /usr/src/python; \ + \ + find /usr/local -depth \ + \( \ + \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \ + -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \) \) \ + \) -exec rm -rf '{}' + \ + ; \ + \ + ldconfig; \ + \ + apt-mark auto '.*' > /dev/null; \ + apt-mark manual $savedAptMark; \ + find /usr/local -type f -executable -not \( -name '*tkinter*' \) -exec ldd '{}' ';' \ + | awk '/=>/ { so = $(NF-1); if (index(so, "/usr/local/") == 1) { next }; gsub("^/(usr/)?", "", so); printf "*%s\n", so }' \ + | sort -u \ + | xargs -r dpkg-query --search \ + | cut -d: -f1 \ + | sort -u \ + | xargs -r apt-mark manual \ + ; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + rm -rf /var/lib/apt/lists/*; \ + \ + python3 --version + +# make some useful symlinks that are expected to exist ("/usr/local/bin/python" and friends) +RUN set -eux; \ + for src in idle3 pydoc3 python3 python3-config; do \ + dst="$(echo "$src" | tr -d 3)"; \ + [ -s "/usr/local/bin/$src" ]; \ + [ ! -e "/usr/local/bin/$dst" ]; \ + ln -svT "$src" "/usr/local/bin/$dst"; \ + done + +# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" +ENV PYTHON_PIP_VERSION 24.0 +# https://github.com/docker-library/python/issues/365 +ENV PYTHON_SETUPTOOLS_VERSION 65.5.1 +# https://github.com/pypa/get-pip +ENV PYTHON_GET_PIP_URL https://github.com/pypa/get-pip/raw/dbf0c85f76fb6e1ab42aa672ffca6f0a675d9ee4/public/get-pip.py +ENV PYTHON_GET_PIP_SHA256 dfe9fd5c28dc98b5ac17979a953ea550cec37ae1b47a5116007395bfacff2ab9 + +RUN set -eux; \ + \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + apt-get install -y --no-install-recommends wget; \ + \ + wget -O get-pip.py "$PYTHON_GET_PIP_URL"; \ + echo "$PYTHON_GET_PIP_SHA256 *get-pip.py" | sha256sum -c -; \ + \ + apt-mark auto '.*' > /dev/null; \ + [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + rm -rf /var/lib/apt/lists/*; \ + \ + export PYTHONDONTWRITEBYTECODE=1; \ + \ + python get-pip.py \ + --disable-pip-version-check \ + --no-cache-dir \ + --no-compile \ + "pip==$PYTHON_PIP_VERSION" \ + "setuptools==$PYTHON_SETUPTOOLS_VERSION" \ + ; \ + rm -f get-pip.py; \ + \ + pip --version + +# ---- build maxkb --- # + +# 创建工作目录 +RUN mkdir -p /opt/maxkb/app && mkdir -p /opt/maxkb/model && mkdir -p /opt/maxkb/conf +VOLUME /opt/maxkb +# 拷贝项目 +COPY . /opt/maxkb/app +COPY installer/config.yaml /opt/maxkb/conf +RUN rm -rf /opt/maxkb/app/ui /opt/maxkb/app/installer +COPY --from=vector-model-build model /opt/maxkb/app/model +COPY --from=web-build ui /opt/maxkb/app/ui +RUN ls -la /opt/maxkb/app +# 复制模型 +RUN mv /opt/maxkb/app/model/* /opt/maxkb/model +RUN ls /opt/maxkb/model +RUN cp -r /opt/maxkb/model/base/hub /opt/maxkb/model/tokenizer +# 设置工作目录 +WORKDIR /opt/maxkb/app +# 更新apt-get包管理器 +RUN apt-get update&&apt-get install -y curl +# 更新pip +RUN pip3 install --upgrade pip +# 安装 poetry包管理器 +RUN pip3 install poetry +# 导出依赖 +RUN poetry export -f requirements.txt --output requirements.txt --without-hashes +# 下载python依赖 +RUN pip3 install --no-cache-dir -r requirements.txt +EXPOSE 8000 +# 启动命令 +COPY installer/run-maxkb.sh /usr/bin/ +RUN chmod 755 /usr/bin/run-maxkb.sh +ENTRYPOINT ["bash", "-c"] +CMD [ "/usr/bin/run-maxkb.sh" ] \ No newline at end of file diff --git a/installer/config.yaml b/installer/config.yaml new file mode 100644 index 000000000..9dae5f276 --- /dev/null +++ b/installer/config.yaml @@ -0,0 +1,20 @@ +# 邮箱配置 +EMAIL_ADDRESS: ${EMAIL_ADDRESS} +EMAIL_USE_TLS: ${EMAIL_USE_TLS} +EMAIL_USE_SSL: ${EMAIL_USE_SSL} +EMAIL_HOST: ${EMAIL_HOST} +EMAIL_PORT: ${EMAIL_PORT} +EMAIL_HOST_USER: ${EMAIL_HOST_USER} +EMAIL_HOST_PASSWORD: ${EMAIL_HOST_PASSWORD} +# +# # 数据库链接信息 +DB_NAME: maxkb +DB_HOST: 127.0.0.1 +DB_PORT: 5432 +DB_USER: root +DB_PASSWORD: Password123@mysql +DB_ENGINE: django.db.backends.postgresql_psycopg2 +EMBEDDING_MODEL_PATH: /opt/maxkb/model/embedding +EMBEDDING_MODEL_NAME: /opt/maxkb/model/embedding/shibing624_text2vec-base-chinese + +DEBUG: false \ No newline at end of file diff --git a/installer/init.sql b/installer/init.sql new file mode 100644 index 000000000..98550891a --- /dev/null +++ b/installer/init.sql @@ -0,0 +1,5 @@ +CREATE DATABASE "maxkb"; + +\c "maxkb"; + +CREATE EXTENSION "vector" VERSION '0.5.1'; \ No newline at end of file diff --git a/installer/run-maxkb.sh b/installer/run-maxkb.sh new file mode 100644 index 000000000..0c26a471d --- /dev/null +++ b/installer/run-maxkb.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Start postgress +docker-entrypoint.sh postgres & + +# Wait postgress +until pg_isready --host=3333; do sleep 1 && echo "waiting for postgres"; done + +# Start MaxKB +python /opt/maxkb/app/main.py start & + +# Wait for any process to exit +wait -n + +# Exit with status of process that exited first +exit $? \ No newline at end of file