From af413498fa9f5efbab0134a76f76f9ff5ad77379 Mon Sep 17 00:00:00 2001 From: liqiang-fit2cloud Date: Fri, 15 Mar 2024 11:28:46 +0800 Subject: [PATCH 1/8] =?UTF-8?q?build:=20allinone=E6=89=93=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build-and-push.yml | 42 +++++++++++++++++----------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 635cba113..cd3cd274a 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -12,15 +12,20 @@ jobs: build-and-push-fit2cloud-registry: runs-on: ubuntu-latest steps: - - name: Maximize build space - uses: easimon/maximize-build-space@master + - name: Check Disk Space + run: df -h + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main with: - root-reserve-mb: 35840 - remove-dotnet: 'true' - remove-android: 'true' - remove-haskell: 'true' - remove-codeql: 'true' - remove-docker-images: 'true' + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + - name: Check Disk Space + run: df -h - name: Checkout uses: actions/checkout@v4 with: @@ -57,15 +62,20 @@ jobs: build-and-push-to-ghcr: runs-on: ubuntu-latest steps: - - name: Maximize build space - uses: easimon/maximize-build-space@master + - name: Check Disk Space + run: df -h + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main with: - root-reserve-mb: 35840 - remove-dotnet: 'true' - remove-android: 'true' - remove-haskell: 'true' - remove-codeql: 'true' - remove-docker-images: 'true' + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + - name: Check Disk Space + run: df -h - name: Checkout uses: actions/checkout@v4 with: From 985d476207dba40f25fa5f8fc4af8c5fdc483617 Mon Sep 17 00:00:00 2001 From: liqiang-fit2cloud Date: Fri, 15 Mar 2024 12:58:46 +0800 Subject: [PATCH 2/8] =?UTF-8?q?build:=20allinone=E6=89=93=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 174 ++--------------------------------------- install_model.py | 69 ---------------- installer/config.yaml | 20 ----- installer/init.sql | 5 -- installer/run-maxkb.sh | 16 ---- 5 files changed, 8 insertions(+), 276 deletions(-) delete mode 100644 install_model.py delete mode 100644 installer/config.yaml delete mode 100644 installer/init.sql delete mode 100644 installer/run-maxkb.sh diff --git a/Dockerfile b/Dockerfile index c7cab0d44..db573c475 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM python:3.11-slim as vector-model-build -COPY install_model.py install_model.py +COPY build/install_model.py install_model.py RUN pip3 install --upgrade pip setuptools && \ pip install pycrawlers && \ pip install transformers && \ @@ -20,7 +20,7 @@ RUN apt-get update RUN apt-get install postgresql-15-pgvector -COPY installer/init.sql /docker-entrypoint-initdb.d +COPY build/init.sql /docker-entrypoint-initdb.d RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo "Asia/Shanghai" > /etc/timezone @@ -28,179 +28,21 @@ ENV POSTGRES_USER root ENV POSTGRES_PASSWORD Password123@postgres + # ---- prepare python env --- # -# ---- copy from https://github.com/docker-library/python/blob/master/3.11/slim-bookworm/Dockerfile --- # -# ensure local python is preferred over distribution python ENV PATH /usr/local/bin:$PATH - -# http://bugs.python.org/issue19846 -# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. -ENV LANG C.UTF-8 - -# runtime dependencies -RUN set -eux; \ - apt-get update; \ - apt-get install -y --no-install-recommends \ - ca-certificates \ - netbase \ - tzdata \ - ; \ - rm -rf /var/lib/apt/lists/* - -ENV GPG_KEY A035C8C19219BA821ECEA86B64E628F8D684696D +COPY build/install-python.sh /install-python.sh +RUN chmod 755 /install-python.sh; bash -c "/install-python.sh > /dev/null 2>&1" ; rm -f /install-python.sh ENV PYTHON_VERSION 3.11.8 -RUN set -eux; \ - \ - savedAptMark="$(apt-mark showmanual)"; \ - apt-get update; \ - apt-get install -y --no-install-recommends \ - dpkg-dev \ - gcc \ - gnupg \ - libbluetooth-dev \ - libbz2-dev \ - libc6-dev \ - libdb-dev \ - libexpat1-dev \ - libffi-dev \ - libgdbm-dev \ - liblzma-dev \ - libncursesw5-dev \ - libreadline-dev \ - libsqlite3-dev \ - libssl-dev \ - make \ - tk-dev \ - uuid-dev \ - wget \ - xz-utils \ - zlib1g-dev \ - ; \ - \ - wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz"; \ - wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc"; \ - GNUPGHOME="$(mktemp -d)"; export GNUPGHOME; \ - gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$GPG_KEY"; \ - gpg --batch --verify python.tar.xz.asc python.tar.xz; \ - gpgconf --kill all; \ - rm -rf "$GNUPGHOME" python.tar.xz.asc; \ - mkdir -p /usr/src/python; \ - tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz; \ - rm python.tar.xz; \ - \ - cd /usr/src/python; \ - gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \ - ./configure \ - --build="$gnuArch" \ - --enable-loadable-sqlite-extensions \ - --enable-optimizations \ - --enable-option-checking=fatal \ - --enable-shared \ - --with-lto \ - --with-system-expat \ - --without-ensurepip \ - ; \ - nproc="$(nproc)"; \ - EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)"; \ - LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"; \ - LDFLAGS="${LDFLAGS:--Wl},--strip-all"; \ - make -j "$nproc" \ - "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ - "LDFLAGS=${LDFLAGS:-}" \ - "PROFILE_TASK=${PROFILE_TASK:-}" \ - ; \ -# https://github.com/docker-library/python/issues/784 -# prevent accidental usage of a system installed libpython of the same version - rm python; \ - make -j "$nproc" \ - "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ - "LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" \ - "PROFILE_TASK=${PROFILE_TASK:-}" \ - python \ - ; \ - make install; \ - \ - cd /; \ - rm -rf /usr/src/python; \ - \ - find /usr/local -depth \ - \( \ - \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \ - -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \) \) \ - \) -exec rm -rf '{}' + \ - ; \ - \ - ldconfig; \ - \ - apt-mark auto '.*' > /dev/null; \ - apt-mark manual $savedAptMark; \ - find /usr/local -type f -executable -not \( -name '*tkinter*' \) -exec ldd '{}' ';' \ - | awk '/=>/ { so = $(NF-1); if (index(so, "/usr/local/") == 1) { next }; gsub("^/(usr/)?", "", so); printf "*%s\n", so }' \ - | sort -u \ - | xargs -r dpkg-query --search \ - | cut -d: -f1 \ - | sort -u \ - | xargs -r apt-mark manual \ - ; \ - apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ - rm -rf /var/lib/apt/lists/*; \ - \ - python3 --version - -# make some useful symlinks that are expected to exist ("/usr/local/bin/python" and friends) -RUN set -eux; \ - for src in idle3 pydoc3 python3 python3-config; do \ - dst="$(echo "$src" | tr -d 3)"; \ - [ -s "/usr/local/bin/$src" ]; \ - [ ! -e "/usr/local/bin/$dst" ]; \ - ln -svT "$src" "/usr/local/bin/$dst"; \ - done - -# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 24.0 -# https://github.com/docker-library/python/issues/365 -ENV PYTHON_SETUPTOOLS_VERSION 65.5.1 -# https://github.com/pypa/get-pip -ENV PYTHON_GET_PIP_URL https://github.com/pypa/get-pip/raw/dbf0c85f76fb6e1ab42aa672ffca6f0a675d9ee4/public/get-pip.py -ENV PYTHON_GET_PIP_SHA256 dfe9fd5c28dc98b5ac17979a953ea550cec37ae1b47a5116007395bfacff2ab9 - -RUN set -eux; \ - \ - savedAptMark="$(apt-mark showmanual)"; \ - apt-get update; \ - apt-get install -y --no-install-recommends wget; \ - \ - wget -O get-pip.py "$PYTHON_GET_PIP_URL"; \ - echo "$PYTHON_GET_PIP_SHA256 *get-pip.py" | sha256sum -c -; \ - \ - apt-mark auto '.*' > /dev/null; \ - [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ - apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ - rm -rf /var/lib/apt/lists/*; \ - \ - export PYTHONDONTWRITEBYTECODE=1; \ - \ - python get-pip.py \ - --disable-pip-version-check \ - --no-cache-dir \ - --no-compile \ - "pip==$PYTHON_PIP_VERSION" \ - "setuptools==$PYTHON_SETUPTOOLS_VERSION" \ - ; \ - rm -f get-pip.py; \ - \ - pip --version - # ---- build maxkb --- # - # 创建工作目录 RUN mkdir -p /opt/maxkb/app && mkdir -p /opt/maxkb/model && mkdir -p /opt/maxkb/conf VOLUME /opt/maxkb # 拷贝项目 COPY . /opt/maxkb/app -COPY installer/config.yaml /opt/maxkb/conf -RUN rm -rf /opt/maxkb/app/ui /opt/maxkb/app/installer +COPY build/config.yaml /opt/maxkb/conf +RUN rm -rf /opt/maxkb/app/ui /opt/maxkb/app/build COPY --from=vector-model-build model /opt/maxkb/app/model COPY --from=web-build ui /opt/maxkb/app/ui RUN ls -la /opt/maxkb/app @@ -222,7 +64,7 @@ RUN poetry export -f requirements.txt --output requirements.txt --without-hashes RUN pip3 install --no-cache-dir -r requirements.txt EXPOSE 8000 # 启动命令 -COPY installer/run-maxkb.sh /usr/bin/ +COPY build/run-maxkb.sh /usr/bin/ RUN chmod 755 /usr/bin/run-maxkb.sh ENTRYPOINT ["bash", "-c"] CMD [ "/usr/bin/run-maxkb.sh" ] \ No newline at end of file diff --git a/install_model.py b/install_model.py deleted file mode 100644 index fb464611a..000000000 --- a/install_model.py +++ /dev/null @@ -1,69 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： install_model.py - @date：2023/12/18 14:02 - @desc: -""" -import json -import os.path -from pycrawlers import huggingface -from transformers import GPT2TokenizerFast -hg = huggingface() -prefix_dir = "./model" -model_config = [ - { - 'download_params': { - 'cache_dir': os.path.join(prefix_dir, 'base/hub'), - 'pretrained_model_name_or_path': 'gpt2' - }, - 'download_function': GPT2TokenizerFast.from_pretrained - }, - { - 'download_params': { - 'cache_dir': os.path.join(prefix_dir, 'base/hub'), - 'pretrained_model_name_or_path': 'gpt2-medium' - }, - 'download_function': GPT2TokenizerFast.from_pretrained - }, - { - 'download_params': { - 'cache_dir': os.path.join(prefix_dir, 'base/hub'), - 'pretrained_model_name_or_path': 'gpt2-large' - }, - 'download_function': GPT2TokenizerFast.from_pretrained - }, - { - 'download_params': { - 'cache_dir': os.path.join(prefix_dir, 'base/hub'), - 'pretrained_model_name_or_path': 'gpt2-xl' - }, - 'download_function': GPT2TokenizerFast.from_pretrained - }, - { - 'download_params': { - 'cache_dir': os.path.join(prefix_dir, 'base/hub'), - 'pretrained_model_name_or_path': 'distilgpt2' - }, - 'download_function': GPT2TokenizerFast.from_pretrained - }, - { - 'download_params': { - 'urls': ["https://huggingface.co/shibing624/text2vec-base-chinese/tree/main"], - 'file_save_paths': [os.path.join(prefix_dir, 'embedding',"shibing624_text2vec-base-chinese")] - }, - 'download_function': hg.get_batch_data - } - -] - - -def install(): - for model in model_config: - print(json.dumps(model.get('download_params'))) - model.get('download_function')(**model.get('download_params')) - - -if __name__ == '__main__': - install() diff --git a/installer/config.yaml b/installer/config.yaml deleted file mode 100644 index 9e19ef0b3..000000000 --- a/installer/config.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# 邮箱配置 -EMAIL_ADDRESS: ${EMAIL_ADDRESS} -EMAIL_USE_TLS: ${EMAIL_USE_TLS} -EMAIL_USE_SSL: ${EMAIL_USE_SSL} -EMAIL_HOST: ${EMAIL_HOST} -EMAIL_PORT: ${EMAIL_PORT} -EMAIL_HOST_USER: ${EMAIL_HOST_USER} -EMAIL_HOST_PASSWORD: ${EMAIL_HOST_PASSWORD} -# -# # 数据库链接信息 -DB_NAME: maxkb -DB_HOST: 127.0.0.1 -DB_PORT: 5432 -DB_USER: root -DB_PASSWORD: Password123@postgres -DB_ENGINE: django.db.backends.postgresql_psycopg2 -EMBEDDING_MODEL_PATH: /opt/maxkb/model/embedding -EMBEDDING_MODEL_NAME: /opt/maxkb/model/embedding/shibing624_text2vec-base-chinese - -DEBUG: false \ No newline at end of file diff --git a/installer/init.sql b/installer/init.sql deleted file mode 100644 index 98550891a..000000000 --- a/installer/init.sql +++ /dev/null @@ -1,5 +0,0 @@ -CREATE DATABASE "maxkb"; - -\c "maxkb"; - -CREATE EXTENSION "vector" VERSION '0.5.1'; \ No newline at end of file diff --git a/installer/run-maxkb.sh b/installer/run-maxkb.sh deleted file mode 100644 index 4f388f695..000000000 --- a/installer/run-maxkb.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# Start postgress -docker-entrypoint.sh postgres & - -# Wait postgress -until pg_isready --host=127.0.0.1; do sleep 1 && echo "waiting for postgres"; done - -# Start MaxKB -python /opt/maxkb/app/main.py start & - -# Wait for any process to exit -wait -n - -# Exit with status of process that exited first -exit $? \ No newline at end of file From 3acc202a76dc6396a6dfad8afbb68b39c3db9251 Mon Sep 17 00:00:00 2001 From: liqiang-fit2cloud Date: Fri, 15 Mar 2024 13:01:33 +0800 Subject: [PATCH 3/8] =?UTF-8?q?build:=20allinone=E6=89=93=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 +- Dockerfile | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index fb5ff526a..a3dcf531b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ __pycache__/ # Distribution / packaging .Python -build/ +installer/ develop-eggs/ dist/ downloads/ diff --git a/Dockerfile b/Dockerfile index db573c475..746d033e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM python:3.11-slim as vector-model-build -COPY build/install_model.py install_model.py +COPY installer/install_model.py install_model.py RUN pip3 install --upgrade pip setuptools && \ pip install pycrawlers && \ pip install transformers && \ @@ -20,7 +20,7 @@ RUN apt-get update RUN apt-get install postgresql-15-pgvector -COPY build/init.sql /docker-entrypoint-initdb.d +COPY installer/init.sql /docker-entrypoint-initdb.d RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo "Asia/Shanghai" > /etc/timezone @@ -31,7 +31,7 @@ ENV POSTGRES_PASSWORD Password123@postgres # ---- prepare python env --- # ENV PATH /usr/local/bin:$PATH -COPY build/install-python.sh /install-python.sh +COPY installer/install-python.sh /install-python.sh RUN chmod 755 /install-python.sh; bash -c "/install-python.sh > /dev/null 2>&1" ; rm -f /install-python.sh ENV PYTHON_VERSION 3.11.8 @@ -41,7 +41,7 @@ RUN mkdir -p /opt/maxkb/app && mkdir -p /opt/maxkb/model && mkdir -p /opt/maxkb/ VOLUME /opt/maxkb # 拷贝项目 COPY . /opt/maxkb/app -COPY build/config.yaml /opt/maxkb/conf +COPY installer/config.yaml /opt/maxkb/conf RUN rm -rf /opt/maxkb/app/ui /opt/maxkb/app/build COPY --from=vector-model-build model /opt/maxkb/app/model COPY --from=web-build ui /opt/maxkb/app/ui @@ -64,7 +64,7 @@ RUN poetry export -f requirements.txt --output requirements.txt --without-hashes RUN pip3 install --no-cache-dir -r requirements.txt EXPOSE 8000 # 启动命令 -COPY build/run-maxkb.sh /usr/bin/ +COPY installer/run-maxkb.sh /usr/bin/ RUN chmod 755 /usr/bin/run-maxkb.sh ENTRYPOINT ["bash", "-c"] CMD [ "/usr/bin/run-maxkb.sh" ] \ No newline at end of file From ba72889a183c611ec666ba329d23e9b0a4b8c850 Mon Sep 17 00:00:00 2001 From: liqiang-fit2cloud Date: Fri, 15 Mar 2024 13:03:01 +0800 Subject: [PATCH 4/8] =?UTF-8?q?build:=20allinone=E6=89=93=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 - installer/config.yaml | 20 +++++++ installer/init.sql | 5 ++ installer/install-python.sh | 101 ++++++++++++++++++++++++++++++++++++ installer/install_model.py | 69 ++++++++++++++++++++++++ installer/run-maxkb.sh | 16 ++++++ 6 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 installer/config.yaml create mode 100644 installer/init.sql create mode 100644 installer/install-python.sh create mode 100644 installer/install_model.py create mode 100644 installer/run-maxkb.sh diff --git a/.gitignore b/.gitignore index a3dcf531b..7147555f8 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,6 @@ __pycache__/ # Distribution / packaging .Python -installer/ develop-eggs/ dist/ downloads/ diff --git a/installer/config.yaml b/installer/config.yaml new file mode 100644 index 000000000..9e19ef0b3 --- /dev/null +++ b/installer/config.yaml @@ -0,0 +1,20 @@ +# 邮箱配置 +EMAIL_ADDRESS: ${EMAIL_ADDRESS} +EMAIL_USE_TLS: ${EMAIL_USE_TLS} +EMAIL_USE_SSL: ${EMAIL_USE_SSL} +EMAIL_HOST: ${EMAIL_HOST} +EMAIL_PORT: ${EMAIL_PORT} +EMAIL_HOST_USER: ${EMAIL_HOST_USER} +EMAIL_HOST_PASSWORD: ${EMAIL_HOST_PASSWORD} +# +# # 数据库链接信息 +DB_NAME: maxkb +DB_HOST: 127.0.0.1 +DB_PORT: 5432 +DB_USER: root +DB_PASSWORD: Password123@postgres +DB_ENGINE: django.db.backends.postgresql_psycopg2 +EMBEDDING_MODEL_PATH: /opt/maxkb/model/embedding +EMBEDDING_MODEL_NAME: /opt/maxkb/model/embedding/shibing624_text2vec-base-chinese + +DEBUG: false \ No newline at end of file diff --git a/installer/init.sql b/installer/init.sql new file mode 100644 index 000000000..98550891a --- /dev/null +++ b/installer/init.sql @@ -0,0 +1,5 @@ +CREATE DATABASE "maxkb"; + +\c "maxkb"; + +CREATE EXTENSION "vector" VERSION '0.5.1'; \ No newline at end of file diff --git a/installer/install-python.sh b/installer/install-python.sh new file mode 100644 index 000000000..0835f28cc --- /dev/null +++ b/installer/install-python.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +# ---- prepare python env --- # +# ---- copy from https://github.com/docker-library/python/blob/master/3.11/slim-bookworm/Dockerfile --- # + +export PATH=/usr/local/bin:$PATH + +export LANG=C.UTF-8 + +apt-get update; +apt-get install -y --no-install-recommends ca-certificates netbase tzdata +rm -rf /var/lib/apt/lists/* + +export GPG_KEY=A035C8C19219BA821ECEA86B64E628F8D684696D +export PYTHON_VERSION=3.11.8 + +savedAptMark="$(apt-mark showmanual)" +apt-get update +apt-get install -y --no-install-recommends dpkg-dev gcc gnupg libbluetooth-dev libbz2-dev libc6-dev libdb-dev libexpat1-dev libffi-dev libgdbm-dev liblzma-dev libncursesw5-dev libreadline-dev libsqlite3-dev libssl-dev make \ + tk-dev uuid-dev wget xz-utils zlib1g-dev +wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" +wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" +GNUPGHOME="$(mktemp -d)" +export GNUPGHOME +gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$GPG_KEY" +gpg --batch --verify python.tar.xz.asc python.tar.xz +gpgconf --kill all +rm -rf "$GNUPGHOME" python.tar.xz.asc +mkdir -p /usr/src/python +tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz +rm python.tar.xz; +cd /usr/src/python +gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" +./configure --build="$gnuArch" --enable-loadable-sqlite-extensions --enable-optimizations --enable-option-checking=fatal --enable-shared --with-lto --with-system-expat --without-ensurepip \ +nproc="$(nproc)" +EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)" +LDFLAGS="$(dpkg-buildflags --get LDFLAGS)" +LDFLAGS="${LDFLAGS:--Wl},--strip-all" +make -j "$nproc" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" "LDFLAGS=${LDFLAGS:-}" "PROFILE_TASK=${PROFILE_TASK:-}" + +rm python +make -j "$nproc" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" "LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" "PROFILE_TASK=${PROFILE_TASK:-}" python +make install +cd / +rm -rf /usr/src/python + +find /usr/local -depth \ + \( \ + \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \ + -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \) \) \ + \) -exec rm -rf '{}' +ldconfig +apt-mark auto '.*' > /dev/null +apt-mark manual $savedAptMark +find /usr/local -type f -executable -not \( -name '*tkinter*' \) -exec ldd '{}' ';' \ + | awk '/=>/ { so = $(NF-1); if (index(so, "/usr/local/") == 1) { next }; gsub("^/(usr/)?", "", so); printf "*%s\n", so }' \ + | sort -u \ + | xargs -r dpkg-query --search \ + | cut -d: -f1 \ + | sort -u \ + | xargs -r apt-mark manual +apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false +rm -rf /var/lib/apt/lists/* +python3 --version + +# make some useful symlinks that are expected to exist ("/usr/local/bin/python" and friends) + +for src in idle3 pydoc3 python3 python3-config; do \ + dst="$(echo "$src" | tr -d 3)"; \ + [ -s "/usr/local/bin/$src" ]; \ + [ ! -e "/usr/local/bin/$dst" ]; \ + ln -svT "$src" "/usr/local/bin/$dst"; \ +done + +# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" +export PYTHON_PIP_VERSION=24.0 +# https://github.com/docker-library/python/issues/365 +export PYTHON_SETUPTOOLS_VERSION=65.5.1 +# https://github.com/pypa/get-pip +export PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/dbf0c85f76fb6e1ab42aa672ffca6f0a675d9ee4/public/get-pip.py +export PYTHON_GET_PIP_SHA256=dfe9fd5c28dc98b5ac17979a953ea550cec37ae1b47a5116007395bfacff2ab9 + + +savedAptMark="$(apt-mark showmanual)" +apt-get update +apt-get install -y --no-install-recommends wget + +wget -O get-pip.py "$PYTHON_GET_PIP_URL" +echo "$PYTHON_GET_PIP_SHA256 *get-pip.py" | sha256sum -c - +apt-mark auto '.*' > /dev/null +[ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null +apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false +rm -rf /var/lib/apt/lists/* + +export PYTHONDONTWRITEBYTECODE=1 + +python get-pip.py --disable-pip-version-check --no-cache-dir --no-compile "pip==$PYTHON_PIP_VERSION" "setuptools==$PYTHON_SETUPTOOLS_VERSION" + +rm -f get-pip.py + +pip --version \ No newline at end of file diff --git a/installer/install_model.py b/installer/install_model.py new file mode 100644 index 000000000..fb464611a --- /dev/null +++ b/installer/install_model.py @@ -0,0 +1,69 @@ +# coding=utf-8 +""" + @project: maxkb + @Author：虎 + @file： install_model.py + @date：2023/12/18 14:02 + @desc: +""" +import json +import os.path +from pycrawlers import huggingface +from transformers import GPT2TokenizerFast +hg = huggingface() +prefix_dir = "./model" +model_config = [ + { + 'download_params': { + 'cache_dir': os.path.join(prefix_dir, 'base/hub'), + 'pretrained_model_name_or_path': 'gpt2' + }, + 'download_function': GPT2TokenizerFast.from_pretrained + }, + { + 'download_params': { + 'cache_dir': os.path.join(prefix_dir, 'base/hub'), + 'pretrained_model_name_or_path': 'gpt2-medium' + }, + 'download_function': GPT2TokenizerFast.from_pretrained + }, + { + 'download_params': { + 'cache_dir': os.path.join(prefix_dir, 'base/hub'), + 'pretrained_model_name_or_path': 'gpt2-large' + }, + 'download_function': GPT2TokenizerFast.from_pretrained + }, + { + 'download_params': { + 'cache_dir': os.path.join(prefix_dir, 'base/hub'), + 'pretrained_model_name_or_path': 'gpt2-xl' + }, + 'download_function': GPT2TokenizerFast.from_pretrained + }, + { + 'download_params': { + 'cache_dir': os.path.join(prefix_dir, 'base/hub'), + 'pretrained_model_name_or_path': 'distilgpt2' + }, + 'download_function': GPT2TokenizerFast.from_pretrained + }, + { + 'download_params': { + 'urls': ["https://huggingface.co/shibing624/text2vec-base-chinese/tree/main"], + 'file_save_paths': [os.path.join(prefix_dir, 'embedding',"shibing624_text2vec-base-chinese")] + }, + 'download_function': hg.get_batch_data + } + +] + + +def install(): + for model in model_config: + print(json.dumps(model.get('download_params'))) + model.get('download_function')(**model.get('download_params')) + + +if __name__ == '__main__': + install() diff --git a/installer/run-maxkb.sh b/installer/run-maxkb.sh new file mode 100644 index 000000000..4f388f695 --- /dev/null +++ b/installer/run-maxkb.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Start postgress +docker-entrypoint.sh postgres & + +# Wait postgress +until pg_isready --host=127.0.0.1; do sleep 1 && echo "waiting for postgres"; done + +# Start MaxKB +python /opt/maxkb/app/main.py start & + +# Wait for any process to exit +wait -n + +# Exit with status of process that exited first +exit $? \ No newline at end of file From 7a7541c8df0805850dddcb97c9cda073cad60868 Mon Sep 17 00:00:00 2001 From: liqiang-fit2cloud Date: Fri, 15 Mar 2024 13:31:31 +0800 Subject: [PATCH 5/8] =?UTF-8?q?build:=20=E5=88=86=E5=BC=80=E9=95=9C?= =?UTF-8?q?=E5=83=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../workflows/build-and-push-python-pg.yml | 55 ++++++ .../workflows/build-and-push-vector-model.yml | 55 ++++++ .github/workflows/build-and-push.yml | 2 +- Dockerfile => installer/Dockerfile | 32 +--- installer/Dockerfile-python-pg | 176 ++++++++++++++++++ installer/Dockerfile-vector-model | 6 + installer/install-python.sh | 101 ---------- 7 files changed, 295 insertions(+), 132 deletions(-) create mode 100644 .github/workflows/build-and-push-python-pg.yml create mode 100644 .github/workflows/build-and-push-vector-model.yml rename Dockerfile => installer/Dockerfile (57%) create mode 100644 installer/Dockerfile-python-pg create mode 100644 installer/Dockerfile-vector-model delete mode 100644 installer/install-python.sh diff --git a/.github/workflows/build-and-push-python-pg.yml b/.github/workflows/build-and-push-python-pg.yml new file mode 100644 index 000000000..d3fe1c31d --- /dev/null +++ b/.github/workflows/build-and-push-python-pg.yml @@ -0,0 +1,55 @@ +name: build-and-push-python-pg + +on: + workflow_dispatch: + +jobs: + build-and-push-python-pg-to-ghcr: + runs-on: ubuntu-latest + steps: + - name: Check Disk Space + run: df -h + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + - name: Check Disk Space + run: df -h + - name: Checkout + uses: actions/checkout@v4 + with: + ref: main + - name: Prepare + id: prepare + run: | + DOCKER_IMAGE=ghcr.io/maxkb-dev/python-pg + DOCKER_PLATFORMS=linux/amd64,linux/arm64 + TAG_NAME=python3.11.8-pg15.6 + DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:latest" + echo ::set-output name=docker_image::${DOCKER_IMAGE} + echo ::set-output name=version::${TAG_NAME} + echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \ + --build-arg VERSION=${TAG_NAME} \ + --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ + --build-arg --no-cache \ + --build-arg VCS_REF=${GITHUB_SHA::8} \ + ${DOCKER_IMAGE_TAGS} . + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GH_TOKEN }} + - name: Docker Buildx (build-and-push) + run: | + docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} -f installer/Dockerfile-python-pg \ No newline at end of file diff --git a/.github/workflows/build-and-push-vector-model.yml b/.github/workflows/build-and-push-vector-model.yml new file mode 100644 index 000000000..f83e5192f --- /dev/null +++ b/.github/workflows/build-and-push-vector-model.yml @@ -0,0 +1,55 @@ +name: build-and-push-vector-model + +on: + workflow_dispatch: + +jobs: + build-and-push-vector-model-to-ghcr: + runs-on: ubuntu-latest + steps: + - name: Check Disk Space + run: df -h + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + - name: Check Disk Space + run: df -h + - name: Checkout + uses: actions/checkout@v4 + with: + ref: main + - name: Prepare + id: prepare + run: | + DOCKER_IMAGE=ghcr.io/maxkb-dev/vector-model + DOCKER_PLATFORMS=linux/amd64,linux/arm64 + TAG_NAME=v1.0.0 + DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:latest" + echo ::set-output name=docker_image::${DOCKER_IMAGE} + echo ::set-output name=version::${TAG_NAME} + echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \ + --build-arg VERSION=${TAG_NAME} \ + --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ + --build-arg --no-cache \ + --build-arg VCS_REF=${GITHUB_SHA::8} \ + ${DOCKER_IMAGE_TAGS} . + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GH_TOKEN }} + - name: Docker Buildx (build-and-push) + run: | + docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} -f installer/Dockerfile-vector-model \ No newline at end of file diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index cd3cd274a..011c02aa1 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -107,4 +107,4 @@ jobs: password: ${{ secrets.GH_TOKEN }} - name: Docker Buildx (build-and-push) run: | - docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} \ No newline at end of file + docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} -f installer/Dockerfile \ No newline at end of file diff --git a/Dockerfile b/installer/Dockerfile similarity index 57% rename from Dockerfile rename to installer/Dockerfile index 746d033e0..7a26a0376 100644 --- a/Dockerfile +++ b/installer/Dockerfile @@ -1,9 +1,4 @@ -FROM python:3.11-slim as vector-model-build -COPY installer/install_model.py install_model.py -RUN pip3 install --upgrade pip setuptools && \ - pip install pycrawlers && \ - pip install transformers && \ - python3 install_model.py +FROM ghcr.io/maxkb-dev/python-pg:v1.0.0 as vector-model FROM node:18-alpine3.18 as web-build COPY ui ui @@ -12,30 +7,7 @@ RUN cd ui && \ npm run build && \ rm -rf ./node_modules -FROM postgres:15.6-bookworm - -ENV LANG=C.UTF-8 - -RUN apt-get update - -RUN apt-get install postgresql-15-pgvector - -COPY installer/init.sql /docker-entrypoint-initdb.d - -RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo "Asia/Shanghai" > /etc/timezone - -ENV POSTGRES_USER root - -ENV POSTGRES_PASSWORD Password123@postgres - - -# ---- prepare python env --- # -ENV PATH /usr/local/bin:$PATH -COPY installer/install-python.sh /install-python.sh -RUN chmod 755 /install-python.sh; bash -c "/install-python.sh > /dev/null 2>&1" ; rm -f /install-python.sh -ENV PYTHON_VERSION 3.11.8 - -# ---- build maxkb --- # +FROM ghcr.io/maxkb-dev/python-pg:python3.11.8-pg15.6 # 创建工作目录 RUN mkdir -p /opt/maxkb/app && mkdir -p /opt/maxkb/model && mkdir -p /opt/maxkb/conf VOLUME /opt/maxkb diff --git a/installer/Dockerfile-python-pg b/installer/Dockerfile-python-pg new file mode 100644 index 000000000..0032f9da0 --- /dev/null +++ b/installer/Dockerfile-python-pg @@ -0,0 +1,176 @@ +FROM postgres:15.6-bookworm + +ENV LANG=C.UTF-8 + +RUN apt-get update + +RUN apt-get install postgresql-15-pgvector + +COPY installer/init.sql /docker-entrypoint-initdb.d + +RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo "Asia/Shanghai" > /etc/timezone + +ENV POSTGRES_USER root + +ENV POSTGRES_PASSWORD Password123@postgres + +ENV PATH /usr/local/bin:$PATH + +# http://bugs.python.org/issue19846 +# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. +ENV LANG C.UTF-8 + +# runtime dependencies +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + netbase \ + tzdata \ + ; \ + rm -rf /var/lib/apt/lists/* + +ENV GPG_KEY A035C8C19219BA821ECEA86B64E628F8D684696D +ENV PYTHON_VERSION 3.11.8 + +RUN set -eux; \ + \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + dpkg-dev \ + gcc \ + gnupg \ + libbluetooth-dev \ + libbz2-dev \ + libc6-dev \ + libdb-dev \ + libexpat1-dev \ + libffi-dev \ + libgdbm-dev \ + liblzma-dev \ + libncursesw5-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + make \ + tk-dev \ + uuid-dev \ + wget \ + xz-utils \ + zlib1g-dev \ + ; \ + \ + wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz"; \ + wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc"; \ + GNUPGHOME="$(mktemp -d)"; export GNUPGHOME; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$GPG_KEY"; \ + gpg --batch --verify python.tar.xz.asc python.tar.xz; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" python.tar.xz.asc; \ + mkdir -p /usr/src/python; \ + tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz; \ + rm python.tar.xz; \ + \ + cd /usr/src/python; \ + gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \ + ./configure \ + --build="$gnuArch" \ + --enable-loadable-sqlite-extensions \ + --enable-optimizations \ + --enable-option-checking=fatal \ + --enable-shared \ + --with-lto \ + --with-system-expat \ + --without-ensurepip \ + ; \ + nproc="$(nproc)"; \ + EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)"; \ + LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"; \ + LDFLAGS="${LDFLAGS:--Wl},--strip-all"; \ + make -j "$nproc" \ + "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ + "LDFLAGS=${LDFLAGS:-}" \ + "PROFILE_TASK=${PROFILE_TASK:-}" \ + ; \ +# https://github.com/docker-library/python/issues/784 +# prevent accidental usage of a system installed libpython of the same version + rm python; \ + make -j "$nproc" \ + "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ + "LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" \ + "PROFILE_TASK=${PROFILE_TASK:-}" \ + python \ + ; \ + make install; \ + \ + cd /; \ + rm -rf /usr/src/python; \ + \ + find /usr/local -depth \ + \( \ + \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \ + -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \) \) \ + \) -exec rm -rf '{}' + \ + ; \ + \ + ldconfig; \ + \ + apt-mark auto '.*' > /dev/null; \ + apt-mark manual $savedAptMark; \ + find /usr/local -type f -executable -not \( -name '*tkinter*' \) -exec ldd '{}' ';' \ + | awk '/=>/ { so = $(NF-1); if (index(so, "/usr/local/") == 1) { next }; gsub("^/(usr/)?", "", so); printf "*%s\n", so }' \ + | sort -u \ + | xargs -r dpkg-query --search \ + | cut -d: -f1 \ + | sort -u \ + | xargs -r apt-mark manual \ + ; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + rm -rf /var/lib/apt/lists/*; \ + \ + python3 --version + +# make some useful symlinks that are expected to exist ("/usr/local/bin/python" and friends) +RUN set -eux; \ + for src in idle3 pydoc3 python3 python3-config; do \ + dst="$(echo "$src" | tr -d 3)"; \ + [ -s "/usr/local/bin/$src" ]; \ + [ ! -e "/usr/local/bin/$dst" ]; \ + ln -svT "$src" "/usr/local/bin/$dst"; \ + done + +# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" +ENV PYTHON_PIP_VERSION 24.0 +# https://github.com/docker-library/python/issues/365 +ENV PYTHON_SETUPTOOLS_VERSION 65.5.1 +# https://github.com/pypa/get-pip +ENV PYTHON_GET_PIP_URL https://github.com/pypa/get-pip/raw/dbf0c85f76fb6e1ab42aa672ffca6f0a675d9ee4/public/get-pip.py +ENV PYTHON_GET_PIP_SHA256 dfe9fd5c28dc98b5ac17979a953ea550cec37ae1b47a5116007395bfacff2ab9 + +RUN set -eux; \ + \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + apt-get install -y --no-install-recommends wget; \ + \ + wget -O get-pip.py "$PYTHON_GET_PIP_URL"; \ + echo "$PYTHON_GET_PIP_SHA256 *get-pip.py" | sha256sum -c -; \ + \ + apt-mark auto '.*' > /dev/null; \ + [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + rm -rf /var/lib/apt/lists/*; \ + \ + export PYTHONDONTWRITEBYTECODE=1; \ + \ + python get-pip.py \ + --disable-pip-version-check \ + --no-cache-dir \ + --no-compile \ + "pip==$PYTHON_PIP_VERSION" \ + "setuptools==$PYTHON_SETUPTOOLS_VERSION" \ + ; \ + rm -f get-pip.py; \ + \ + pip --version diff --git a/installer/Dockerfile-vector-model b/installer/Dockerfile-vector-model new file mode 100644 index 000000000..97e35b4e4 --- /dev/null +++ b/installer/Dockerfile-vector-model @@ -0,0 +1,6 @@ +FROM python:3.11-slim +COPY installer/install_model.py install_model.py +RUN pip3 install --upgrade pip setuptools && \ + pip install pycrawlers && \ + pip install transformers && \ + python3 install_model.py \ No newline at end of file diff --git a/installer/install-python.sh b/installer/install-python.sh deleted file mode 100644 index 0835f28cc..000000000 --- a/installer/install-python.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash - -# ---- prepare python env --- # -# ---- copy from https://github.com/docker-library/python/blob/master/3.11/slim-bookworm/Dockerfile --- # - -export PATH=/usr/local/bin:$PATH - -export LANG=C.UTF-8 - -apt-get update; -apt-get install -y --no-install-recommends ca-certificates netbase tzdata -rm -rf /var/lib/apt/lists/* - -export GPG_KEY=A035C8C19219BA821ECEA86B64E628F8D684696D -export PYTHON_VERSION=3.11.8 - -savedAptMark="$(apt-mark showmanual)" -apt-get update -apt-get install -y --no-install-recommends dpkg-dev gcc gnupg libbluetooth-dev libbz2-dev libc6-dev libdb-dev libexpat1-dev libffi-dev libgdbm-dev liblzma-dev libncursesw5-dev libreadline-dev libsqlite3-dev libssl-dev make \ - tk-dev uuid-dev wget xz-utils zlib1g-dev -wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" -wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" -GNUPGHOME="$(mktemp -d)" -export GNUPGHOME -gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$GPG_KEY" -gpg --batch --verify python.tar.xz.asc python.tar.xz -gpgconf --kill all -rm -rf "$GNUPGHOME" python.tar.xz.asc -mkdir -p /usr/src/python -tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz -rm python.tar.xz; -cd /usr/src/python -gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" -./configure --build="$gnuArch" --enable-loadable-sqlite-extensions --enable-optimizations --enable-option-checking=fatal --enable-shared --with-lto --with-system-expat --without-ensurepip \ -nproc="$(nproc)" -EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)" -LDFLAGS="$(dpkg-buildflags --get LDFLAGS)" -LDFLAGS="${LDFLAGS:--Wl},--strip-all" -make -j "$nproc" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" "LDFLAGS=${LDFLAGS:-}" "PROFILE_TASK=${PROFILE_TASK:-}" - -rm python -make -j "$nproc" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" "LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" "PROFILE_TASK=${PROFILE_TASK:-}" python -make install -cd / -rm -rf /usr/src/python - -find /usr/local -depth \ - \( \ - \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \ - -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \) \) \ - \) -exec rm -rf '{}' -ldconfig -apt-mark auto '.*' > /dev/null -apt-mark manual $savedAptMark -find /usr/local -type f -executable -not \( -name '*tkinter*' \) -exec ldd '{}' ';' \ - | awk '/=>/ { so = $(NF-1); if (index(so, "/usr/local/") == 1) { next }; gsub("^/(usr/)?", "", so); printf "*%s\n", so }' \ - | sort -u \ - | xargs -r dpkg-query --search \ - | cut -d: -f1 \ - | sort -u \ - | xargs -r apt-mark manual -apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false -rm -rf /var/lib/apt/lists/* -python3 --version - -# make some useful symlinks that are expected to exist ("/usr/local/bin/python" and friends) - -for src in idle3 pydoc3 python3 python3-config; do \ - dst="$(echo "$src" | tr -d 3)"; \ - [ -s "/usr/local/bin/$src" ]; \ - [ ! -e "/usr/local/bin/$dst" ]; \ - ln -svT "$src" "/usr/local/bin/$dst"; \ -done - -# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -export PYTHON_PIP_VERSION=24.0 -# https://github.com/docker-library/python/issues/365 -export PYTHON_SETUPTOOLS_VERSION=65.5.1 -# https://github.com/pypa/get-pip -export PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/dbf0c85f76fb6e1ab42aa672ffca6f0a675d9ee4/public/get-pip.py -export PYTHON_GET_PIP_SHA256=dfe9fd5c28dc98b5ac17979a953ea550cec37ae1b47a5116007395bfacff2ab9 - - -savedAptMark="$(apt-mark showmanual)" -apt-get update -apt-get install -y --no-install-recommends wget - -wget -O get-pip.py "$PYTHON_GET_PIP_URL" -echo "$PYTHON_GET_PIP_SHA256 *get-pip.py" | sha256sum -c - -apt-mark auto '.*' > /dev/null -[ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null -apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false -rm -rf /var/lib/apt/lists/* - -export PYTHONDONTWRITEBYTECODE=1 - -python get-pip.py --disable-pip-version-check --no-cache-dir --no-compile "pip==$PYTHON_PIP_VERSION" "setuptools==$PYTHON_SETUPTOOLS_VERSION" - -rm -f get-pip.py - -pip --version \ No newline at end of file From 0fa47f64b45ed9f598ed3d775be27037c998d30e Mon Sep 17 00:00:00 2001 From: liqiang-fit2cloud Date: Fri, 15 Mar 2024 13:35:41 +0800 Subject: [PATCH 6/8] =?UTF-8?q?build:=20=E5=88=86=E5=BC=80=E9=95=9C?= =?UTF-8?q?=E5=83=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build-and-push-python-pg.yml | 14 -------------- .github/workflows/build-and-push-vector-model.yml | 14 -------------- 2 files changed, 28 deletions(-) diff --git a/.github/workflows/build-and-push-python-pg.yml b/.github/workflows/build-and-push-python-pg.yml index d3fe1c31d..478fdbab4 100644 --- a/.github/workflows/build-and-push-python-pg.yml +++ b/.github/workflows/build-and-push-python-pg.yml @@ -7,20 +7,6 @@ jobs: build-and-push-python-pg-to-ghcr: runs-on: ubuntu-latest steps: - - name: Check Disk Space - run: df -h - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - tool-cache: true - android: true - dotnet: true - haskell: true - large-packages: true - docker-images: true - swap-storage: true - - name: Check Disk Space - run: df -h - name: Checkout uses: actions/checkout@v4 with: diff --git a/.github/workflows/build-and-push-vector-model.yml b/.github/workflows/build-and-push-vector-model.yml index f83e5192f..6412f7fd7 100644 --- a/.github/workflows/build-and-push-vector-model.yml +++ b/.github/workflows/build-and-push-vector-model.yml @@ -7,20 +7,6 @@ jobs: build-and-push-vector-model-to-ghcr: runs-on: ubuntu-latest steps: - - name: Check Disk Space - run: df -h - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - tool-cache: true - android: true - dotnet: true - haskell: true - large-packages: true - docker-images: true - swap-storage: true - - name: Check Disk Space - run: df -h - name: Checkout uses: actions/checkout@v4 with: From 226f4a426bfa5ec1f6fe3240512f4724a0170586 Mon Sep 17 00:00:00 2001 From: liqiang-fit2cloud Date: Fri, 15 Mar 2024 13:38:10 +0800 Subject: [PATCH 7/8] =?UTF-8?q?build:=20=E5=88=86=E5=BC=80=E9=95=9C?= =?UTF-8?q?=E5=83=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- installer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/Dockerfile b/installer/Dockerfile index 7a26a0376..c2bcd1225 100644 --- a/installer/Dockerfile +++ b/installer/Dockerfile @@ -15,7 +15,7 @@ VOLUME /opt/maxkb COPY . /opt/maxkb/app COPY installer/config.yaml /opt/maxkb/conf RUN rm -rf /opt/maxkb/app/ui /opt/maxkb/app/build -COPY --from=vector-model-build model /opt/maxkb/app/model +COPY --from=vector-model model /opt/maxkb/app/model COPY --from=web-build ui /opt/maxkb/app/ui RUN ls -la /opt/maxkb/app # 复制模型 From fed53c455e4fc05fa39a308ca7ec83aebb10d590 Mon Sep 17 00:00:00 2001 From: liqiang-fit2cloud Date: Fri, 15 Mar 2024 13:46:31 +0800 Subject: [PATCH 8/8] =?UTF-8?q?build:=20=E5=88=86=E5=BC=80=E9=95=9C?= =?UTF-8?q?=E5=83=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build-and-push-vector-model.yml | 14 ++++++++++++++ .github/workflows/build-and-push.yml | 2 +- installer/Dockerfile | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-push-vector-model.yml b/.github/workflows/build-and-push-vector-model.yml index 6412f7fd7..f83e5192f 100644 --- a/.github/workflows/build-and-push-vector-model.yml +++ b/.github/workflows/build-and-push-vector-model.yml @@ -7,6 +7,20 @@ jobs: build-and-push-vector-model-to-ghcr: runs-on: ubuntu-latest steps: + - name: Check Disk Space + run: df -h + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + - name: Check Disk Space + run: df -h - name: Checkout uses: actions/checkout@v4 with: diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 011c02aa1..2afe5e026 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -57,7 +57,7 @@ jobs: password: ${{ secrets.FIT2CLOUD_REGISTRY_PASSWORD }} - name: Docker Buildx (build-and-push) run: | - docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} + docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} -f installer/Dockerfile build-and-push-to-ghcr: runs-on: ubuntu-latest diff --git a/installer/Dockerfile b/installer/Dockerfile index c2bcd1225..1d50b06e5 100644 --- a/installer/Dockerfile +++ b/installer/Dockerfile @@ -34,8 +34,8 @@ RUN pip3 install poetry RUN poetry export -f requirements.txt --output requirements.txt --without-hashes # 下载python依赖 RUN pip3 install --no-cache-dir -r requirements.txt -EXPOSE 8000 # 启动命令 +EXPOSE 8000 COPY installer/run-maxkb.sh /usr/bin/ RUN chmod 755 /usr/bin/run-maxkb.sh ENTRYPOINT ["bash", "-c"]