From ffe8900fb26d52fd14fe5bfbd51e1744377fc94f Mon Sep 17 00:00:00 2001 From: liqiang-fit2cloud Date: Fri, 18 Apr 2025 14:10:16 +0800 Subject: [PATCH] build: init directory. --- .github/workflows/build-and-push.yml | 2 +- apps/common/config/tokenizer_manage_config.py | 2 +- installer/Dockerfile | 77 ++++++++ installer/Dockerfile-python-pg | 10 +- installer/init.sql | 5 + installer/start-all.sh | 16 ++ installer/start-maxkb.sh | 7 + installer/start-postgres.sh | 7 + installer/start-redis.sh | 13 ++ installer/wait-for-it.sh | 182 ++++++++++++++++++ pyproject.toml | 5 +- 11 files changed, 321 insertions(+), 5 deletions(-) create mode 100644 installer/Dockerfile create mode 100644 installer/init.sql create mode 100644 installer/start-all.sh create mode 100644 installer/start-maxkb.sh create mode 100644 installer/start-postgres.sh create mode 100644 installer/start-redis.sh create mode 100644 installer/wait-for-it.sh diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 26d2b86d2..54ba87660 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -7,7 +7,7 @@ on: inputs: dockerImageTag: description: 'Image Tag' - default: 'v1.10.3-dev' + default: 'v2.0.0-dev' required: true dockerImageTagWithLatest: description: '是否发布latest tag（正式发版时选择，测试版本切勿选择）' diff --git a/apps/common/config/tokenizer_manage_config.py b/apps/common/config/tokenizer_manage_config.py index b590d3a00..47a6d61e9 100644 --- a/apps/common/config/tokenizer_manage_config.py +++ b/apps/common/config/tokenizer_manage_config.py @@ -17,7 +17,7 @@ class TokenizerManage: if TokenizerManage.tokenizer is None: TokenizerManage.tokenizer = BertTokenizer.from_pretrained( 'bert-base-cased', - cache_dir="/opt/maxkb/model/tokenizer", + cache_dir="/opt/maxkb-app/model/tokenizer", local_files_only=True, resume_download=False, force_download=False) diff --git a/installer/Dockerfile b/installer/Dockerfile new file mode 100644 index 000000000..594612943 --- /dev/null +++ b/installer/Dockerfile @@ -0,0 +1,77 @@ +FROM ghcr.io/1panel-dev/maxkb-vector-model:v1.0.1 AS vector-model +FROM node:18-alpine3.18 AS web-build +COPY ui ui +RUN cd ui && \ + npm install && \ + npm run build && \ + rm -rf ./node_modules +FROM ghcr.io/1panel-dev/maxkb-python-pg:python3.11-pg17.4 AS stage-build + +ARG DEPENDENCIES=" \ + python3-pip" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends $DEPENDENCIES && \ + apt-get clean all && \ + rm -rf /var/lib/apt/lists/* + +COPY . /opt/maxkb-app +RUN mkdir -p /opt/maxkb-app /opt/maxkb-app/model && \ + rm -rf /opt/maxkb/app/ui + +COPY --from=web-build ui /opt/maxkb-app/ui +WORKDIR /opt/maxkb-app +RUN python3 -m venv /opt/py3 && \ + pip install poetry==1.8.5 --break-system-packages && \ + poetry config virtualenvs.create false && \ + . /opt/py3/bin/activate && \ + if [ "$(uname -m)" = "x86_64" ]; then sed -i 's/^torch.*/torch = {version = "^2.6.0+cpu", source = "pytorch"}/g' pyproject.toml; fi && \ + poetry install && \ + #export MAXKB_CONFIG_TYPE=ENV && python3 /opt/maxkb-app/apps/manage.py compilemessages \ + echo test + +FROM ghcr.io/1panel-dev/maxkb-python-pg:python3.11-pg17.4 +ARG DOCKER_IMAGE_TAG=dev \ + BUILD_AT \ + GITHUB_COMMIT + +ENV MAXKB_VERSION="${DOCKER_IMAGE_TAG} (build at ${BUILD_AT}, commit: ${GITHUB_COMMIT})" \ + MAXKB_CONFIG_TYPE=ENV \ + MAXKB_DB_NAME=maxkb \ + MAXKB_DB_HOST=127.0.0.1 \ + MAXKB_DB_PORT=5432 \ + MAXKB_DB_USER=${POSTGRES_USER} \ + MAXKB_DB_PASSWORD=${POSTGRES_PASSWORD} \ + MAXKB_DB_MAX_OVERFLOW=80 \ + MAXKB_EMBEDDING_MODEL_NAME=/opt/maxkb-app/model/embedding/shibing624_text2vec-base-chinese \ + MAXKB_EMBEDDING_MODEL_PATH=/opt/maxkb-app/model/embedding \ + MAXKB_SANDBOX=1 \ + LANG=en_US.UTF-8 \ + PATH=/opt/py3/bin:$PATH \ + PIP_TARGET=/opt/maxkb/python-packages \ + PYTHONPATH=/opt/maxkb/python-packages \ + PYTHONUNBUFFERED=1 + +WORKDIR /opt/maxkb/app +COPY --from=stage-build /opt/maxkb-app /opt/maxkb=app +COPY --from=stage-build /opt/py3 /opt/py3 +COPY --from=vector-model /opt/maxkb/app/model /opt/maxkb-app/model + +RUN chmod 755 /opt/maxkb-app/installer/*.sh && \ + cp -f /opt/maxkb-app/installer/*.sh /usr/bin/ && \ + cp -f /opt/maxkb-app/installer/init.sql /docker-entrypoint-initdb.d && \ + rm -rf /opt/maxkb-app/installer + cp -r /opt/maxkb-app/model/base/hub /opt/maxkb-app/model/tokenizer && \ + curl -L --connect-timeout 120 -m 1800 https://resource.fit2cloud.com/maxkb/ffmpeg/get-ffmpeg-linux | sh && \ + mkdir -p /opt/maxkb-app/sandbox/ && \ + find /opt/maxkb-app -mindepth 1 -not -name 'sandbox' -exec chmod 700 {} + && \ + chmod 755 /tmp && \ + useradd --no-create-home --home /opt/maxkb-app/sandbox sandbox -g root && \ + chown -R sandbox:root /opt/maxkb-app/sandbox && \ + chmod g-x /usr/local/bin/* /usr/bin/* /bin/* /usr/sbin/* /sbin/* /usr/lib/postgresql/15/bin/* && \ + chmod g+x /usr/local/bin/python* + +EXPOSE 8080 +VOLUME /opt/maxkb +ENTRYPOINT ["bash", "-c"] +CMD [ "/usr/bin/start-all.sh" ] \ No newline at end of file diff --git a/installer/Dockerfile-python-pg b/installer/Dockerfile-python-pg index b7bc2d588..a26e5148d 100644 --- a/installer/Dockerfile-python-pg +++ b/installer/Dockerfile-python-pg @@ -8,6 +8,7 @@ ARG DEPENDENCIES=" \ ca-certificates \ vim \ gettext \ + redis-server \ postgresql-17-pgvector \ postgresql-17-age" @@ -17,4 +18,11 @@ RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ apt-get clean all && \ rm -rf /var/lib/apt/lists/* -COPY --from=python-stage /usr/local /usr/local \ No newline at end of file +COPY --from=python-stage /usr/local /usr/local + +ENV PGDATA=/opt/maxkb/data/postgresql/pgdata \ + POSTGRES_USER=root \ + POSTGRES_PASSWORD=Password123@postgres \ + POSTGRES_MAX_CONNECTIONS=1000 \ + REDIS_PASSWORD=Password123@redis \ + LANG=en_US.UTF-8 diff --git a/installer/init.sql b/installer/init.sql new file mode 100644 index 000000000..dfc30f9d7 --- /dev/null +++ b/installer/init.sql @@ -0,0 +1,5 @@ +CREATE DATABASE "maxkb"; + +\c "maxkb"; + +CREATE EXTENSION "vector"; \ No newline at end of file diff --git a/installer/start-all.sh b/installer/start-all.sh new file mode 100644 index 000000000..4f8b494ad --- /dev/null +++ b/installer/start-all.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +if [ "$MAXKB_DB_HOST" = "127.0.0.1" ]; then + echo "PostgreSQL starting..." + /usr/bin/start-postgres.sh + echo "PostgreSQL started." +fi + + +if [ "$MAXKB_REDIS_HOST" = "127.0.0.1" ]; then + echo "Redis starting..." + /usr/bin/start-redis.sh + echo "Redis started." +fi + +/usr/bin/start-maxkb.sh \ No newline at end of file diff --git a/installer/start-maxkb.sh b/installer/start-maxkb.sh new file mode 100644 index 000000000..07c06d4ef --- /dev/null +++ b/installer/start-maxkb.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +mkdir -p /opt/maxkb/logs +mkdir -p /opt/maxkb/local +mkdir -p /opt/maxkb/python-packages + +python /opt/maxkb-app/main.py start \ No newline at end of file diff --git a/installer/start-postgres.sh b/installer/start-postgres.sh new file mode 100644 index 000000000..9d27071f1 --- /dev/null +++ b/installer/start-postgres.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +mkdir -p /opt/maxkb/data/postgresql + +docker-entrypoint.sh postgres -c max_connections=${POSTGRES_MAX_CONNECTIONS} & +sleep 10 +/usr/bin/wait-for-it.sh 127.0.0.1:5432 --timeout=120 --strict \ No newline at end of file diff --git a/installer/start-redis.sh b/installer/start-redis.sh new file mode 100644 index 000000000..040040880 --- /dev/null +++ b/installer/start-redis.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +mkdir -p /opt/maxkb/data/redis + +if [ ! -f /opt/maxkb/conf/redis.conf ]; then + mkdir -p /opt/maxkb/conf + touch /opt/maxkb/conf/redis.conf + printf "bind 0.0.0.0\nport 6379\nmaxmemory 1G\nmaxmemory-policy allkeys-lru\ndir /opt/maxkb/data/redis\nrequirepass "${REDIS_PASSWORD}"\n" > /opt/maxkb/conf/redis.conf +fi + +redis-server /opt/maxkb/conf/redis.conf & +sleep 5 +/usr/bin/wait-for-it.sh 127.0.0.1:6379 --timeout=60 --strict \ No newline at end of file diff --git a/installer/wait-for-it.sh b/installer/wait-for-it.sh new file mode 100644 index 000000000..3974640b0 --- /dev/null +++ b/installer/wait-for-it.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# Check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) + +WAITFORIT_BUSYTIMEFLAG="" +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + # Check if busybox timeout uses -t flag + # (recent Alpine versions don't support -t anymore) + if timeout &>/dev/stdout | grep -q -e '-t '; then + WAITFORIT_BUSYTIMEFLAG="-t" + fi +else + WAITFORIT_ISBUSY=0 +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index b8f536d08..b28236877 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,10 @@ [tool.poetry] name = "maxkb" -version = "2.1.0" -description = "智能知识库问答系统" +version = "2.0.0" +description = "企业级 AI 助手" authors = ["shaohuzhang1 "] readme = "README.md" +package-mode = false [tool.poetry.dependencies] python = "^3.11"