mirror of
https://github.com/1Panel-dev/MaxKB.git
synced 2025-12-26 18:22:46 +00:00
Compare commits
No commits in common. "main" and "v1.10.1-lts" have entirely different histories.
main
...
v1.10.1-lt
|
|
@ -1,17 +0,0 @@
|
|||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "pip"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
timezone: "Asia/Shanghai"
|
||||
day: "friday"
|
||||
target-branch: "v2"
|
||||
groups:
|
||||
python-dependencies:
|
||||
patterns:
|
||||
- "*"
|
||||
# ignore:
|
||||
# - dependency-name: "pymupdf"
|
||||
# versions: ["*"]
|
||||
|
||||
|
|
@ -7,7 +7,7 @@ on:
|
|||
inputs:
|
||||
dockerImageTag:
|
||||
description: 'Image Tag'
|
||||
default: 'v1.10.7-dev'
|
||||
default: 'v1.10.0-dev'
|
||||
required: true
|
||||
dockerImageTagWithLatest:
|
||||
description: '是否发布latest tag（正式发版时选择，测试版本切勿选择）'
|
||||
|
|
@ -36,7 +36,7 @@ on:
|
|||
jobs:
|
||||
build-and-push-to-fit2cloud-registry:
|
||||
if: ${{ contains(github.event.inputs.registry, 'fit2cloud') }}
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- name: Check Disk Space
|
||||
run: df -h
|
||||
|
|
@ -64,15 +64,18 @@ jobs:
|
|||
TAG_NAME=${{ github.event.inputs.dockerImageTag }}
|
||||
TAG_NAME_WITH_LATEST=${{ github.event.inputs.dockerImageTagWithLatest }}
|
||||
if [[ ${TAG_NAME_WITH_LATEST} == 'true' ]]; then
|
||||
DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:${TAG_NAME%%.*}"
|
||||
DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:latest"
|
||||
else
|
||||
DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME}"
|
||||
fi
|
||||
echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} --memory-swap -1 \
|
||||
--build-arg DOCKER_IMAGE_TAG=${{ github.event.inputs.dockerImageTag }} --build-arg BUILD_AT=$(TZ=Asia/Shanghai date +'%Y-%m-%dT%H:%M') --build-arg GITHUB_COMMIT=`git rev-parse --short HEAD` --no-cache \
|
||||
echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \
|
||||
--build-arg DOCKER_IMAGE_TAG=${{ github.event.inputs.dockerImageTag }} --build-arg BUILD_AT=$(TZ=Asia/Shanghai date +'%Y-%m-%dT%H:%M') --build-arg GITHUB_COMMIT=${GITHUB_SHA::8} --no-cache \
|
||||
${DOCKER_IMAGE_TAGS} .
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
with:
|
||||
# Until https://github.com/tonistiigi/binfmt/issues/215
|
||||
image: tonistiigi/binfmt:qemu-v7.0.0-28
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
- name: Login to GitHub Container Registry
|
||||
|
|
@ -89,12 +92,11 @@ jobs:
|
|||
password: ${{ secrets.FIT2CLOUD_REGISTRY_PASSWORD }}
|
||||
- name: Docker Buildx (build-and-push)
|
||||
run: |
|
||||
sudo sync && echo 3 | sudo tee /proc/sys/vm/drop_caches && free -m
|
||||
docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} -f installer/Dockerfile
|
||||
|
||||
build-and-push-to-dockerhub:
|
||||
if: ${{ contains(github.event.inputs.registry, 'dockerhub') }}
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- name: Check Disk Space
|
||||
run: df -h
|
||||
|
|
@ -122,15 +124,18 @@ jobs:
|
|||
TAG_NAME=${{ github.event.inputs.dockerImageTag }}
|
||||
TAG_NAME_WITH_LATEST=${{ github.event.inputs.dockerImageTagWithLatest }}
|
||||
if [[ ${TAG_NAME_WITH_LATEST} == 'true' ]]; then
|
||||
DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:${TAG_NAME%%.*}"
|
||||
DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:latest"
|
||||
else
|
||||
DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME}"
|
||||
fi
|
||||
echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} --memory-swap -1 \
|
||||
--build-arg DOCKER_IMAGE_TAG=${{ github.event.inputs.dockerImageTag }} --build-arg BUILD_AT=$(TZ=Asia/Shanghai date +'%Y-%m-%dT%H:%M') --build-arg GITHUB_COMMIT=`git rev-parse --short HEAD` --no-cache \
|
||||
echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \
|
||||
--build-arg DOCKER_IMAGE_TAG=${{ github.event.inputs.dockerImageTag }} --build-arg BUILD_AT=$(TZ=Asia/Shanghai date +'%Y-%m-%dT%H:%M') --build-arg GITHUB_COMMIT=${GITHUB_SHA::8} --no-cache \
|
||||
${DOCKER_IMAGE_TAGS} .
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
with:
|
||||
# Until https://github.com/tonistiigi/binfmt/issues/215
|
||||
image: tonistiigi/binfmt:qemu-v7.0.0-28
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
- name: Login to GitHub Container Registry
|
||||
|
|
@ -146,5 +151,4 @@ jobs:
|
|||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- name: Docker Buildx (build-and-push)
|
||||
run: |
|
||||
sudo sync && echo 3 | sudo tee /proc/sys/vm/drop_caches && free -m
|
||||
docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} -f installer/Dockerfile
|
||||
|
|
|
|||
|
|
@ -183,5 +183,4 @@ apps/xpack
|
|||
data
|
||||
.dev
|
||||
poetry.lock
|
||||
apps/setting/models_provider/impl/*/icon/
|
||||
tmp/
|
||||
apps/setting/models_provider/impl/*/icon/
|
||||
72
README.md
72
README.md
|
|
@ -1,6 +1,6 @@
|
|||
<p align="center"><img src= "https://github.com/1Panel-dev/maxkb/assets/52996290/c0694996-0eed-40d8-b369-322bf2a380bf" alt="MaxKB" width="300" /></p>
|
||||
<h3 align="center">Open-source platform for building enterprise-grade agents</h3>
|
||||
<h3 align="center">强大易用的企业级智能体平台</h3>
|
||||
<h3 align="center">Ready-to-use, flexible RAG Chatbot</h3>
|
||||
<h3 align="center">基于大模型和 RAG 的开源知识库问答系统</h3>
|
||||
<p align="center"><a href="https://trendshift.io/repositories/9113" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9113" alt="1Panel-dev%2FMaxKB | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a></p>
|
||||
<p align="center">
|
||||
<a href="https://www.gnu.org/licenses/gpl-3.0.html#license-text"><img src="https://img.shields.io/github/license/1Panel-dev/maxkb?color=%231890FF" alt="License: GPL v3"></a>
|
||||
|
|
@ -11,13 +11,12 @@
|
|||
</p>
|
||||
<hr/>
|
||||
|
||||
MaxKB = Max Knowledge Brain, it is an open-source platform for building enterprise-grade agents. MaxKB integrates Retrieval-Augmented Generation (RAG) pipelines, supports robust workflows, and provides advanced MCP tool-use capabilities. MaxKB is widely applied in scenarios such as intelligent customer service, corporate internal knowledge bases, academic research, and education.
|
||||
MaxKB = Max Knowledge Base, it is a chatbot based on Large Language Models (LLM) and Retrieval-Augmented Generation (RAG). MaxKB is widely applied in scenarios such as intelligent customer service, corporate internal knowledge bases, academic research, and education.
|
||||
|
||||
- **RAG Pipeline**: Supports direct uploading of documents / automatic crawling of online documents, with features for automatic text splitting, vectorization. This effectively reduces hallucinations in large models, providing a superior smart Q&A interaction experience.
|
||||
- **Agentic Workflow**: Equipped with a powerful workflow engine, function library and MCP tool-use, enabling the orchestration of AI processes to meet the needs of complex business scenarios.
|
||||
- **Ready-to-Use**: Supports direct uploading of documents / automatic crawling of online documents, with features for automatic text splitting, vectorization, and RAG (Retrieval-Augmented Generation). This effectively reduces hallucinations in large models, providing a superior smart Q&A interaction experience.
|
||||
- **Flexible Orchestration**: Equipped with a powerful workflow engine and function library, enabling the orchestration of AI processes to meet the needs of complex business scenarios.
|
||||
- **Seamless Integration**: Facilitates zero-coding rapid integration into third-party business systems, quickly equipping existing systems with intelligent Q&A capabilities to enhance user satisfaction.
|
||||
- **Model-Agnostic**: Supports various large models, including private models (such as DeepSeek, Llama, Qwen, etc.) and public models (like OpenAI, Claude, Gemini, etc.).
|
||||
- **Multi Modal**: Native support for input and output text, image, audio and video.
|
||||
|
||||
## Quick start
|
||||
|
||||
|
|
@ -54,67 +53,6 @@ Access MaxKB web interface at `http://your_server_ip:8080` with default admin cr
|
|||
- LLM Framework：[LangChain](https://www.langchain.com/)
|
||||
- Database：[PostgreSQL + pgvector](https://www.postgresql.org/)
|
||||
|
||||
## Feature Comparison
|
||||
|
||||
<table style="width: 100%;">
|
||||
<tr>
|
||||
<th align="center">Feature</th>
|
||||
<th align="center">LangChain</th>
|
||||
<th align="center">Dify.AI</th>
|
||||
<th align="center">Flowise</th>
|
||||
<th align="center">MaxKB <br>（Built upon LangChain）</th>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center">Supported LLMs</td>
|
||||
<td align="center">Rich Variety</td>
|
||||
<td align="center">Rich Variety</td>
|
||||
<td align="center">Rich Variety</td>
|
||||
<td align="center">Rich Variety</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center">RAG Engine</td>
|
||||
<td align="center">✅</td>
|
||||
<td align="center">✅</td>
|
||||
<td align="center">✅</td>
|
||||
<td align="center">✅</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center">Agent</td>
|
||||
<td align="center">✅</td>
|
||||
<td align="center">✅</td>
|
||||
<td align="center">❌</td>
|
||||
<td align="center">✅</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center">Workflow</td>
|
||||
<td align="center">❌</td>
|
||||
<td align="center">✅</td>
|
||||
<td align="center">✅</td>
|
||||
<td align="center">✅</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center">Observability</td>
|
||||
<td align="center">✅</td>
|
||||
<td align="center">✅</td>
|
||||
<td align="center">❌</td>
|
||||
<td align="center">✅</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center">SSO/Access control</td>
|
||||
<td align="center">❌</td>
|
||||
<td align="center">✅</td>
|
||||
<td align="center">❌</td>
|
||||
<td align="center">✅ (Pro)</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center">On-premise Deployment</td>
|
||||
<td align="center">✅</td>
|
||||
<td align="center">✅</td>
|
||||
<td align="center">✅</td>
|
||||
<td align="center">✅</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
## Star History
|
||||
|
||||
[](https://star-history.com/#1Panel-dev/MaxKB&Date)
|
||||
|
|
|
|||
33
README_CN.md
33
README_CN.md
|
|
@ -1,25 +1,25 @@
|
|||
<p align="center"><img src= "https://github.com/1Panel-dev/maxkb/assets/52996290/c0694996-0eed-40d8-b369-322bf2a380bf" alt="MaxKB" width="300" /></p>
|
||||
<h3 align="center">强大易用的企业级智能体平台</h3>
|
||||
<h3 align="center">基于大模型和 RAG 的知识库问答系统</h3>
|
||||
<h4 align="center">Ready-to-use, flexible RAG Chatbot</h4>
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/9113" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9113" alt="1Panel-dev%2FMaxKB | Trendshift" style="width: 250px; height: auto;" /></a>
|
||||
<a href="https://market.aliyun.com/products/53690006/cmjj00067609.html?userCode=kmemb8jp" target="_blank"><img src="https://img.alicdn.com/imgextra/i2/O1CN01H5JIwY1rZ0OobDjnJ_!!6000000005644-2-tps-1000-216.png" alt="1Panel-dev%2FMaxKB | Aliyun" style="width: 250px; height: auto;" /></a>
|
||||
</p>
|
||||
<p align="center">
|
||||
<a href="README_EN.md"><img src="https://img.shields.io/badge/English_README-blue" alt="English README"></a>
|
||||
<a href="https://www.gnu.org/licenses/gpl-3.0.html#license-text"><img src="https://img.shields.io/github/license/1Panel-dev/maxkb?color=%231890FF" alt="License: GPL v3"></a>
|
||||
<a href="https://www.gnu.org/licenses/gpl-3.0.html#license-text"><img src="https://img.shields.io/github/license/1Panel-dev/maxkb" alt="License: GPL v3"></a>
|
||||
<a href="https://github.com/1Panel-dev/maxkb/releases/latest"><img src="https://img.shields.io/github/v/release/1Panel-dev/maxkb" alt="Latest release"></a>
|
||||
<a href="https://github.com/1Panel-dev/maxkb"><img src="https://img.shields.io/github/stars/1Panel-dev/maxkb?style=flat-square" alt="Stars"></a>
|
||||
<a href="https://hub.docker.com/r/1panel/maxkb"><img src="https://img.shields.io/docker/pulls/1panel/maxkb?label=downloads" alt="Download"></a>
|
||||
<a href="https://gitee.com/fit2cloud-feizhiyun/MaxKB"><img src="https://gitee.com/fit2cloud-feizhiyun/MaxKB/badge/star.svg?theme=gvp" alt="Gitee Stars"></a>
|
||||
<a href="https://gitcode.com/feizhiyun/MaxKB"><img src="https://gitcode.com/feizhiyun/MaxKB/star/badge.svg" alt="GitCode Stars"></a>
|
||||
<a href="https://github.com/1Panel-dev/maxkb"><img src="https://img.shields.io/github/stars/1Panel-dev/maxkb?style=flat-square" alt="Stars"></a>
|
||||
<a href="https://hub.docker.com/r/1panel/maxkb"><img src="https://img.shields.io/docker/pulls/1panel/maxkb?label=downloads" alt="Download"></a>
|
||||
</p>
|
||||
<hr/>
|
||||
|
||||
MaxKB = Max Knowledge Brain，是一款强大易用的企业级智能体平台，支持 RAG 检索增强生成、工作流编排、MCP 工具调用能力。MaxKB 支持对接各种主流大语言模型，广泛应用于智能客服、企业内部知识库问答、员工助手、学术研究与教育等场景。
|
||||
MaxKB = Max Knowledge Base，是一款基于大语言模型和 RAG 的开源知识库问答系统，广泛应用于智能客服、企业内部知识库、学术研究与教育等场景。
|
||||
|
||||
- **RAG 检索增强生成**：高效搭建本地 AI 知识库，支持直接上传文档 / 自动爬取在线文档，支持文本自动拆分、向量化，有效减少大模型幻觉，提升问答效果；
|
||||
- **灵活编排**：内置强大的工作流引擎、函数库和 MCP 工具调用能力，支持编排 AI 工作过程，满足复杂业务场景下的需求；
|
||||
- **无缝嵌入**：支持零编码快速嵌入到第三方业务系统，让已有系统快速拥有智能问答能力，提高用户满意度；
|
||||
- **模型中立**：支持对接各种大模型，包括本地私有大模型（DeepSeek R1 / Llama 3 / Qwen 2 等）、国内公共大模型（通义千问 / 腾讯混元 / 字节豆包 / 百度千帆 / 智谱 AI / Kimi 等）和国外公共大模型（OpenAI / Claude / Gemini 等）。
|
||||
- **开箱即用**：支持直接上传文档 / 自动爬取在线文档，支持文本自动拆分、向量化和 RAG（检索增强生成），有效减少大模型幻觉，智能问答交互体验好；
|
||||
- **模型中立**：支持对接各种大模型，包括本地私有大模型（DeekSeek R1 / Llama 3 / Qwen 2 等）、国内公共大模型（通义千问 / 腾讯混元 / 字节豆包 / 百度千帆 / 智谱 AI / Kimi 等）和国外公共大模型（OpenAI / Claude / Gemini 等）；
|
||||
- **灵活编排**：内置强大的工作流引擎和函数库，支持编排 AI 工作过程，满足复杂业务场景下的需求；
|
||||
- **无缝嵌入**：支持零编码快速嵌入到第三方业务系统，让已有系统快速拥有智能问答能力，提高用户满意度。
|
||||
|
||||
MaxKB 三分钟视频介绍：https://www.bilibili.com/video/BV18JypYeEkj/
|
||||
|
||||
|
|
@ -39,17 +39,24 @@ docker run -d --name=maxkb --restart=always -p 8080:8080 -v C:/maxkb:/var/lib/po
|
|||
- 你也可以通过 [1Panel 应用商店](https://apps.fit2cloud.com/1panel) 快速部署 MaxKB；
|
||||
- 如果是内网环境，推荐使用 [离线安装包](https://community.fit2cloud.com/#/products/maxkb/downloads) 进行安装部署；
|
||||
- MaxKB 产品版本分为社区版和专业版，详情请参见：[MaxKB 产品版本对比](https://maxkb.cn/pricing.html)；
|
||||
- 如果您需要向团队介绍 MaxKB，可以使用这个 [官方 PPT 材料](https://maxkb.cn/download/introduce-maxkb_202503.pdf)。
|
||||
- 如果您需要向团队介绍 MaxKB，可以使用这个 [官方 PPT 材料](https://maxkb.cn/download/introduce-maxkb_202501.pdf)。
|
||||
|
||||
如你有更多问题，可以查看使用手册，或者通过论坛与我们交流。
|
||||
|
||||
- [案例展示](USE-CASES.md)
|
||||
- [使用手册](https://maxkb.cn/docs/)
|
||||
- [论坛求助](https://bbs.fit2cloud.com/c/mk/11)
|
||||
- 技术交流群
|
||||
|
||||
<image height="150px" width="150px" src="https://github.com/1Panel-dev/MaxKB/assets/52996290/a083d214-02be-4178-a1db-4f428124153a"/>
|
||||
|
||||
## 案例展示
|
||||
|
||||
MaxKB 自发布以来，日均安装下载超过 1000 次，被广泛应用于智能客服、企业内部知识库、学术教育研究等场景。
|
||||
|
||||
- [华莱士智能客服](https://ai.cnhls.com/ui/chat/1fc0f6a9b5a6fb27)
|
||||
- [JumpServer 小助手](https://maxkb.fit2cloud.com/ui/chat/b4e27a6e72d349a3)
|
||||
- [重庆交通大学教务在线](http://jwc.anyquestion.cn/ui/chat/b75496390f7d935d)
|
||||
|
||||
## UI 展示
|
||||
|
||||
<table style="border-collapse: collapse; border: 1px solid black;">
|
||||
|
|
|
|||
33
USE-CASES.md
33
USE-CASES.md
|
|
@ -1,39 +1,12 @@
|
|||
<h3 align="center">MaxKB 应用案例，持续更新中...</h3>
|
||||
<h3 align="center">MaxKB Use Cases</h3>
|
||||
|
||||
------------------------------
|
||||
|
||||
- [MaxKB 应用案例：中国农业大学-小鹉哥](https://mp.weixin.qq.com/s/4g_gySMBQZCJ9OZ-yBkmvw)
|
||||
- [MaxKB 应用案例：东北财经大学-小银杏](https://mp.weixin.qq.com/s/3BoxkY7EMomMmmvFYxvDIA)
|
||||
- [MaxKB 应用案例：中铁水务](https://mp.weixin.qq.com/s/voNAddbK2CJOrJJs1ewZ8g)
|
||||
- [MaxKB 应用案例：解放军总医院](https://mp.weixin.qq.com/s/ETrZC-vrA4Aap0eF-15EeQ)
|
||||
- [MaxKB 应用案例：无锡市数据局](https://mp.weixin.qq.com/s/enfUFLevvL_La74PQ0kIXw)
|
||||
- [MaxKB 应用案例：中核西仪研究院-西仪睿答](https://mp.weixin.qq.com/s/CbKr4mev8qahKLAtV6Dxdg)
|
||||
- [MaxKB 应用案例：南京中医药大学](https://mp.weixin.qq.com/s/WUmAKYbZjp3272HIecpRFA)
|
||||
- [MaxKB 应用案例：西北电力设计院-AI数字助理Memex](https://mp.weixin.qq.com/s/ezHFdB7C7AVL9MTtDwYGSA)
|
||||
- [MaxKB 应用案例：西安国际医院中心医院-国医小助](https://mp.weixin.qq.com/s/DSOUvwrQrxbqQxKBilTCFQ)
|
||||
- [MaxKB 应用案例：深圳信用中心 AI 助手](https://www.bilibili.com/video/BV12H4y1c7bq)
|
||||
- [MaxKB 应用案例：华莱士智能AI客服助手上线啦！](https://www.bilibili.com/video/BV1hQtVeXEBL)
|
||||
- [MaxKB 应用案例：把医疗行业知识转化为知识库问答助手！](https://www.bilibili.com/video/BV157wme9EgB)
|
||||
- [把医疗行业知识转化为知识库问答助手！](https://www.bilibili.com/video/BV157wme9EgB)
|
||||
- [MaxKB 应用案例：会展AI智能客服体验](https://www.bilibili.com/video/BV1J7BqY6EKA)
|
||||
- [MaxKB 应用案例：孩子要上幼儿园了，AI 智能助手择校好帮手](https://www.bilibili.com/video/BV1wKrhYvEer)
|
||||
- [MaxKB 应用案例：产品使用指南AI助手，新手小白也能轻松搞定！](https://www.bilibili.com/video/BV1Yz6gYtEqX)
|
||||
- [MaxKB 应用案例：生物医药AI客服智能体验!](https://www.bilibili.com/video/BV13JzvYsE3e)
|
||||
- [MaxKB 应用案例：高校行政管理AI小助手](https://www.bilibili.com/video/BV1yvBMYvEdy)
|
||||
- [MaxKB 应用案例：岳阳市人民医院-OA小助手](https://mp.weixin.qq.com/s/O94Qo3UH-MiUtDdWCVg8sQ)
|
||||
- [MaxKB 应用案例：常熟市第一人民医院](https://mp.weixin.qq.com/s/s5XXGTR3_MUo41NbJ8WzZQ)
|
||||
- [MaxKB 应用案例：华北水利水电大学](https://mp.weixin.qq.com/s/PoOFAcMCr9qJdvSj8c08qg)
|
||||
- [MaxKB 应用案例：唐山海事局-“小海”AI语音助手](https://news.qq.com/rain/a/20250223A030BE00)
|
||||
- [MaxKB 应用案例：湖南汉寿政务](http://hsds.hsdj.gov.cn:19999/ui/chat/a2c976736739aadc)
|
||||
- [MaxKB 应用案例：广州市妇女儿童医疗中心-AI医疗数据分类分级小助手](https://mp.weixin.qq.com/s/YHUMkUOAaUomBV8bswpK3g)
|
||||
- [MaxKB 应用案例：苏州热工研究院有限公司-维修大纲评估质量自查AI小助手](https://mp.weixin.qq.com/s/Ts5FQdnv7Tu9Jp7bvofCVA)
|
||||
- [MaxKB 应用案例：国核自仪系统工程有限公司-NuCON AI帮](https://mp.weixin.qq.com/s/HNPc7u5xVfGLJr8IQz3vjQ)
|
||||
- [MaxKB 应用案例：深圳通开启Deep Seek智能应用新篇章](https://mp.weixin.qq.com/s/SILN0GSescH9LyeQqYP0VQ)
|
||||
- [MaxKB 应用案例：南通智慧出行领跑长三角！首款接入DeepSeek的"畅行南通"APP上线AI新场景](https://mp.weixin.qq.com/s/WEC9UQ6msY0VS8LhTZh-Ew)
|
||||
- [MaxKB 应用案例：中船动力人工智能"智慧动力云助手"及首批数字员工正式上线](https://mp.weixin.qq.com/s/OGcEkjh9DzGO1Tkc9nr7qg)
|
||||
- [MaxKB 应用案例：AI+矿山：DeepSeek助力绿色智慧矿山智慧“升级”](https://mp.weixin.qq.com/s/SZstxTvVoLZg0ECbZbfpIA)
|
||||
- [MaxKB 应用案例：DeepSeek落地弘盛铜业：国产大模型点亮"黑灯工厂"新引擎](https://mp.weixin.qq.com/s/Eczdx574MS5RMF7WfHN7_A)
|
||||
- [MaxKB 应用案例：拥抱智能时代！中国五矿以 “AI+”赋能企业发展](https://mp.weixin.qq.com/s/D5vBtlX2E81pWE3_2OgWSw)
|
||||
- [MaxKB 应用案例：DeepSeek赋能中冶武勘AI智能体](https://mp.weixin.qq.com/s/8m0vxGcWXNdZazziQrLyxg)
|
||||
- [MaxKB 应用案例：重磅！陕西广电网络“秦岭云”平台实现DeepSeek本地化部署](https://mp.weixin.qq.com/s/ZKmEU_wWShK1YDomKJHQeA)
|
||||
- [MaxKB 应用案例：粤海集团完成DeepSeek私有化部署，助力集团智能化管理](https://mp.weixin.qq.com/s/2JbVp0-kr9Hfp-0whH4cvg)
|
||||
- [MaxKB 应用案例：建筑材料工业信息中心完成DeepSeek本地化部署，推动行业数智化转型新发展](https://mp.weixin.qq.com/s/HThGSnND3qDF8ySEqiM4jw)
|
||||
- [MaxKB 应用案例：一起DeepSeek！福建设计以AI大模型开启新篇章](https://mp.weixin.qq.com/s/m67e-H7iQBg3d24NM82UjA)
|
||||
|
|
|
|||
|
|
@ -124,11 +124,9 @@ def event_content(response,
|
|||
request_token = 0
|
||||
response_token = 0
|
||||
write_context(step, manage, request_token, response_token, all_text)
|
||||
asker = manage.context.get('form_data', {}).get('asker', None)
|
||||
post_response_handler.handler(chat_id, chat_record_id, paragraph_list, problem_text,
|
||||
all_text, manage, step, padding_problem_text, client_id,
|
||||
reasoning_content=reasoning_content if reasoning_content_enable else ''
|
||||
, asker=asker)
|
||||
reasoning_content=reasoning_content if reasoning_content_enable else '')
|
||||
yield manage.get_base_to_response().to_stream_chunk_response(chat_id, str(chat_record_id), 'ai-chat-node',
|
||||
[], '', True,
|
||||
request_token, response_token,
|
||||
|
|
@ -137,21 +135,16 @@ def event_content(response,
|
|||
add_access_num(client_id, client_type, manage.context.get('application_id'))
|
||||
except Exception as e:
|
||||
logging.getLogger("max_kb_error").error(f'{str(e)}:{traceback.format_exc()}')
|
||||
all_text = 'Exception:' + str(e)
|
||||
all_text = '异常' + str(e)
|
||||
write_context(step, manage, 0, 0, all_text)
|
||||
asker = manage.context.get('form_data', {}).get('asker', None)
|
||||
post_response_handler.handler(chat_id, chat_record_id, paragraph_list, problem_text,
|
||||
all_text, manage, step, padding_problem_text, client_id, reasoning_content='',
|
||||
asker=asker)
|
||||
all_text, manage, step, padding_problem_text, client_id)
|
||||
add_access_num(client_id, client_type, manage.context.get('application_id'))
|
||||
yield manage.get_base_to_response().to_stream_chunk_response(chat_id, str(chat_record_id), 'ai-chat-node',
|
||||
[], all_text,
|
||||
False,
|
||||
0, 0, {'node_is_end': False,
|
||||
'view_type': 'many_view',
|
||||
'node_type': 'ai-chat-node',
|
||||
'real_node_id': 'ai-chat-node',
|
||||
'reasoning_content': ''})
|
||||
yield manage.get_base_to_response().to_stream_chunk_response(chat_id, str(chat_record_id), all_text,
|
||||
'ai-chat-node',
|
||||
[], True, 0, 0,
|
||||
{'node_is_end': True, 'view_type': 'many_view',
|
||||
'node_type': 'ai-chat-node'})
|
||||
|
||||
|
||||
class BaseChatStep(IChatStep):
|
||||
|
|
@ -307,28 +300,19 @@ class BaseChatStep(IChatStep):
|
|||
else:
|
||||
reasoning_content = reasoning_result.get('reasoning_content') + reasoning_result_end.get(
|
||||
'reasoning_content')
|
||||
asker = manage.context.get('form_data', {}).get('asker', None)
|
||||
post_response_handler.handler(chat_id, chat_record_id, paragraph_list, problem_text,
|
||||
content, manage, self, padding_problem_text, client_id,
|
||||
reasoning_content=reasoning_content if reasoning_content_enable else '',
|
||||
asker=asker)
|
||||
chat_result.content, manage, self, padding_problem_text, client_id,
|
||||
reasoning_content=reasoning_content if reasoning_content_enable else '')
|
||||
add_access_num(client_id, client_type, manage.context.get('application_id'))
|
||||
return manage.get_base_to_response().to_block_response(str(chat_id), str(chat_record_id),
|
||||
content, True,
|
||||
request_token, response_token,
|
||||
{
|
||||
'reasoning_content': reasoning_content if reasoning_content_enable else '',
|
||||
'answer_list': [{
|
||||
'content': content,
|
||||
'reasoning_content': reasoning_content if reasoning_content_enable else ''
|
||||
}]})
|
||||
{'reasoning_content': reasoning_content})
|
||||
except Exception as e:
|
||||
all_text = 'Exception:' + str(e)
|
||||
write_context(self, manage, 0, 0, all_text)
|
||||
asker = manage.context.get('form_data', {}).get('asker', None)
|
||||
post_response_handler.handler(chat_id, chat_record_id, paragraph_list, problem_text,
|
||||
all_text, manage, self, padding_problem_text, client_id, reasoning_content='',
|
||||
asker=asker)
|
||||
all_text, manage, self, padding_problem_text, client_id)
|
||||
add_access_num(client_id, client_type, manage.context.get('application_id'))
|
||||
return manage.get_base_to_response().to_block_response(str(chat_id), str(chat_record_id), all_text, True, 0,
|
||||
0, _status=status.HTTP_500_INTERNAL_SERVER_ERROR)
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ class ISearchDatasetStep(IBaseChatPipelineStep):
|
|||
error_messages=ErrMessage.float(_("Similarity")))
|
||||
search_mode = serializers.CharField(required=True, validators=[
|
||||
validators.RegexValidator(regex=re.compile("^embedding|keywords|blend$"),
|
||||
message=_("The type only supports embedding|keywords|blend"), code=500)
|
||||
message=_("The type only supports register|reset_password"), code=500)
|
||||
], error_messages=ErrMessage.char(_("Retrieval Mode")))
|
||||
user_id = serializers.UUIDField(required=True, error_messages=ErrMessage.uuid(_("User ID")))
|
||||
|
||||
|
|
|
|||
|
|
@ -84,8 +84,7 @@ class WorkFlowPostHandler:
|
|||
answer_text_list=answer_text_list,
|
||||
run_time=time.time() - workflow.context['start_time'],
|
||||
index=0)
|
||||
asker = workflow.context.get('asker', None)
|
||||
self.chat_info.append_chat_record(chat_record, self.client_id, asker)
|
||||
self.chat_info.append_chat_record(chat_record, self.client_id)
|
||||
# 重新设置缓存
|
||||
chat_cache.set(chat_id,
|
||||
self.chat_info, timeout=60 * 30)
|
||||
|
|
|
|||
|
|
@ -24,15 +24,11 @@ from .search_dataset_node import *
|
|||
from .speech_to_text_step_node import BaseSpeechToTextNode
|
||||
from .start_node import *
|
||||
from .text_to_speech_step_node.impl.base_text_to_speech_node import BaseTextToSpeechNode
|
||||
from .variable_assign_node import BaseVariableAssignNode
|
||||
from .mcp_node import BaseMcpNode
|
||||
|
||||
node_list = [BaseStartStepNode, BaseChatNode, BaseSearchDatasetNode, BaseQuestionNode,
|
||||
BaseConditionNode, BaseReplyNode,
|
||||
node_list = [BaseStartStepNode, BaseChatNode, BaseSearchDatasetNode, BaseQuestionNode, BaseConditionNode, BaseReplyNode,
|
||||
BaseFunctionNodeNode, BaseFunctionLibNodeNode, BaseRerankerNode, BaseApplicationNode,
|
||||
BaseDocumentExtractNode,
|
||||
BaseImageUnderstandNode, BaseFormNode, BaseSpeechToTextNode, BaseTextToSpeechNode,
|
||||
BaseImageGenerateNode, BaseVariableAssignNode, BaseMcpNode]
|
||||
BaseImageUnderstandNode, BaseFormNode, BaseSpeechToTextNode, BaseTextToSpeechNode,BaseImageGenerateNode]
|
||||
|
||||
|
||||
def get_node(node_type):
|
||||
|
|
|
|||
|
|
@ -33,9 +33,6 @@ class ChatNodeSerializer(serializers.Serializer):
|
|||
error_messages=ErrMessage.dict('Model settings'))
|
||||
dialogue_type = serializers.CharField(required=False, allow_blank=True, allow_null=True,
|
||||
error_messages=ErrMessage.char(_("Context Type")))
|
||||
mcp_enable = serializers.BooleanField(required=False,
|
||||
error_messages=ErrMessage.boolean(_("Whether to enable MCP")))
|
||||
mcp_servers = serializers.JSONField(required=False, error_messages=ErrMessage.list(_("MCP Server")))
|
||||
|
||||
|
||||
class IChatNode(INode):
|
||||
|
|
@ -52,7 +49,5 @@ class IChatNode(INode):
|
|||
model_params_setting=None,
|
||||
dialogue_type=None,
|
||||
model_setting=None,
|
||||
mcp_enable=False,
|
||||
mcp_servers=None,
|
||||
**kwargs) -> NodeResult:
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -6,19 +6,14 @@
|
|||
@date：2024/6/4 14:30
|
||||
@desc:
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from functools import reduce
|
||||
from types import AsyncGeneratorType
|
||||
from typing import List, Dict
|
||||
|
||||
from django.db.models import QuerySet
|
||||
from langchain.schema import HumanMessage, SystemMessage
|
||||
from langchain_core.messages import BaseMessage, AIMessage, AIMessageChunk, ToolMessage
|
||||
from langchain_mcp_adapters.client import MultiServerMCPClient
|
||||
from langgraph.prebuilt import create_react_agent
|
||||
from langchain_core.messages import BaseMessage, AIMessage
|
||||
|
||||
from application.flow.i_step_node import NodeResult, INode
|
||||
from application.flow.step_node.ai_chat_step_node.i_chat_node import IChatNode
|
||||
|
|
@ -27,19 +22,6 @@ from setting.models import Model
|
|||
from setting.models_provider import get_model_credential
|
||||
from setting.models_provider.tools import get_model_instance_by_model_user_id
|
||||
|
||||
tool_message_template = """
|
||||
<details>
|
||||
<summary>
|
||||
<strong>Called MCP Tool: <em>%s</em></strong>
|
||||
</summary>
|
||||
|
||||
```json
|
||||
%s
|
||||
```
|
||||
</details>
|
||||
|
||||
"""
|
||||
|
||||
|
||||
def _write_context(node_variable: Dict, workflow_variable: Dict, node: INode, workflow, answer: str,
|
||||
reasoning_content: str):
|
||||
|
|
@ -74,7 +56,6 @@ def write_context_stream(node_variable: Dict, workflow_variable: Dict, node: INo
|
|||
reasoning = Reasoning(model_setting.get('reasoning_content_start', '<think>'),
|
||||
model_setting.get('reasoning_content_end', '</think>'))
|
||||
response_reasoning_content = False
|
||||
|
||||
for chunk in response:
|
||||
reasoning_chunk = reasoning.get_reasoning_content(chunk)
|
||||
content_chunk = reasoning_chunk.get('content')
|
||||
|
|
@ -103,39 +84,6 @@ def write_context_stream(node_variable: Dict, workflow_variable: Dict, node: INo
|
|||
_write_context(node_variable, workflow_variable, node, workflow, answer, reasoning_content)
|
||||
|
||||
|
||||
async def _yield_mcp_response(chat_model, message_list, mcp_servers):
|
||||
async with MultiServerMCPClient(json.loads(mcp_servers)) as client:
|
||||
agent = create_react_agent(chat_model, client.get_tools())
|
||||
response = agent.astream({"messages": message_list}, stream_mode='messages')
|
||||
async for chunk in response:
|
||||
if isinstance(chunk[0], ToolMessage):
|
||||
content = tool_message_template % (chunk[0].name, chunk[0].content)
|
||||
chunk[0].content = content
|
||||
yield chunk[0]
|
||||
if isinstance(chunk[0], AIMessageChunk):
|
||||
yield chunk[0]
|
||||
|
||||
|
||||
def mcp_response_generator(chat_model, message_list, mcp_servers):
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
async_gen = _yield_mcp_response(chat_model, message_list, mcp_servers)
|
||||
while True:
|
||||
try:
|
||||
chunk = loop.run_until_complete(anext_async(async_gen))
|
||||
yield chunk
|
||||
except StopAsyncIteration:
|
||||
break
|
||||
except Exception as e:
|
||||
print(f'exception: {e}')
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
async def anext_async(agen):
|
||||
return await agen.__anext__()
|
||||
|
||||
|
||||
def write_context(node_variable: Dict, workflow_variable: Dict, node: INode, workflow):
|
||||
"""
|
||||
写入上下文数据
|
||||
|
|
@ -188,15 +136,12 @@ class BaseChatNode(IChatNode):
|
|||
self.context['answer'] = details.get('answer')
|
||||
self.context['question'] = details.get('question')
|
||||
self.context['reasoning_content'] = details.get('reasoning_content')
|
||||
if self.node_params.get('is_result', False):
|
||||
self.answer_text = details.get('answer')
|
||||
self.answer_text = details.get('answer')
|
||||
|
||||
def execute(self, model_id, system, prompt, dialogue_number, history_chat_record, stream, chat_id, chat_record_id,
|
||||
model_params_setting=None,
|
||||
dialogue_type=None,
|
||||
model_setting=None,
|
||||
mcp_enable=False,
|
||||
mcp_servers=None,
|
||||
**kwargs) -> NodeResult:
|
||||
if dialogue_type is None:
|
||||
dialogue_type = 'WORKFLOW'
|
||||
|
|
@ -218,14 +163,6 @@ class BaseChatNode(IChatNode):
|
|||
self.context['system'] = system
|
||||
message_list = self.generate_message_list(system, prompt, history_message)
|
||||
self.context['message_list'] = message_list
|
||||
|
||||
if mcp_enable and mcp_servers is not None and '"stdio"' not in mcp_servers:
|
||||
r = mcp_response_generator(chat_model, message_list, mcp_servers)
|
||||
return NodeResult(
|
||||
{'result': r, 'chat_model': chat_model, 'message_list': message_list,
|
||||
'history_message': history_message, 'question': question.content}, {},
|
||||
_write_context=write_context_stream)
|
||||
|
||||
if stream:
|
||||
r = chat_model.stream(message_list)
|
||||
return NodeResult({'result': r, 'chat_model': chat_model, 'message_list': message_list,
|
||||
|
|
|
|||
|
|
@ -11,16 +11,13 @@ from django.utils.translation import gettext_lazy as _
|
|||
|
||||
class ApplicationNodeSerializer(serializers.Serializer):
|
||||
application_id = serializers.CharField(required=True, error_messages=ErrMessage.char(_("Application ID")))
|
||||
question_reference_address = serializers.ListField(required=True,
|
||||
error_messages=ErrMessage.list(_("User Questions")))
|
||||
question_reference_address = serializers.ListField(required=True, error_messages=ErrMessage.list(_("User Questions")))
|
||||
api_input_field_list = serializers.ListField(required=False, error_messages=ErrMessage.list(_("API Input Fields")))
|
||||
user_input_field_list = serializers.ListField(required=False,
|
||||
error_messages=ErrMessage.uuid(_("User Input Fields")))
|
||||
user_input_field_list = serializers.ListField(required=False, error_messages=ErrMessage.uuid(_("User Input Fields")))
|
||||
image_list = serializers.ListField(required=False, error_messages=ErrMessage.list(_("picture")))
|
||||
document_list = serializers.ListField(required=False, error_messages=ErrMessage.list(_("document")))
|
||||
audio_list = serializers.ListField(required=False, error_messages=ErrMessage.list(_("Audio")))
|
||||
child_node = serializers.DictField(required=False, allow_null=True,
|
||||
error_messages=ErrMessage.dict(_("Child Nodes")))
|
||||
child_node = serializers.DictField(required=False, allow_null=True, error_messages=ErrMessage.dict(_("Child Nodes")))
|
||||
node_data = serializers.DictField(required=False, allow_null=True, error_messages=ErrMessage.dict(_("Form Data")))
|
||||
|
||||
|
||||
|
|
@ -36,16 +33,11 @@ class IApplicationNode(INode):
|
|||
self.node_params_serializer.data.get('question_reference_address')[1:])
|
||||
kwargs = {}
|
||||
for api_input_field in self.node_params_serializer.data.get('api_input_field_list', []):
|
||||
value = api_input_field.get('value', [''])[0] if api_input_field.get('value') else ''
|
||||
kwargs[api_input_field['variable']] = self.workflow_manage.get_reference_field(value,
|
||||
api_input_field['value'][
|
||||
1:]) if value != '' else ''
|
||||
|
||||
kwargs[api_input_field['variable']] = self.workflow_manage.get_reference_field(api_input_field['value'][0],
|
||||
api_input_field['value'][1:])
|
||||
for user_input_field in self.node_params_serializer.data.get('user_input_field_list', []):
|
||||
value = user_input_field.get('value', [''])[0] if user_input_field.get('value') else ''
|
||||
kwargs[user_input_field['field']] = self.workflow_manage.get_reference_field(value,
|
||||
user_input_field['value'][
|
||||
1:]) if value != '' else ''
|
||||
kwargs[user_input_field['field']] = self.workflow_manage.get_reference_field(user_input_field['value'][0],
|
||||
user_input_field['value'][1:])
|
||||
# 判断是否包含这个属性
|
||||
app_document_list = self.node_params_serializer.data.get('document_list', [])
|
||||
if app_document_list and len(app_document_list) > 0:
|
||||
|
|
@ -54,8 +46,7 @@ class IApplicationNode(INode):
|
|||
app_document_list[1:])
|
||||
for document in app_document_list:
|
||||
if 'file_id' not in document:
|
||||
raise ValueError(
|
||||
_("Parameter value error: The uploaded document lacks file_id, and the document upload fails"))
|
||||
raise ValueError(_("Parameter value error: The uploaded document lacks file_id, and the document upload fails"))
|
||||
app_image_list = self.node_params_serializer.data.get('image_list', [])
|
||||
if app_image_list and len(app_image_list) > 0:
|
||||
app_image_list = self.workflow_manage.get_reference_field(
|
||||
|
|
@ -63,8 +54,7 @@ class IApplicationNode(INode):
|
|||
app_image_list[1:])
|
||||
for image in app_image_list:
|
||||
if 'file_id' not in image:
|
||||
raise ValueError(
|
||||
_("Parameter value error: The uploaded image lacks file_id, and the image upload fails"))
|
||||
raise ValueError(_("Parameter value error: The uploaded image lacks file_id, and the image upload fails"))
|
||||
|
||||
app_audio_list = self.node_params_serializer.data.get('audio_list', [])
|
||||
if app_audio_list and len(app_audio_list) > 0:
|
||||
|
|
@ -73,8 +63,7 @@ class IApplicationNode(INode):
|
|||
app_audio_list[1:])
|
||||
for audio in app_audio_list:
|
||||
if 'file_id' not in audio:
|
||||
raise ValueError(
|
||||
_("Parameter value error: The uploaded audio lacks file_id, and the audio upload fails."))
|
||||
raise ValueError(_("Parameter value error: The uploaded audio lacks file_id, and the audio upload fails."))
|
||||
return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data,
|
||||
app_document_list=app_document_list, app_image_list=app_image_list,
|
||||
app_audio_list=app_audio_list,
|
||||
|
|
|
|||
|
|
@ -115,10 +115,6 @@ def write_context(node_variable: Dict, workflow_variable: Dict, node: INode, wor
|
|||
'prompt_tokens': response.get('prompt_tokens')}}
|
||||
answer = response.get('content', '') or "抱歉，没有查找到相关内容，请重新描述您的问题或提供更多信息。"
|
||||
reasoning_content = response.get('reasoning_content', '')
|
||||
answer_list = response.get('answer_list', [])
|
||||
node_variable['application_node_dict'] = {answer.get('real_node_id'): {**answer, 'index': index} for answer, index
|
||||
in
|
||||
zip(answer_list, range(len(answer_list)))}
|
||||
_write_context(node_variable, workflow_variable, node, workflow, answer, reasoning_content)
|
||||
|
||||
|
||||
|
|
@ -168,8 +164,7 @@ class BaseApplicationNode(IApplicationNode):
|
|||
self.context['question'] = details.get('question')
|
||||
self.context['type'] = details.get('type')
|
||||
self.context['reasoning_content'] = details.get('reasoning_content')
|
||||
if self.node_params.get('is_result', False):
|
||||
self.answer_text = details.get('answer')
|
||||
self.answer_text = details.get('answer')
|
||||
|
||||
def execute(self, application_id, message, chat_id, chat_record_id, stream, re_chat, client_id, client_type,
|
||||
app_document_list=None, app_image_list=None, app_audio_list=None, child_node=None, node_data=None,
|
||||
|
|
@ -179,8 +174,7 @@ class BaseApplicationNode(IApplicationNode):
|
|||
current_chat_id = string_to_uuid(chat_id + application_id)
|
||||
Chat.objects.get_or_create(id=current_chat_id, defaults={
|
||||
'application_id': application_id,
|
||||
'abstract': message[0:1024],
|
||||
'client_id': client_id,
|
||||
'abstract': message
|
||||
})
|
||||
if app_document_list is None:
|
||||
app_document_list = []
|
||||
|
|
@ -226,25 +220,20 @@ class BaseApplicationNode(IApplicationNode):
|
|||
def get_details(self, index: int, **kwargs):
|
||||
global_fields = []
|
||||
for api_input_field in self.node_params_serializer.data.get('api_input_field_list', []):
|
||||
value = api_input_field.get('value', [''])[0] if api_input_field.get('value') else ''
|
||||
global_fields.append({
|
||||
'label': api_input_field['variable'],
|
||||
'key': api_input_field['variable'],
|
||||
'value': self.workflow_manage.get_reference_field(
|
||||
value,
|
||||
api_input_field['value'][1:]
|
||||
) if value != '' else ''
|
||||
api_input_field['value'][0],
|
||||
api_input_field['value'][1:])
|
||||
})
|
||||
|
||||
for user_input_field in self.node_params_serializer.data.get('user_input_field_list', []):
|
||||
value = user_input_field.get('value', [''])[0] if user_input_field.get('value') else ''
|
||||
global_fields.append({
|
||||
'label': user_input_field['label'],
|
||||
'key': user_input_field['field'],
|
||||
'value': self.workflow_manage.get_reference_field(
|
||||
value,
|
||||
user_input_field['value'][1:]
|
||||
) if value != '' else ''
|
||||
user_input_field['value'][0],
|
||||
user_input_field['value'][1:])
|
||||
})
|
||||
return {
|
||||
'name': self.node.properties.get('stepName'),
|
||||
|
|
|
|||
|
|
@ -9,22 +9,20 @@
|
|||
|
||||
from .contain_compare import *
|
||||
from .equal_compare import *
|
||||
from .ge_compare import *
|
||||
from .gt_compare import *
|
||||
from .is_not_null_compare import *
|
||||
from .is_not_true import IsNotTrueCompare
|
||||
from .is_null_compare import *
|
||||
from .is_true import IsTrueCompare
|
||||
from .ge_compare import *
|
||||
from .le_compare import *
|
||||
from .len_equal_compare import *
|
||||
from .lt_compare import *
|
||||
from .len_ge_compare import *
|
||||
from .len_gt_compare import *
|
||||
from .len_le_compare import *
|
||||
from .len_lt_compare import *
|
||||
from .lt_compare import *
|
||||
from .len_equal_compare import *
|
||||
from .is_not_null_compare import *
|
||||
from .is_null_compare import *
|
||||
from .not_contain_compare import *
|
||||
|
||||
compare_handle_list = [GECompare(), GTCompare(), ContainCompare(), EqualCompare(), LTCompare(), LECompare(),
|
||||
LenLECompare(), LenGECompare(), LenEqualCompare(), LenGTCompare(), LenLTCompare(),
|
||||
IsNullCompare(),
|
||||
IsNotNullCompare(), NotContainCompare(), IsTrueCompare(), IsNotTrueCompare()]
|
||||
IsNotNullCompare(), NotContainCompare()]
|
||||
|
|
|
|||
|
|
@ -1,24 +0,0 @@
|
|||
# coding=utf-8
|
||||
"""
|
||||
@project: MaxKB
|
||||
@Author：虎
|
||||
@file： is_not_true.py
|
||||
@date：2025/4/7 13:44
|
||||
@desc:
|
||||
"""
|
||||
from typing import List
|
||||
|
||||
from application.flow.step_node.condition_node.compare import Compare
|
||||
|
||||
|
||||
class IsNotTrueCompare(Compare):
|
||||
|
||||
def support(self, node_id, fields: List[str], source_value, compare, target_value):
|
||||
if compare == 'is_not_true':
|
||||
return True
|
||||
|
||||
def compare(self, source_value, compare, target_value):
|
||||
try:
|
||||
return source_value is False
|
||||
except Exception as e:
|
||||
return False
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
# coding=utf-8
|
||||
"""
|
||||
@project: MaxKB
|
||||
@Author：虎
|
||||
@file： IsTrue.py
|
||||
@date：2025/4/7 13:38
|
||||
@desc:
|
||||
"""
|
||||
from typing import List
|
||||
|
||||
from application.flow.step_node.condition_node.compare import Compare
|
||||
|
||||
|
||||
class IsTrueCompare(Compare):
|
||||
|
||||
def support(self, node_id, fields: List[str], source_value, compare, target_value):
|
||||
if compare == 'is_true':
|
||||
return True
|
||||
|
||||
def compare(self, source_value, compare, target_value):
|
||||
try:
|
||||
return source_value is True
|
||||
except Exception as e:
|
||||
return False
|
||||
|
|
@ -36,15 +36,7 @@ class BaseConditionNode(IConditionNode):
|
|||
return all(condition_list) if condition == 'and' else any(condition_list)
|
||||
|
||||
def assertion(self, field_list: List[str], compare: str, value):
|
||||
try:
|
||||
value = self.workflow_manage.generate_prompt(value)
|
||||
except Exception as e:
|
||||
pass
|
||||
field_value = None
|
||||
try:
|
||||
field_value = self.workflow_manage.get_reference_field(field_list[0], field_list[1:])
|
||||
except Exception as e:
|
||||
pass
|
||||
field_value = self.workflow_manage.get_reference_field(field_list[0], field_list[1:])
|
||||
for compare_handler in compare_handle_list:
|
||||
if compare_handler.support(field_list[0], field_list[1:], field_value, compare, value):
|
||||
return compare_handler.compare(field_value, compare, value)
|
||||
|
|
|
|||
|
|
@ -15,9 +15,7 @@ from application.flow.step_node.direct_reply_node.i_reply_node import IReplyNode
|
|||
class BaseReplyNode(IReplyNode):
|
||||
def save_context(self, details, workflow_manage):
|
||||
self.context['answer'] = details.get('answer')
|
||||
if self.node_params.get('is_result', False):
|
||||
self.answer_text = details.get('answer')
|
||||
|
||||
self.answer_text = details.get('answer')
|
||||
def execute(self, reply_type, stream, fields=None, content=None, **kwargs) -> NodeResult:
|
||||
if reply_type == 'referencing':
|
||||
result = self.get_reference_content(fields)
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ splitter = '\n`-----------------------------------`\n'
|
|||
class BaseDocumentExtractNode(IDocumentExtractNode):
|
||||
def save_context(self, details, workflow_manage):
|
||||
self.context['content'] = details.get('content')
|
||||
self.answer_text = details.get('content')
|
||||
|
||||
|
||||
def execute(self, document, chat_id, **kwargs):
|
||||
|
|
|
|||
|
|
@ -38,8 +38,7 @@ class BaseFormNode(IFormNode):
|
|||
self.context['start_time'] = details.get('start_time')
|
||||
self.context['form_data'] = form_data
|
||||
self.context['is_submit'] = details.get('is_submit')
|
||||
if self.node_params.get('is_result', False):
|
||||
self.answer_text = details.get('result')
|
||||
self.answer_text = details.get('result')
|
||||
if form_data is not None:
|
||||
for key in form_data:
|
||||
self.context[key] = form_data[key]
|
||||
|
|
@ -71,7 +70,7 @@ class BaseFormNode(IFormNode):
|
|||
"chat_record_id": self.flow_params_serializer.data.get("chat_record_id"),
|
||||
'form_data': self.context.get('form_data', {}),
|
||||
"is_submit": self.context.get("is_submit", False)}
|
||||
form = f'<form_rander>{json.dumps(form_setting, ensure_ascii=False)}</form_rander>'
|
||||
form = f'<form_rander>{json.dumps(form_setting,ensure_ascii=False)}</form_rander>'
|
||||
context = self.workflow_manage.get_workflow_content()
|
||||
form_content_format = self.workflow_manage.reset_prompt(form_content_format)
|
||||
prompt_template = PromptTemplate.from_template(form_content_format, template_format='jinja2')
|
||||
|
|
@ -86,7 +85,7 @@ class BaseFormNode(IFormNode):
|
|||
"chat_record_id": self.flow_params_serializer.data.get("chat_record_id"),
|
||||
'form_data': self.context.get('form_data', {}),
|
||||
"is_submit": self.context.get("is_submit", False)}
|
||||
form = f'<form_rander>{json.dumps(form_setting, ensure_ascii=False)}</form_rander>'
|
||||
form = f'<form_rander>{json.dumps(form_setting,ensure_ascii=False)}</form_rander>'
|
||||
context = self.workflow_manage.get_workflow_content()
|
||||
form_content_format = self.workflow_manage.reset_prompt(form_content_format)
|
||||
prompt_template = PromptTemplate.from_template(form_content_format, template_format='jinja2')
|
||||
|
|
|
|||
|
|
@ -11,13 +11,11 @@ import time
|
|||
from typing import Dict
|
||||
|
||||
from django.db.models import QuerySet
|
||||
from django.utils.translation import gettext as _
|
||||
|
||||
from application.flow.i_step_node import NodeResult
|
||||
from application.flow.step_node.function_lib_node.i_function_lib_node import IFunctionLibNode
|
||||
from common.exception.app_exception import AppApiException
|
||||
from common.util.function_code import FunctionExecutor
|
||||
from common.util.rsa_util import rsa_long_decrypt
|
||||
from function_lib.models.function import FunctionLib
|
||||
from smartdoc.const import CONFIG
|
||||
|
||||
|
|
@ -40,15 +38,15 @@ def get_field_value(debug_field_list, name, is_required):
|
|||
if len(result) > 0:
|
||||
return result[-1]['value']
|
||||
if is_required:
|
||||
raise AppApiException(500, _('Field: {name} No value set').format(name=name))
|
||||
raise AppApiException(500, f"{name}字段未设置值")
|
||||
return None
|
||||
|
||||
|
||||
def valid_reference_value(_type, value, name):
|
||||
if _type == 'int':
|
||||
instance_type = int | float
|
||||
instance_type = int
|
||||
elif _type == 'float':
|
||||
instance_type = float | int
|
||||
instance_type = float
|
||||
elif _type == 'dict':
|
||||
instance_type = dict
|
||||
elif _type == 'array':
|
||||
|
|
@ -56,16 +54,13 @@ def valid_reference_value(_type, value, name):
|
|||
elif _type == 'string':
|
||||
instance_type = str
|
||||
else:
|
||||
raise Exception(_('Field: {name} Type: {_type} Value: {value} Unsupported types').format(name=name,
|
||||
_type=_type))
|
||||
raise Exception(500, f'字段:{name}类型:{_type} 不支持的类型')
|
||||
if not isinstance(value, instance_type):
|
||||
raise Exception(
|
||||
_('Field: {name} Type: {_type} Value: {value} Type error').format(name=name, _type=_type,
|
||||
value=value))
|
||||
raise Exception(f'字段:{name}类型:{_type}值:{value}类型错误')
|
||||
|
||||
|
||||
def convert_value(name: str, value, _type, is_required, source, node):
|
||||
if not is_required and (value is None or (isinstance(value, str) and len(value) == 0)):
|
||||
if not is_required and value is None:
|
||||
return None
|
||||
if not is_required and source == 'reference' and (value is None or len(value) == 0):
|
||||
return None
|
||||
|
|
@ -74,10 +69,6 @@ def convert_value(name: str, value, _type, is_required, source, node):
|
|||
value[0],
|
||||
value[1:])
|
||||
valid_reference_value(_type, value, name)
|
||||
if _type == 'int':
|
||||
return int(value)
|
||||
if _type == 'float':
|
||||
return float(value)
|
||||
return value
|
||||
try:
|
||||
if _type == 'int':
|
||||
|
|
@ -88,37 +79,26 @@ def convert_value(name: str, value, _type, is_required, source, node):
|
|||
v = json.loads(value)
|
||||
if isinstance(v, dict):
|
||||
return v
|
||||
raise Exception(_('type error'))
|
||||
raise Exception("类型错误")
|
||||
if _type == 'array':
|
||||
v = json.loads(value)
|
||||
if isinstance(v, list):
|
||||
return v
|
||||
raise Exception(_('type error'))
|
||||
raise Exception("类型错误")
|
||||
return value
|
||||
except Exception as e:
|
||||
raise Exception(
|
||||
_('Field: {name} Type: {_type} Value: {value} Type error').format(name=name, _type=_type,
|
||||
value=value))
|
||||
|
||||
|
||||
def valid_function(function_lib, user_id):
|
||||
if function_lib is None:
|
||||
raise Exception(_('Function does not exist'))
|
||||
if function_lib.permission_type == 'PRIVATE' and str(function_lib.user_id) != str(user_id):
|
||||
raise Exception(_('No permission to use this function {name}').format(name=function_lib.name))
|
||||
if not function_lib.is_active:
|
||||
raise Exception(_('Function {name} is unavailable').format(name=function_lib.name))
|
||||
raise Exception(f'字段:{name}类型:{_type}值:{value}类型错误')
|
||||
|
||||
|
||||
class BaseFunctionLibNodeNode(IFunctionLibNode):
|
||||
def save_context(self, details, workflow_manage):
|
||||
self.context['result'] = details.get('result')
|
||||
if self.node_params.get('is_result'):
|
||||
self.answer_text = str(details.get('result'))
|
||||
self.answer_text = str(details.get('result'))
|
||||
|
||||
def execute(self, function_lib_id, input_field_list, **kwargs) -> NodeResult:
|
||||
function_lib = QuerySet(FunctionLib).filter(id=function_lib_id).first()
|
||||
valid_function(function_lib, self.flow_params_serializer.data.get('user_id'))
|
||||
if not function_lib.is_active:
|
||||
raise Exception(f'函数:{function_lib.name} 不可用')
|
||||
params = {field.get('name'): convert_value(field.get('name'), field.get('value'), field.get('type'),
|
||||
field.get('is_required'),
|
||||
field.get('source'), self)
|
||||
|
|
@ -127,14 +107,8 @@ class BaseFunctionLibNodeNode(IFunctionLibNode):
|
|||
), **field}
|
||||
for field in
|
||||
function_lib.input_field_list]}
|
||||
|
||||
self.context['params'] = params
|
||||
# 合并初始化参数
|
||||
if function_lib.init_params is not None:
|
||||
all_params = json.loads(rsa_long_decrypt(function_lib.init_params)) | params
|
||||
else:
|
||||
all_params = params
|
||||
result = function_executor.exec_code(function_lib.code, all_params)
|
||||
result = function_executor.exec_code(function_lib.code, params)
|
||||
return NodeResult({'result': result}, {}, _write_context=write_context)
|
||||
|
||||
def get_details(self, index: int, **kwargs):
|
||||
|
|
|
|||
|
|
@ -33,9 +33,9 @@ def write_context(step_variable: Dict, global_variable: Dict, node, workflow):
|
|||
|
||||
def valid_reference_value(_type, value, name):
|
||||
if _type == 'int':
|
||||
instance_type = int | float
|
||||
instance_type = int
|
||||
elif _type == 'float':
|
||||
instance_type = float | int
|
||||
instance_type = float
|
||||
elif _type == 'dict':
|
||||
instance_type = dict
|
||||
elif _type == 'array':
|
||||
|
|
@ -49,17 +49,13 @@ def valid_reference_value(_type, value, name):
|
|||
|
||||
|
||||
def convert_value(name: str, value, _type, is_required, source, node):
|
||||
if not is_required and (value is None or (isinstance(value, str) and len(value) == 0)):
|
||||
if not is_required and value is None:
|
||||
return None
|
||||
if source == 'reference':
|
||||
value = node.workflow_manage.get_reference_field(
|
||||
value[0],
|
||||
value[1:])
|
||||
valid_reference_value(_type, value, name)
|
||||
if _type == 'int':
|
||||
return int(value)
|
||||
if _type == 'float':
|
||||
return float(value)
|
||||
return value
|
||||
try:
|
||||
if _type == 'int':
|
||||
|
|
@ -84,8 +80,7 @@ def convert_value(name: str, value, _type, is_required, source, node):
|
|||
class BaseFunctionNodeNode(IFunctionNode):
|
||||
def save_context(self, details, workflow_manage):
|
||||
self.context['result'] = details.get('result')
|
||||
if self.node_params.get('is_result', False):
|
||||
self.answer_text = str(details.get('result'))
|
||||
self.answer_text = str(details.get('result'))
|
||||
|
||||
def execute(self, input_field_list, code, **kwargs) -> NodeResult:
|
||||
params = {field.get('name'): convert_value(field.get('name'), field.get('value'), field.get('type'),
|
||||
|
|
|
|||
|
|
@ -16,8 +16,7 @@ class BaseImageGenerateNode(IImageGenerateNode):
|
|||
def save_context(self, details, workflow_manage):
|
||||
self.context['answer'] = details.get('answer')
|
||||
self.context['question'] = details.get('question')
|
||||
if self.node_params.get('is_result', False):
|
||||
self.answer_text = details.get('answer')
|
||||
self.answer_text = details.get('answer')
|
||||
|
||||
def execute(self, model_id, prompt, negative_prompt, dialogue_number, dialogue_type, history_chat_record, chat_id,
|
||||
model_params_setting,
|
||||
|
|
@ -25,8 +24,7 @@ class BaseImageGenerateNode(IImageGenerateNode):
|
|||
**kwargs) -> NodeResult:
|
||||
print(model_params_setting)
|
||||
application = self.workflow_manage.work_flow_post_handler.chat_info.application
|
||||
tti_model = get_model_instance_by_model_user_id(model_id, self.flow_params_serializer.data.get('user_id'),
|
||||
**model_params_setting)
|
||||
tti_model = get_model_instance_by_model_user_id(model_id, self.flow_params_serializer.data.get('user_id'), **model_params_setting)
|
||||
history_message = self.get_history_message(history_chat_record, dialogue_number)
|
||||
self.context['history_message'] = history_message
|
||||
question = self.generate_prompt_question(prompt)
|
||||
|
|
@ -49,7 +47,7 @@ class BaseImageGenerateNode(IImageGenerateNode):
|
|||
file_url = FileSerializer(data={'file': file, 'meta': meta}).upload()
|
||||
file_urls.append(file_url)
|
||||
self.context['image_list'] = [{'file_id': path.split('/')[-1], 'url': path} for path in file_urls]
|
||||
answer = ' '.join([f"" for path in file_urls])
|
||||
answer = '\n'.join([f"" for path in file_urls])
|
||||
return NodeResult({'answer': answer, 'chat_model': tti_model, 'message_list': message_list,
|
||||
'image': [{'file_id': path.split('/')[-1], 'url': path} for path in file_urls],
|
||||
'history_message': history_message, 'question': question}, {})
|
||||
|
|
|
|||
|
|
@ -69,8 +69,7 @@ class BaseImageUnderstandNode(IImageUnderstandNode):
|
|||
def save_context(self, details, workflow_manage):
|
||||
self.context['answer'] = details.get('answer')
|
||||
self.context['question'] = details.get('question')
|
||||
if self.node_params.get('is_result', False):
|
||||
self.answer_text = details.get('answer')
|
||||
self.answer_text = details.get('answer')
|
||||
|
||||
def execute(self, model_id, system, prompt, dialogue_number, dialogue_type, history_chat_record, stream, chat_id,
|
||||
model_params_setting,
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
# coding=utf-8
|
||||
|
||||
from .impl import *
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
# coding=utf-8
|
||||
|
||||
from typing import Type
|
||||
|
||||
from rest_framework import serializers
|
||||
|
||||
from application.flow.i_step_node import INode, NodeResult
|
||||
from common.util.field_message import ErrMessage
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
|
||||
class McpNodeSerializer(serializers.Serializer):
|
||||
mcp_servers = serializers.JSONField(required=True,
|
||||
error_messages=ErrMessage.char(_("Mcp servers")))
|
||||
|
||||
mcp_server = serializers.CharField(required=True,
|
||||
error_messages=ErrMessage.char(_("Mcp server")))
|
||||
|
||||
mcp_tool = serializers.CharField(required=True, error_messages=ErrMessage.char(_("Mcp tool")))
|
||||
|
||||
tool_params = serializers.DictField(required=True,
|
||||
error_messages=ErrMessage.char(_("Tool parameters")))
|
||||
|
||||
|
||||
class IMcpNode(INode):
|
||||
type = 'mcp-node'
|
||||
|
||||
def get_node_params_serializer_class(self) -> Type[serializers.Serializer]:
|
||||
return McpNodeSerializer
|
||||
|
||||
def _run(self):
|
||||
return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data)
|
||||
|
||||
def execute(self, mcp_servers, mcp_server, mcp_tool, tool_params, **kwargs) -> NodeResult:
|
||||
pass
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
# coding=utf-8
|
||||
|
||||
from .base_mcp_node import BaseMcpNode
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
# coding=utf-8
|
||||
import asyncio
|
||||
import json
|
||||
from typing import List
|
||||
|
||||
from langchain_mcp_adapters.client import MultiServerMCPClient
|
||||
|
||||
from application.flow.i_step_node import NodeResult
|
||||
from application.flow.step_node.mcp_node.i_mcp_node import IMcpNode
|
||||
|
||||
|
||||
class BaseMcpNode(IMcpNode):
|
||||
def save_context(self, details, workflow_manage):
|
||||
self.context['result'] = details.get('result')
|
||||
self.context['tool_params'] = details.get('tool_params')
|
||||
self.context['mcp_tool'] = details.get('mcp_tool')
|
||||
if self.node_params.get('is_result', False):
|
||||
self.answer_text = details.get('result')
|
||||
|
||||
def execute(self, mcp_servers, mcp_server, mcp_tool, tool_params, **kwargs) -> NodeResult:
|
||||
servers = json.loads(mcp_servers)
|
||||
params = json.loads(json.dumps(tool_params))
|
||||
params = self.handle_variables(params)
|
||||
|
||||
async def call_tool(s, session, t, a):
|
||||
async with MultiServerMCPClient(s) as client:
|
||||
s = await client.sessions[session].call_tool(t, a)
|
||||
return s
|
||||
|
||||
res = asyncio.run(call_tool(servers, mcp_server, mcp_tool, params))
|
||||
return NodeResult(
|
||||
{'result': [content.text for content in res.content], 'tool_params': params, 'mcp_tool': mcp_tool}, {})
|
||||
|
||||
def handle_variables(self, tool_params):
|
||||
# 处理参数中的变量
|
||||
for k, v in tool_params.items():
|
||||
if type(v) == str:
|
||||
tool_params[k] = self.workflow_manage.generate_prompt(tool_params[k])
|
||||
if type(v) == dict:
|
||||
self.handle_variables(v)
|
||||
if (type(v) == list) and (type(v[0]) == str):
|
||||
tool_params[k] = self.get_reference_content(v)
|
||||
return tool_params
|
||||
|
||||
def get_reference_content(self, fields: List[str]):
|
||||
return str(self.workflow_manage.get_reference_field(
|
||||
fields[0],
|
||||
fields[1:]))
|
||||
|
||||
def get_details(self, index: int, **kwargs):
|
||||
return {
|
||||
'name': self.node.properties.get('stepName'),
|
||||
"index": index,
|
||||
'run_time': self.context.get('run_time'),
|
||||
'status': self.status,
|
||||
'err_message': self.err_message,
|
||||
'type': self.node.type,
|
||||
'mcp_tool': self.context.get('mcp_tool'),
|
||||
'tool_params': self.context.get('tool_params'),
|
||||
'result': self.context.get('result'),
|
||||
}
|
||||
|
|
@ -80,8 +80,7 @@ class BaseQuestionNode(IQuestionNode):
|
|||
self.context['answer'] = details.get('answer')
|
||||
self.context['message_tokens'] = details.get('message_tokens')
|
||||
self.context['answer_tokens'] = details.get('answer_tokens')
|
||||
if self.node_params.get('is_result', False):
|
||||
self.answer_text = details.get('answer')
|
||||
self.answer_text = details.get('answer')
|
||||
|
||||
def execute(self, model_id, system, prompt, dialogue_number, history_chat_record, stream, chat_id, chat_record_id,
|
||||
model_params_setting=None,
|
||||
|
|
|
|||
|
|
@ -23,14 +23,9 @@ def merge_reranker_list(reranker_list, result=None):
|
|||
merge_reranker_list(document, result)
|
||||
elif isinstance(document, dict):
|
||||
content = document.get('title', '') + document.get('content', '')
|
||||
title = document.get("title")
|
||||
dataset_name = document.get("dataset_name")
|
||||
document_name = document.get('document_name')
|
||||
result.append(
|
||||
Document(page_content=str(document) if len(content) == 0 else content,
|
||||
metadata={'title': title, 'dataset_name': dataset_name, 'document_name': document_name}))
|
||||
result.append(str(document) if len(content) == 0 else content)
|
||||
else:
|
||||
result.append(Document(page_content=str(document), metadata={}))
|
||||
result.append(str(document))
|
||||
return result
|
||||
|
||||
|
||||
|
|
@ -48,21 +43,6 @@ def filter_result(document_list: List[Document], max_paragraph_char_number, top_
|
|||
return result
|
||||
|
||||
|
||||
def reset_result_list(result_list: List[Document], document_list: List[Document]):
|
||||
r = []
|
||||
document_list = document_list.copy()
|
||||
for result in result_list:
|
||||
filter_result_list = [document for document in document_list if document.page_content == result.page_content]
|
||||
if len(filter_result_list) > 0:
|
||||
item = filter_result_list[0]
|
||||
document_list.remove(item)
|
||||
r.append(Document(page_content=item.page_content,
|
||||
metadata={**item.metadata, 'relevance_score': result.metadata.get('relevance_score')}))
|
||||
else:
|
||||
r.append(result)
|
||||
return r
|
||||
|
||||
|
||||
class BaseRerankerNode(IRerankerNode):
|
||||
def save_context(self, details, workflow_manage):
|
||||
self.context['document_list'] = details.get('document_list', [])
|
||||
|
|
@ -75,18 +55,16 @@ class BaseRerankerNode(IRerankerNode):
|
|||
**kwargs) -> NodeResult:
|
||||
documents = merge_reranker_list(reranker_list)
|
||||
top_n = reranker_setting.get('top_n', 3)
|
||||
self.context['document_list'] = [{'page_content': document.page_content, 'metadata': document.metadata} for
|
||||
document in documents]
|
||||
self.context['document_list'] = documents
|
||||
self.context['question'] = question
|
||||
reranker_model = get_model_instance_by_model_user_id(reranker_model_id,
|
||||
self.flow_params_serializer.data.get('user_id'),
|
||||
top_n=top_n)
|
||||
result = reranker_model.compress_documents(
|
||||
documents,
|
||||
[Document(page_content=document) for document in documents if document is not None and len(document) > 0],
|
||||
question)
|
||||
similarity = reranker_setting.get('similarity', 0.6)
|
||||
max_paragraph_char_number = reranker_setting.get('max_paragraph_char_number', 5000)
|
||||
result = reset_result_list(result, documents)
|
||||
r = filter_result(result, max_paragraph_char_number, top_n, similarity)
|
||||
return NodeResult({'result_list': r, 'result': ''.join([item.get('page_content') for item in r])}, {})
|
||||
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ class DatasetSettingSerializer(serializers.Serializer):
|
|||
error_messages=ErrMessage.float(_('similarity')))
|
||||
search_mode = serializers.CharField(required=True, validators=[
|
||||
validators.RegexValidator(regex=re.compile("^embedding|keywords|blend$"),
|
||||
message=_("The type only supports embedding|keywords|blend"), code=500)
|
||||
message=_("The type only supports register|reset_password"), code=500)
|
||||
], error_messages=ErrMessage.char(_("Retrieval Mode")))
|
||||
max_paragraph_char_number = serializers.IntegerField(required=True,
|
||||
error_messages=ErrMessage.float(_("Maximum number of words in a quoted segment")))
|
||||
|
|
@ -66,7 +66,7 @@ class ISearchDatasetStepNode(INode):
|
|||
if self.flow_params_serializer.data.get('re_chat', False):
|
||||
history_chat_record = self.flow_params_serializer.data.get('history_chat_record', [])
|
||||
paragraph_id_list = [p.get('id') for p in flat_map(
|
||||
[get_paragraph_list(chat_record, self.runtime_node_id) for chat_record in history_chat_record if
|
||||
[get_paragraph_list(chat_record, self.node.id) for chat_record in history_chat_record if
|
||||
chat_record.problem_text == question])]
|
||||
exclude_paragraph_id_list = list(set(paragraph_id_list))
|
||||
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ class BaseSearchDatasetNode(ISearchDatasetStepNode):
|
|||
'is_hit_handling_method_list': [row for row in result if row.get('is_hit_handling_method')],
|
||||
'data': '\n'.join(
|
||||
[f"{reset_title(paragraph.get('title', ''))}{paragraph.get('content')}" for paragraph in
|
||||
result])[0:dataset_setting.get('max_paragraph_char_number', 5000)],
|
||||
paragraph_list])[0:dataset_setting.get('max_paragraph_char_number', 5000)],
|
||||
'directly_return': '\n'.join(
|
||||
[paragraph.get('content') for paragraph in
|
||||
result if
|
||||
|
|
|
|||
|
|
@ -18,8 +18,7 @@ class BaseSpeechToTextNode(ISpeechToTextNode):
|
|||
|
||||
def save_context(self, details, workflow_manage):
|
||||
self.context['answer'] = details.get('answer')
|
||||
if self.node_params.get('is_result', False):
|
||||
self.answer_text = details.get('answer')
|
||||
self.answer_text = details.get('answer')
|
||||
|
||||
def execute(self, stt_model_id, chat_id, audio, **kwargs) -> NodeResult:
|
||||
stt_model = get_model_instance_by_model_user_id(stt_model_id, self.flow_params_serializer.data.get('user_id'))
|
||||
|
|
|
|||
|
|
@ -40,13 +40,10 @@ class BaseStartStepNode(IStarNode):
|
|||
self.context['document'] = details.get('document_list')
|
||||
self.context['image'] = details.get('image_list')
|
||||
self.context['audio'] = details.get('audio_list')
|
||||
self.context['other'] = details.get('other_list')
|
||||
self.status = details.get('status')
|
||||
self.err_message = details.get('err_message')
|
||||
for key, value in workflow_variable.items():
|
||||
workflow_manage.context[key] = value
|
||||
for item in details.get('global_fields', []):
|
||||
workflow_manage.context[item.get('key')] = item.get('value')
|
||||
|
||||
def get_node_params_serializer_class(self) -> Type[serializers.Serializer]:
|
||||
pass
|
||||
|
|
@ -62,8 +59,7 @@ class BaseStartStepNode(IStarNode):
|
|||
'question': question,
|
||||
'image': self.workflow_manage.image_list,
|
||||
'document': self.workflow_manage.document_list,
|
||||
'audio': self.workflow_manage.audio_list,
|
||||
'other': self.workflow_manage.other_list,
|
||||
'audio': self.workflow_manage.audio_list
|
||||
}
|
||||
return NodeResult(node_variable, workflow_variable)
|
||||
|
||||
|
|
@ -87,6 +83,5 @@ class BaseStartStepNode(IStarNode):
|
|||
'image_list': self.context.get('image'),
|
||||
'document_list': self.context.get('document'),
|
||||
'audio_list': self.context.get('audio'),
|
||||
'other_list': self.context.get('other'),
|
||||
'global_fields': global_fields
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,8 +37,7 @@ def bytes_to_uploaded_file(file_bytes, file_name="generated_audio.mp3"):
|
|||
class BaseTextToSpeechNode(ITextToSpeechNode):
|
||||
def save_context(self, details, workflow_manage):
|
||||
self.context['answer'] = details.get('answer')
|
||||
if self.node_params.get('is_result', False):
|
||||
self.answer_text = details.get('answer')
|
||||
self.answer_text = details.get('answer')
|
||||
|
||||
def execute(self, tts_model_id, chat_id,
|
||||
content, model_params_setting=None,
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
# coding=utf-8
|
||||
|
||||
from .impl import *
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
# coding=utf-8
|
||||
|
||||
from typing import Type
|
||||
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
from rest_framework import serializers
|
||||
|
||||
from application.flow.i_step_node import INode, NodeResult
|
||||
from common.util.field_message import ErrMessage
|
||||
|
||||
|
||||
class VariableAssignNodeParamsSerializer(serializers.Serializer):
|
||||
variable_list = serializers.ListField(required=True,
|
||||
error_messages=ErrMessage.list(_("Reference Field")))
|
||||
|
||||
|
||||
class IVariableAssignNode(INode):
|
||||
type = 'variable-assign-node'
|
||||
|
||||
def get_node_params_serializer_class(self) -> Type[serializers.Serializer]:
|
||||
return VariableAssignNodeParamsSerializer
|
||||
|
||||
def _run(self):
|
||||
return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data)
|
||||
|
||||
def execute(self, variable_list, stream, **kwargs) -> NodeResult:
|
||||
pass
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
# coding=utf-8
|
||||
"""
|
||||
@project: maxkb
|
||||
@Author：虎
|
||||
@file： __init__.py
|
||||
@date：2024/6/11 17:49
|
||||
@desc:
|
||||
"""
|
||||
from .base_variable_assign_node import *
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
# coding=utf-8
|
||||
import json
|
||||
from typing import List
|
||||
|
||||
from application.flow.i_step_node import NodeResult
|
||||
from application.flow.step_node.variable_assign_node.i_variable_assign_node import IVariableAssignNode
|
||||
|
||||
|
||||
class BaseVariableAssignNode(IVariableAssignNode):
|
||||
def save_context(self, details, workflow_manage):
|
||||
self.context['variable_list'] = details.get('variable_list')
|
||||
self.context['result_list'] = details.get('result_list')
|
||||
|
||||
def execute(self, variable_list, stream, **kwargs) -> NodeResult:
|
||||
#
|
||||
result_list = []
|
||||
for variable in variable_list:
|
||||
if 'fields' not in variable:
|
||||
continue
|
||||
if 'global' == variable['fields'][0]:
|
||||
result = {
|
||||
'name': variable['name'],
|
||||
'input_value': self.get_reference_content(variable['fields']),
|
||||
}
|
||||
if variable['source'] == 'custom':
|
||||
if variable['type'] == 'json':
|
||||
if isinstance(variable['value'], dict) or isinstance(variable['value'], list):
|
||||
val = variable['value']
|
||||
else:
|
||||
val = json.loads(variable['value'])
|
||||
self.workflow_manage.context[variable['fields'][1]] = val
|
||||
result['output_value'] = variable['value'] = val
|
||||
elif variable['type'] == 'string':
|
||||
# 变量解析 例如：{{global.xxx}}
|
||||
val = self.workflow_manage.generate_prompt(variable['value'])
|
||||
self.workflow_manage.context[variable['fields'][1]] = val
|
||||
result['output_value'] = val
|
||||
else:
|
||||
val = variable['value']
|
||||
self.workflow_manage.context[variable['fields'][1]] = val
|
||||
result['output_value'] = val
|
||||
else:
|
||||
reference = self.get_reference_content(variable['reference'])
|
||||
self.workflow_manage.context[variable['fields'][1]] = reference
|
||||
result['output_value'] = reference
|
||||
result_list.append(result)
|
||||
|
||||
return NodeResult({'variable_list': variable_list, 'result_list': result_list}, {})
|
||||
|
||||
def get_reference_content(self, fields: List[str]):
|
||||
return str(self.workflow_manage.get_reference_field(
|
||||
fields[0],
|
||||
fields[1:]))
|
||||
|
||||
def get_details(self, index: int, **kwargs):
|
||||
return {
|
||||
'name': self.node.properties.get('stepName'),
|
||||
"index": index,
|
||||
'run_time': self.context.get('run_time'),
|
||||
'type': self.node.type,
|
||||
'variable_list': self.context.get('variable_list'),
|
||||
'result_list': self.context.get('result_list'),
|
||||
'status': self.status,
|
||||
'err_message': self.err_message
|
||||
}
|
||||
|
|
@ -238,7 +238,6 @@ class WorkflowManage:
|
|||
base_to_response: BaseToResponse = SystemToResponse(), form_data=None, image_list=None,
|
||||
document_list=None,
|
||||
audio_list=None,
|
||||
other_list=None,
|
||||
start_node_id=None,
|
||||
start_node_data=None, chat_record=None, child_node=None):
|
||||
if form_data is None:
|
||||
|
|
@ -249,15 +248,12 @@ class WorkflowManage:
|
|||
document_list = []
|
||||
if audio_list is None:
|
||||
audio_list = []
|
||||
if other_list is None:
|
||||
other_list = []
|
||||
self.start_node_id = start_node_id
|
||||
self.start_node = None
|
||||
self.form_data = form_data
|
||||
self.image_list = image_list
|
||||
self.document_list = document_list
|
||||
self.audio_list = audio_list
|
||||
self.other_list = other_list
|
||||
self.params = params
|
||||
self.flow = flow
|
||||
self.context = {}
|
||||
|
|
@ -273,36 +269,11 @@ class WorkflowManage:
|
|||
self.child_node = child_node
|
||||
self.future_list = []
|
||||
self.lock = threading.Lock()
|
||||
self.field_list = []
|
||||
self.global_field_list = []
|
||||
self.init_fields()
|
||||
if start_node_id is not None:
|
||||
self.load_node(chat_record, start_node_id, start_node_data)
|
||||
else:
|
||||
self.node_context = []
|
||||
|
||||
def init_fields(self):
|
||||
field_list = []
|
||||
global_field_list = []
|
||||
for node in self.flow.nodes:
|
||||
properties = node.properties
|
||||
node_name = properties.get('stepName')
|
||||
node_id = node.id
|
||||
node_config = properties.get('config')
|
||||
if node_config is not None:
|
||||
fields = node_config.get('fields')
|
||||
if fields is not None:
|
||||
for field in fields:
|
||||
field_list.append({**field, 'node_id': node_id, 'node_name': node_name})
|
||||
global_fields = node_config.get('globalFields')
|
||||
if global_fields is not None:
|
||||
for global_field in global_fields:
|
||||
global_field_list.append({**global_field, 'node_id': node_id, 'node_name': node_name})
|
||||
field_list.sort(key=lambda f: len(f.get('node_name')), reverse=True)
|
||||
global_field_list.sort(key=lambda f: len(f.get('node_name')), reverse=True)
|
||||
self.field_list = field_list
|
||||
self.global_field_list = global_field_list
|
||||
|
||||
def append_answer(self, content):
|
||||
self.answer += content
|
||||
self.answer_list[-1] += content
|
||||
|
|
@ -366,15 +337,13 @@ class WorkflowManage:
|
|||
answer_text = '\n\n'.join(
|
||||
'\n\n'.join([a.get('content') for a in answer]) for answer in
|
||||
answer_text_list)
|
||||
answer_list = reduce(lambda pre, _n: [*pre, *_n], answer_text_list, [])
|
||||
self.work_flow_post_handler.handler(self.params['chat_id'], self.params['chat_record_id'],
|
||||
answer_text,
|
||||
self)
|
||||
return self.base_to_response.to_block_response(self.params['chat_id'],
|
||||
self.params['chat_record_id'], answer_text, True
|
||||
, message_tokens, answer_tokens,
|
||||
_status=status.HTTP_200_OK if self.status == 200 else status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
other_params={'answer_list': answer_list})
|
||||
_status=status.HTTP_200_OK if self.status == 200 else status.HTTP_500_INTERNAL_SERVER_ERROR)
|
||||
|
||||
def run_stream(self, current_node, node_result_future, language='zh'):
|
||||
"""
|
||||
|
|
@ -413,8 +382,6 @@ class WorkflowManage:
|
|||
break
|
||||
yield chunk
|
||||
finally:
|
||||
while self.is_run():
|
||||
pass
|
||||
details = self.get_runtime_details()
|
||||
message_tokens = sum([row.get('message_tokens') for row in details.values() if
|
||||
'message_tokens' in row and row.get('message_tokens') is not None])
|
||||
|
|
@ -768,15 +735,23 @@ class WorkflowManage:
|
|||
|
||||
def reset_prompt(self, prompt: str):
|
||||
placeholder = "{}"
|
||||
for field in self.field_list:
|
||||
globeLabel = f"{field.get('node_name')}.{field.get('value')}"
|
||||
globeValue = f"context.get('{field.get('node_id')}',{placeholder}).get('{field.get('value', '')}','')"
|
||||
prompt = prompt.replace(globeLabel, globeValue)
|
||||
for field in self.global_field_list:
|
||||
globeLabel = f"全局变量.{field.get('value')}"
|
||||
globeLabelNew = f"global.{field.get('value')}"
|
||||
globeValue = f"context.get('global').get('{field.get('value', '')}','')"
|
||||
prompt = prompt.replace(globeLabel, globeValue).replace(globeLabelNew, globeValue)
|
||||
for node in self.flow.nodes:
|
||||
properties = node.properties
|
||||
node_config = properties.get('config')
|
||||
if node_config is not None:
|
||||
fields = node_config.get('fields')
|
||||
if fields is not None:
|
||||
for field in fields:
|
||||
globeLabel = f"{properties.get('stepName')}.{field.get('value')}"
|
||||
globeValue = f"context.get('{node.id}',{placeholder}).get('{field.get('value', '')}','')"
|
||||
prompt = prompt.replace(globeLabel, globeValue)
|
||||
global_fields = node_config.get('globalFields')
|
||||
if global_fields is not None:
|
||||
for field in global_fields:
|
||||
globeLabel = f"全局变量.{field.get('value')}"
|
||||
globeLabelNew = f"global.{field.get('value')}"
|
||||
globeValue = f"context.get('global').get('{field.get('value', '')}','')"
|
||||
prompt = prompt.replace(globeLabel, globeValue).replace(globeLabelNew, globeValue)
|
||||
return prompt
|
||||
|
||||
def generate_prompt(self, prompt: str):
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
# Generated by Django 4.2.13 on 2024-07-15 15:52
|
||||
|
||||
import application.models.application
|
||||
from django.db import migrations, models
|
||||
|
||||
import common.encoder.encoder
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('application', '0009_application_type_application_work_flow_and_more'),
|
||||
]
|
||||
|
|
@ -14,6 +14,6 @@ class Migration(migrations.Migration):
|
|||
migrations.AlterField(
|
||||
model_name='chatrecord',
|
||||
name='details',
|
||||
field=models.JSONField(default=dict, encoder=common.encoder.encoder.SystemEncoder, verbose_name='对话详情'),
|
||||
field=models.JSONField(default=dict, encoder=application.models.application.DateEncoder, verbose_name='对话详情'),
|
||||
),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,20 +0,0 @@
|
|||
# Generated by Django 4.2.18 on 2025-03-18 06:05
|
||||
|
||||
import application.models.application
|
||||
import common.encoder.encoder
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('application', '0025_alter_application_prologue'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='chat',
|
||||
name='asker',
|
||||
field=models.JSONField(default=application.models.application.default_asker, encoder=common.encoder.encoder.SystemEncoder, verbose_name='访问者'),
|
||||
),
|
||||
]
|
||||
|
|
@ -6,13 +6,14 @@
|
|||
@date：2023/9/25 14:24
|
||||
@desc:
|
||||
"""
|
||||
import datetime
|
||||
import json
|
||||
import uuid
|
||||
|
||||
from django.contrib.postgres.fields import ArrayField
|
||||
from django.db import models
|
||||
from langchain.schema import HumanMessage, AIMessage
|
||||
from django.utils.translation import gettext as _
|
||||
from common.encoder.encoder import SystemEncoder
|
||||
|
||||
from common.mixins.app_model_mixin import AppModelMixin
|
||||
from dataset.models.data_set import DataSet
|
||||
from setting.models.model_management import Model
|
||||
|
|
@ -115,15 +116,10 @@ class ApplicationDatasetMapping(AppModelMixin):
|
|||
db_table = "application_dataset_mapping"
|
||||
|
||||
|
||||
def default_asker():
|
||||
return {'user_name': '游客'}
|
||||
|
||||
|
||||
class Chat(AppModelMixin):
|
||||
id = models.UUIDField(primary_key=True, max_length=128, default=uuid.uuid1, editable=False, verbose_name="主键id")
|
||||
application = models.ForeignKey(Application, on_delete=models.CASCADE)
|
||||
abstract = models.CharField(max_length=1024, verbose_name="摘要")
|
||||
asker = models.JSONField(verbose_name="访问者", default=default_asker, encoder=SystemEncoder)
|
||||
client_id = models.UUIDField(verbose_name="客户端id", default=None, null=True)
|
||||
is_deleted = models.BooleanField(verbose_name="", default=False)
|
||||
|
||||
|
|
@ -138,6 +134,16 @@ class VoteChoices(models.TextChoices):
|
|||
TRAMPLE = 1, '反对'
|
||||
|
||||
|
||||
class DateEncoder(json.JSONEncoder):
|
||||
def default(self, obj):
|
||||
if isinstance(obj, uuid.UUID):
|
||||
return str(obj)
|
||||
if isinstance(obj, datetime.datetime):
|
||||
return obj.strftime("%Y-%m-%d %H:%M:%S")
|
||||
else:
|
||||
return json.JSONEncoder.default(self, obj)
|
||||
|
||||
|
||||
class ChatRecord(AppModelMixin):
|
||||
"""
|
||||
对话日志 详情
|
||||
|
|
@ -154,7 +160,7 @@ class ChatRecord(AppModelMixin):
|
|||
message_tokens = models.IntegerField(verbose_name="请求token数量", default=0)
|
||||
answer_tokens = models.IntegerField(verbose_name="响应token数量", default=0)
|
||||
const = models.IntegerField(verbose_name="总费用", default=0)
|
||||
details = models.JSONField(verbose_name="对话详情", default=dict, encoder=SystemEncoder)
|
||||
details = models.JSONField(verbose_name="对话详情", default=dict, encoder=DateEncoder)
|
||||
improve_paragraph_id_list = ArrayField(verbose_name="改进标注列表",
|
||||
base_field=models.UUIDField(max_length=128, blank=True)
|
||||
, default=list)
|
||||
|
|
@ -167,11 +173,7 @@ class ChatRecord(AppModelMixin):
|
|||
return HumanMessage(content=self.problem_text)
|
||||
|
||||
def get_ai_message(self):
|
||||
answer_text = self.answer_text
|
||||
if answer_text is None or len(str(answer_text).strip()) == 0:
|
||||
answer_text = _(
|
||||
'Sorry, no relevant content was found. Please re-describe your problem or provide more information. ')
|
||||
return AIMessage(content=answer_text)
|
||||
return AIMessage(content=self.answer_text)
|
||||
|
||||
def get_node_details_runtime_node_id(self, runtime_node_id):
|
||||
return self.details.get(runtime_node_id, None)
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@
|
|||
@date：2023/11/7 10:02
|
||||
@desc:
|
||||
"""
|
||||
import asyncio
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
|
|
@ -24,8 +23,6 @@ from django.db.models import QuerySet
|
|||
from django.db.models.expressions import RawSQL
|
||||
from django.http import HttpResponse
|
||||
from django.template import Template, Context
|
||||
from langchain_mcp_adapters.client import MultiServerMCPClient
|
||||
from mcp.client.sse import sse_client
|
||||
from rest_framework import serializers, status
|
||||
from rest_framework.utils.formatting import lazy_format
|
||||
|
||||
|
|
@ -42,13 +39,13 @@ from common.exception.app_exception import AppApiException, NotFound404, AppUnau
|
|||
from common.field.common import UploadedImageField, UploadedFileField
|
||||
from common.models.db_model_manage import DBModelManage
|
||||
from common.response import result
|
||||
from common.util.common import valid_license, password_encrypt, restricted_loads
|
||||
from common.util.common import valid_license, password_encrypt
|
||||
from common.util.field_message import ErrMessage
|
||||
from common.util.file_util import get_file_content
|
||||
from dataset.models import DataSet, Document, Image
|
||||
from dataset.serializers.common_serializers import list_paragraph, get_embedding_model_by_dataset_id_list
|
||||
from embedding.models import SearchMode
|
||||
from function_lib.models.function import FunctionLib, PermissionType, FunctionType
|
||||
from function_lib.models.function import FunctionLib, PermissionType
|
||||
from function_lib.serializers.function_lib_serializer import FunctionLibSerializer, FunctionLibModelSerializer
|
||||
from setting.models import AuthOperate, TeamMemberPermission
|
||||
from setting.models.model_management import Model
|
||||
|
|
@ -63,7 +60,6 @@ chat_cache = cache.caches['chat_cache']
|
|||
|
||||
|
||||
class MKInstance:
|
||||
|
||||
def __init__(self, application: dict, function_lib_list: List[dict], version: str):
|
||||
self.application = application
|
||||
self.function_lib_list = function_lib_list
|
||||
|
|
@ -121,7 +117,7 @@ def valid_model_params_setting(model_id, model_params_setting):
|
|||
|
||||
|
||||
class DatasetSettingSerializer(serializers.Serializer):
|
||||
top_n = serializers.FloatField(required=True, max_value=10000, min_value=1,
|
||||
top_n = serializers.FloatField(required=True, max_value=100, min_value=1,
|
||||
error_messages=ErrMessage.float(_("Reference segment number")))
|
||||
similarity = serializers.FloatField(required=True, max_value=1, min_value=0,
|
||||
error_messages=ErrMessage.float(_("Acquaintance")))
|
||||
|
|
@ -130,7 +126,7 @@ class DatasetSettingSerializer(serializers.Serializer):
|
|||
_("Maximum number of quoted characters")))
|
||||
search_mode = serializers.CharField(required=True, validators=[
|
||||
validators.RegexValidator(regex=re.compile("^embedding|keywords|blend$"),
|
||||
message=_("The type only supports embedding|keywords|blend"), code=500)
|
||||
message=_("The type only supports register|reset_password"), code=500)
|
||||
], error_messages=ErrMessage.char(_("Retrieval Mode")))
|
||||
|
||||
no_references_setting = NoReferencesSetting(required=True,
|
||||
|
|
@ -146,14 +142,10 @@ class ModelSettingSerializer(serializers.Serializer):
|
|||
error_messages=ErrMessage.char(_("No citation segmentation prompt")))
|
||||
reasoning_content_enable = serializers.BooleanField(required=False,
|
||||
error_messages=ErrMessage.char(_("Thinking process switch")))
|
||||
reasoning_content_start = serializers.CharField(required=False, allow_null=True, default="<think>",
|
||||
allow_blank=True, max_length=256,
|
||||
trim_whitespace=False,
|
||||
reasoning_content_start = serializers.CharField(required=False, allow_null=True, allow_blank=True, max_length=256,
|
||||
error_messages=ErrMessage.char(
|
||||
_("The thinking process begins to mark")))
|
||||
reasoning_content_end = serializers.CharField(required=False, allow_null=True, allow_blank=True, default="</think>",
|
||||
max_length=256,
|
||||
trim_whitespace=False,
|
||||
reasoning_content_end = serializers.CharField(required=False, allow_null=True, allow_blank=True, max_length=256,
|
||||
error_messages=ErrMessage.char(_("End of thinking process marker")))
|
||||
|
||||
|
||||
|
|
@ -164,7 +156,7 @@ class ApplicationWorkflowSerializer(serializers.Serializer):
|
|||
max_length=256, min_length=1,
|
||||
error_messages=ErrMessage.char(_("Application Description")))
|
||||
work_flow = serializers.DictField(required=False, error_messages=ErrMessage.dict(_("Workflow Objects")))
|
||||
prologue = serializers.CharField(required=False, allow_null=True, allow_blank=True, max_length=102400,
|
||||
prologue = serializers.CharField(required=False, allow_null=True, allow_blank=True, max_length=4096,
|
||||
error_messages=ErrMessage.char(_("Opening remarks")))
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -227,7 +219,7 @@ class ApplicationSerializer(serializers.Serializer):
|
|||
min_value=0,
|
||||
max_value=1024,
|
||||
error_messages=ErrMessage.integer(_("Historical chat records")))
|
||||
prologue = serializers.CharField(required=False, allow_null=True, allow_blank=True, max_length=102400,
|
||||
prologue = serializers.CharField(required=False, allow_null=True, allow_blank=True, max_length=4096,
|
||||
error_messages=ErrMessage.char(_("Opening remarks")))
|
||||
dataset_id_list = serializers.ListSerializer(required=False, child=serializers.UUIDField(required=True),
|
||||
allow_null=True,
|
||||
|
|
@ -334,8 +326,7 @@ class ApplicationSerializer(serializers.Serializer):
|
|||
for field in input_field_list:
|
||||
if field['assignment_method'] == 'api_input' and field['variable'] in params:
|
||||
query += f"&{field['variable']}={params[field['variable']]}"
|
||||
if 'asker' in params:
|
||||
query += f"&asker={params.get('asker')}"
|
||||
|
||||
return query
|
||||
|
||||
class AccessTokenSerializer(serializers.Serializer):
|
||||
|
|
@ -495,7 +486,7 @@ class ApplicationSerializer(serializers.Serializer):
|
|||
min_value=0,
|
||||
max_value=1024,
|
||||
error_messages=ErrMessage.integer(_("Historical chat records")))
|
||||
prologue = serializers.CharField(required=False, allow_null=True, allow_blank=True, max_length=102400,
|
||||
prologue = serializers.CharField(required=False, allow_null=True, allow_blank=True, max_length=4096,
|
||||
error_messages=ErrMessage.char(_("Opening remarks")))
|
||||
dataset_id_list = serializers.ListSerializer(required=False, child=serializers.UUIDField(required=True),
|
||||
error_messages=ErrMessage.list(_("Related Knowledge Base"))
|
||||
|
|
@ -588,13 +579,13 @@ class ApplicationSerializer(serializers.Serializer):
|
|||
id = serializers.CharField(required=True, error_messages=ErrMessage.uuid(_("Application ID")))
|
||||
user_id = serializers.UUIDField(required=False, error_messages=ErrMessage.uuid(_("User ID")))
|
||||
query_text = serializers.CharField(required=True, error_messages=ErrMessage.char(_("Query text")))
|
||||
top_number = serializers.IntegerField(required=True, max_value=10000, min_value=1,
|
||||
top_number = serializers.IntegerField(required=True, max_value=100, min_value=1,
|
||||
error_messages=ErrMessage.integer(_("topN")))
|
||||
similarity = serializers.FloatField(required=True, max_value=2, min_value=0,
|
||||
error_messages=ErrMessage.float(_("Relevance")))
|
||||
search_mode = serializers.CharField(required=True, validators=[
|
||||
validators.RegexValidator(regex=re.compile("^embedding|keywords|blend$"),
|
||||
message=_("The type only supports embedding|keywords|blend"), code=500)
|
||||
message=_("The type only supports register|reset_password"), code=500)
|
||||
], error_messages=ErrMessage.char(_("Retrieval Mode")))
|
||||
|
||||
def is_valid(self, *, raise_exception=False):
|
||||
|
|
@ -734,7 +725,7 @@ class ApplicationSerializer(serializers.Serializer):
|
|||
user_id = self.data.get('user_id')
|
||||
mk_instance_bytes = self.data.get('file').read()
|
||||
try:
|
||||
mk_instance = restricted_loads(mk_instance_bytes)
|
||||
mk_instance = pickle.loads(mk_instance_bytes)
|
||||
except Exception as e:
|
||||
raise AppApiException(1001, _("Unsupported file format"))
|
||||
application = mk_instance.application
|
||||
|
|
@ -817,10 +808,8 @@ class ApplicationSerializer(serializers.Serializer):
|
|||
if with_valid:
|
||||
self.is_valid(raise_exception=True)
|
||||
application = QuerySet(Application).filter(id=self.data.get("application_id")).first()
|
||||
return FunctionLibSerializer.Query(
|
||||
data={'user_id': application.user_id, 'is_active': True,
|
||||
'function_type': FunctionType.PUBLIC}
|
||||
).list(with_valid=True)
|
||||
return FunctionLibSerializer.Query(data={'user_id': application.user_id, 'is_active': True}).list(
|
||||
with_valid=True)
|
||||
|
||||
def get_function_lib(self, function_lib_id, with_valid=True):
|
||||
if with_valid:
|
||||
|
|
@ -990,7 +979,6 @@ class ApplicationSerializer(serializers.Serializer):
|
|||
'draggable': application_setting.draggable,
|
||||
'show_guide': application_setting.show_guide,
|
||||
'avatar': application_setting.avatar,
|
||||
'show_avatar': application_setting.show_avatar,
|
||||
'float_icon': application_setting.float_icon,
|
||||
'authentication': application_setting.authentication,
|
||||
'authentication_type': application_setting.authentication_value.get(
|
||||
|
|
@ -999,7 +987,6 @@ class ApplicationSerializer(serializers.Serializer):
|
|||
'disclaimer_value': application_setting.disclaimer_value,
|
||||
'custom_theme': application_setting.custom_theme,
|
||||
'user_avatar': application_setting.user_avatar,
|
||||
'show_user_avatar': application_setting.show_user_avatar,
|
||||
'float_location': application_setting.float_location}
|
||||
return ApplicationSerializer.Query.reset_application(
|
||||
{**ApplicationSerializer.ApplicationModel(application).data,
|
||||
|
|
@ -1012,8 +999,7 @@ class ApplicationSerializer(serializers.Serializer):
|
|||
'stt_autosend': application.stt_autosend,
|
||||
'file_upload_enable': application.file_upload_enable,
|
||||
'file_upload_setting': application.file_upload_setting,
|
||||
'work_flow': {'nodes': [node for node in ((application.work_flow or {}).get('nodes', []) or []) if
|
||||
node.get('id') == 'base-node']},
|
||||
'work_flow': application.work_flow,
|
||||
'show_source': application_access_token.show_source,
|
||||
'language': application_access_token.language,
|
||||
**application_setting_dict})
|
||||
|
|
@ -1074,7 +1060,6 @@ class ApplicationSerializer(serializers.Serializer):
|
|||
for update_key in update_keys:
|
||||
if update_key in instance and instance.get(update_key) is not None:
|
||||
application.__setattr__(update_key, instance.get(update_key))
|
||||
print(application.name)
|
||||
application.save()
|
||||
|
||||
if 'dataset_id_list' in instance:
|
||||
|
|
@ -1093,7 +1078,6 @@ class ApplicationSerializer(serializers.Serializer):
|
|||
chat_cache.clear_by_application_id(application_id)
|
||||
application_access_token = QuerySet(ApplicationAccessToken).filter(application_id=application_id).first()
|
||||
# 更新缓存数据
|
||||
print(application.name)
|
||||
get_application_access_token(application_access_token.access_token, False)
|
||||
return self.one(with_valid=False)
|
||||
|
||||
|
|
@ -1146,8 +1130,6 @@ class ApplicationSerializer(serializers.Serializer):
|
|||
instance['file_upload_enable'] = node_data['file_upload_enable']
|
||||
if 'file_upload_setting' in node_data:
|
||||
instance['file_upload_setting'] = node_data['file_upload_setting']
|
||||
if 'name' in node_data:
|
||||
instance['name'] = node_data['name']
|
||||
break
|
||||
|
||||
def speech_to_text(self, file, with_valid=True):
|
||||
|
|
@ -1217,9 +1199,7 @@ class ApplicationSerializer(serializers.Serializer):
|
|||
self.is_valid(raise_exception=True)
|
||||
if with_valid:
|
||||
self.is_valid()
|
||||
embed_application = QuerySet(Application).filter(id=app_id).first()
|
||||
if embed_application is None:
|
||||
raise AppApiException(500, _('Application does not exist'))
|
||||
embed_application = QuerySet(Application).get(id=app_id)
|
||||
if embed_application.type == ApplicationTypeChoices.WORK_FLOW:
|
||||
work_flow_version = QuerySet(WorkFlowVersion).filter(application_id=embed_application.id).order_by(
|
||||
'-create_time')[0:1].first()
|
||||
|
|
@ -1318,29 +1298,3 @@ class ApplicationSerializer(serializers.Serializer):
|
|||
application_api_key.save()
|
||||
# 写入缓存
|
||||
get_application_api_key(application_api_key.secret_key, False)
|
||||
|
||||
class McpServers(serializers.Serializer):
|
||||
mcp_servers = serializers.JSONField(required=True)
|
||||
|
||||
def get_mcp_servers(self, with_valid=True):
|
||||
if with_valid:
|
||||
self.is_valid(raise_exception=True)
|
||||
if '"stdio"' in self.data.get('mcp_servers'):
|
||||
raise AppApiException(500, _('stdio is not supported'))
|
||||
servers = json.loads(self.data.get('mcp_servers'))
|
||||
|
||||
async def get_mcp_tools(servers):
|
||||
async with MultiServerMCPClient(servers) as client:
|
||||
return client.get_tools()
|
||||
|
||||
tools = []
|
||||
for server in servers:
|
||||
tools += [
|
||||
{
|
||||
'server': server,
|
||||
'name': tool.name,
|
||||
'description': tool.description,
|
||||
'args_schema': tool.args_schema,
|
||||
}
|
||||
for tool in asyncio.run(get_mcp_tools({server: servers[server]}))]
|
||||
return tools
|
||||
|
|
|
|||
|
|
@ -116,15 +116,13 @@ class ChatInfo:
|
|||
}
|
||||
|
||||
def to_pipeline_manage_params(self, problem_text: str, post_response_handler: PostResponseHandler,
|
||||
exclude_paragraph_id_list, client_id: str, client_type, stream=True, form_data=None):
|
||||
if form_data is None:
|
||||
form_data = {}
|
||||
exclude_paragraph_id_list, client_id: str, client_type, stream=True):
|
||||
params = self.to_base_pipeline_manage_params()
|
||||
return {**params, 'problem_text': problem_text, 'post_response_handler': post_response_handler,
|
||||
'exclude_paragraph_id_list': exclude_paragraph_id_list, 'stream': stream, 'client_id': client_id,
|
||||
'client_type': client_type, 'form_data': form_data}
|
||||
'client_type': client_type}
|
||||
|
||||
def append_chat_record(self, chat_record: ChatRecord, client_id=None, asker=None):
|
||||
def append_chat_record(self, chat_record: ChatRecord, client_id=None):
|
||||
chat_record.problem_text = chat_record.problem_text[0:10240] if chat_record.problem_text is not None else ""
|
||||
chat_record.answer_text = chat_record.answer_text[0:40960] if chat_record.problem_text is not None else ""
|
||||
is_save = True
|
||||
|
|
@ -139,17 +137,8 @@ class ChatInfo:
|
|||
if self.application.id is not None:
|
||||
# 插入数据库
|
||||
if not QuerySet(Chat).filter(id=self.chat_id).exists():
|
||||
asker_dict = {'user_name': '游客'}
|
||||
if asker is not None:
|
||||
if isinstance(asker, str):
|
||||
asker_dict = {
|
||||
'user_name': asker
|
||||
}
|
||||
elif isinstance(asker, dict):
|
||||
asker_dict = asker
|
||||
|
||||
Chat(id=self.chat_id, application_id=self.application.id, abstract=chat_record.problem_text[0:1024],
|
||||
client_id=client_id, asker=asker_dict, update_time=datetime.now()).save()
|
||||
client_id=client_id, update_time=datetime.now()).save()
|
||||
else:
|
||||
Chat.objects.filter(id=self.chat_id).update(update_time=datetime.now())
|
||||
# 插入会话记录
|
||||
|
|
@ -182,8 +171,7 @@ def get_post_handler(chat_info: ChatInfo):
|
|||
answer_text_list=answer_list,
|
||||
run_time=manage.context['run_time'],
|
||||
index=len(chat_info.chat_record_list) + 1)
|
||||
asker = kwargs.get("asker", None)
|
||||
chat_info.append_chat_record(chat_record, client_id, asker=asker)
|
||||
chat_info.append_chat_record(chat_record, client_id)
|
||||
# 重新设置缓存
|
||||
chat_cache.set(chat_id,
|
||||
chat_info, timeout=60 * 30)
|
||||
|
|
@ -213,21 +201,12 @@ class OpenAIChatSerializer(serializers.Serializer):
|
|||
return instance.get('messages')[-1].get('content')
|
||||
|
||||
@staticmethod
|
||||
def generate_chat(chat_id, application_id, message, client_id, asker=None):
|
||||
def generate_chat(chat_id, application_id, message, client_id):
|
||||
if chat_id is None:
|
||||
chat_id = str(uuid.uuid1())
|
||||
chat = QuerySet(Chat).filter(id=chat_id).first()
|
||||
if chat is None:
|
||||
asker_dict = {'user_name': '游客'}
|
||||
if asker is not None:
|
||||
if isinstance(asker, str):
|
||||
asker_dict = {
|
||||
'user_name': asker
|
||||
}
|
||||
elif isinstance(asker, dict):
|
||||
asker_dict = asker
|
||||
Chat(id=chat_id, application_id=application_id, abstract=message[0:1024], client_id=client_id,
|
||||
asker=asker_dict).save()
|
||||
Chat(id=chat_id, application_id=application_id, abstract=message[0:1024], client_id=client_id).save()
|
||||
return chat_id
|
||||
|
||||
def chat(self, instance: Dict, with_valid=True):
|
||||
|
|
@ -241,23 +220,15 @@ class OpenAIChatSerializer(serializers.Serializer):
|
|||
application_id = self.data.get('application_id')
|
||||
client_id = self.data.get('client_id')
|
||||
client_type = self.data.get('client_type')
|
||||
chat_id = self.generate_chat(chat_id, application_id, message, client_id,
|
||||
asker=instance.get('form_data', {}).get("asker"))
|
||||
chat_id = self.generate_chat(chat_id, application_id, message, client_id)
|
||||
return ChatMessageSerializer(
|
||||
data={
|
||||
'chat_id': chat_id, 'message': message,
|
||||
're_chat': re_chat,
|
||||
'stream': stream,
|
||||
'application_id': application_id,
|
||||
'client_id': client_id,
|
||||
'client_type': client_type,
|
||||
'form_data': instance.get('form_data', {}),
|
||||
'image_list': instance.get('image_list', []),
|
||||
'document_list': instance.get('document_list', []),
|
||||
'audio_list': instance.get('audio_list', []),
|
||||
'other_list': instance.get('other_list', []),
|
||||
}
|
||||
).chat(base_to_response=OpenaiToResponse())
|
||||
data={'chat_id': chat_id, 'message': message,
|
||||
're_chat': re_chat,
|
||||
'stream': stream,
|
||||
'application_id': application_id,
|
||||
'client_id': client_id,
|
||||
'client_type': client_type, 'form_data': instance.get('form_data', {})}).chat(
|
||||
base_to_response=OpenaiToResponse())
|
||||
|
||||
|
||||
class ChatMessageSerializer(serializers.Serializer):
|
||||
|
|
@ -285,7 +256,6 @@ class ChatMessageSerializer(serializers.Serializer):
|
|||
image_list = serializers.ListField(required=False, error_messages=ErrMessage.list(_("picture")))
|
||||
document_list = serializers.ListField(required=False, error_messages=ErrMessage.list(_("document")))
|
||||
audio_list = serializers.ListField(required=False, error_messages=ErrMessage.list(_("Audio")))
|
||||
other_list = serializers.ListField(required=False, error_messages=ErrMessage.list(_("Other")))
|
||||
child_node = serializers.DictField(required=False, allow_null=True,
|
||||
error_messages=ErrMessage.dict(_("Child Nodes")))
|
||||
|
||||
|
|
@ -334,7 +304,6 @@ class ChatMessageSerializer(serializers.Serializer):
|
|||
stream = self.data.get('stream')
|
||||
client_id = self.data.get('client_id')
|
||||
client_type = self.data.get('client_type')
|
||||
form_data = self.data.get("form_data")
|
||||
pipeline_manage_builder = PipelineManage.builder()
|
||||
# 如果开启了问题优化,则添加上问题优化步骤
|
||||
if chat_info.application.problem_optimization:
|
||||
|
|
@ -356,7 +325,7 @@ class ChatMessageSerializer(serializers.Serializer):
|
|||
exclude_paragraph_id_list = list(set(paragraph_id_list))
|
||||
# 构建运行参数
|
||||
params = chat_info.to_pipeline_manage_params(message, get_post_handler(chat_info), exclude_paragraph_id_list,
|
||||
client_id, client_type, stream, form_data)
|
||||
client_id, client_type, stream)
|
||||
# 运行流水线作业
|
||||
pipeline_message.run(params)
|
||||
return pipeline_message.context['chat_result']
|
||||
|
|
@ -384,7 +353,6 @@ class ChatMessageSerializer(serializers.Serializer):
|
|||
image_list = self.data.get('image_list')
|
||||
document_list = self.data.get('document_list')
|
||||
audio_list = self.data.get('audio_list')
|
||||
other_list = self.data.get('other_list')
|
||||
user_id = chat_info.application.user_id
|
||||
chat_record_id = self.data.get('chat_record_id')
|
||||
chat_record = None
|
||||
|
|
@ -401,7 +369,7 @@ class ChatMessageSerializer(serializers.Serializer):
|
|||
'client_id': client_id,
|
||||
'client_type': client_type,
|
||||
'user_id': user_id}, WorkFlowPostHandler(chat_info, client_id, client_type),
|
||||
base_to_response, form_data, image_list, document_list, audio_list, other_list,
|
||||
base_to_response, form_data, image_list, document_list, audio_list,
|
||||
self.data.get('runtime_node_id'),
|
||||
self.data.get('node_data'), chat_record, self.data.get('child_node'))
|
||||
r = work_flow_manage.run()
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import uuid
|
|||
from functools import reduce
|
||||
from io import BytesIO
|
||||
from typing import Dict
|
||||
import pytz
|
||||
|
||||
import openpyxl
|
||||
from django.core import validators
|
||||
from django.core.cache import caches
|
||||
|
|
@ -46,7 +46,6 @@ from embedding.task import embedding_by_paragraph, embedding_by_paragraph_list
|
|||
from setting.models import Model
|
||||
from setting.models_provider import get_model_credential
|
||||
from smartdoc.conf import PROJECT_DIR
|
||||
from smartdoc.settings import TIME_ZONE
|
||||
|
||||
chat_cache = caches['chat_cache']
|
||||
|
||||
|
|
@ -67,10 +66,6 @@ def valid_model_params_setting(model_id, model_params_setting):
|
|||
credential.get_model_params_setting_form(model.model_name).valid_form(model_params_setting)
|
||||
|
||||
|
||||
class ReAbstractInstanceSerializers(serializers.Serializer):
|
||||
abstract = serializers.CharField(required=True, error_messages=ErrMessage.char(_("abstract")))
|
||||
|
||||
|
||||
class ChatSerializers(serializers.Serializer):
|
||||
class Operate(serializers.Serializer):
|
||||
chat_id = serializers.UUIDField(required=True, error_messages=ErrMessage.uuid(_("Conversation ID")))
|
||||
|
|
@ -83,15 +78,6 @@ class ChatSerializers(serializers.Serializer):
|
|||
is_deleted=True)
|
||||
return True
|
||||
|
||||
def re_abstract(self, instance, with_valid=True):
|
||||
if with_valid:
|
||||
self.is_valid(raise_exception=True)
|
||||
ReAbstractInstanceSerializers(data=instance).is_valid(raise_exception=True)
|
||||
|
||||
QuerySet(Chat).filter(id=self.data.get('chat_id'), application_id=self.data.get('application_id')).update(
|
||||
abstract=instance.get('abstract'))
|
||||
return True
|
||||
|
||||
def delete(self, with_valid=True):
|
||||
if with_valid:
|
||||
self.is_valid(raise_exception=True)
|
||||
|
|
@ -174,14 +160,7 @@ class ChatSerializers(serializers.Serializer):
|
|||
condition = base_condition & min_trample_query
|
||||
else:
|
||||
condition = base_condition
|
||||
inner_queryset = QuerySet(Chat).filter(application_id=self.data.get("application_id"))
|
||||
if 'abstract' in self.data and self.data.get('abstract') is not None:
|
||||
inner_queryset = inner_queryset.filter(abstract__icontains=self.data.get('abstract'))
|
||||
|
||||
return {
|
||||
'inner_queryset': inner_queryset,
|
||||
'default_queryset': query_set.filter(condition).order_by("-application_chat.update_time")
|
||||
}
|
||||
return query_set.filter(condition).order_by("-application_chat.update_time")
|
||||
|
||||
def list(self, with_valid=True):
|
||||
if with_valid:
|
||||
|
|
@ -216,6 +195,7 @@ class ChatSerializers(serializers.Serializer):
|
|||
[])) for
|
||||
key, node in search_dataset_node_list])
|
||||
improve_paragraph_list = row.get('improve_paragraph_list')
|
||||
|
||||
vote_status_map = {'-1': '未投票', '0': '赞同', '1': '反对'}
|
||||
return [str(row.get('chat_id')), row.get('abstract'), row.get('problem_text'), padding_problem_text,
|
||||
row.get('answer_text'), vote_status_map.get(row.get('vote_status')), reference_paragraph_len,
|
||||
|
|
@ -223,9 +203,8 @@ class ChatSerializers(serializers.Serializer):
|
|||
"\n".join([
|
||||
f"{improve_paragraph_list[index].get('title')}\n{improve_paragraph_list[index].get('content')}"
|
||||
for index in range(len(improve_paragraph_list))]),
|
||||
row.get('asker').get('user_name'),
|
||||
row.get('message_tokens') + row.get('answer_tokens'), row.get('run_time'),
|
||||
str(row.get('create_time').astimezone(pytz.timezone(TIME_ZONE)).strftime('%Y-%m-%d %H:%M:%S')
|
||||
str(row.get('create_time').strftime('%Y-%m-%d %H:%M:%S')
|
||||
)]
|
||||
|
||||
def export(self, data, with_valid=True):
|
||||
|
|
@ -250,8 +229,7 @@ class ChatSerializers(serializers.Serializer):
|
|||
gettext('answer'), gettext('User feedback'),
|
||||
gettext('Reference segment number'),
|
||||
gettext('Section title + content'),
|
||||
gettext('Annotation'), gettext('USER'), gettext('Consuming tokens'),
|
||||
gettext('Time consumed (s)'),
|
||||
gettext('Annotation'), gettext('Consuming tokens'), gettext('Time consumed (s)'),
|
||||
gettext('Question Time')]
|
||||
for col_idx, header in enumerate(headers, 1):
|
||||
cell = worksheet.cell(row=1, column=col_idx)
|
||||
|
|
@ -265,10 +243,6 @@ class ChatSerializers(serializers.Serializer):
|
|||
cell = worksheet.cell(row=row_idx, column=col_idx)
|
||||
if isinstance(value, str):
|
||||
value = re.sub(ILLEGAL_CHARACTERS_RE, '', value)
|
||||
if isinstance(value, datetime.datetime):
|
||||
eastern = pytz.timezone(TIME_ZONE)
|
||||
c = datetime.timezone(eastern._utcoffset)
|
||||
value = value.astimezone(c)
|
||||
cell.value = value
|
||||
|
||||
output = BytesIO()
|
||||
|
|
@ -481,7 +455,7 @@ class ChatRecordSerializer(serializers.Serializer):
|
|||
class Query(serializers.Serializer):
|
||||
application_id = serializers.UUIDField(required=True)
|
||||
chat_id = serializers.UUIDField(required=True)
|
||||
order_asc = serializers.BooleanField(required=False, allow_null=True)
|
||||
order_asc = serializers.BooleanField(required=False)
|
||||
|
||||
def list(self, with_valid=True):
|
||||
if with_valid:
|
||||
|
|
|
|||
|
|
@ -10,8 +10,7 @@ SELECT
|
|||
application_chat_record_temp."index" as "index",
|
||||
application_chat_record_temp.improve_paragraph_list as improve_paragraph_list,
|
||||
application_chat_record_temp.vote_status as vote_status,
|
||||
application_chat_record_temp.create_time as create_time,
|
||||
to_json(application_chat.asker) as asker
|
||||
application_chat_record_temp.create_time as create_time
|
||||
FROM
|
||||
application_chat application_chat
|
||||
LEFT JOIN (
|
||||
|
|
@ -23,8 +22,6 @@ FROM
|
|||
chat_id
|
||||
FROM
|
||||
application_chat_record
|
||||
WHERE chat_id IN (
|
||||
SELECT id FROM application_chat ${inner_queryset})
|
||||
GROUP BY
|
||||
application_chat_record.chat_id
|
||||
) chat_record_temp ON application_chat."id" = chat_record_temp.chat_id
|
||||
|
|
@ -37,5 +34,4 @@ FROM
|
|||
END as improve_paragraph_list
|
||||
FROM
|
||||
application_chat_record application_chat_record
|
||||
) application_chat_record_temp ON application_chat_record_temp.chat_id = application_chat."id"
|
||||
${default_queryset}
|
||||
) application_chat_record_temp ON application_chat_record_temp.chat_id = application_chat."id"
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
SELECT
|
||||
*,to_json(asker) as asker
|
||||
*
|
||||
FROM
|
||||
application_chat application_chat
|
||||
LEFT JOIN (
|
||||
|
|
@ -11,9 +11,6 @@ FROM
|
|||
chat_id
|
||||
FROM
|
||||
application_chat_record
|
||||
WHERE chat_id IN (
|
||||
SELECT id FROM application_chat ${inner_queryset})
|
||||
GROUP BY
|
||||
application_chat_record.chat_id
|
||||
) chat_record_temp ON application_chat."id" = chat_record_temp.chat_id
|
||||
${default_queryset}
|
||||
) chat_record_temp ON application_chat."id" = chat_record_temp.chat_id
|
||||
|
|
@ -38,15 +38,6 @@ class ApplicationApi(ApiMixin):
|
|||
}
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_response_body_api():
|
||||
return openapi.Schema(
|
||||
type=openapi.TYPE_STRING,
|
||||
title=_("Application authentication token"),
|
||||
description=_("Application authentication token"),
|
||||
default="token"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_response_body_api():
|
||||
return openapi.Schema(
|
||||
|
|
@ -142,27 +133,6 @@ class ApplicationApi(ApiMixin):
|
|||
}
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_response_body_api():
|
||||
return openapi.Schema(
|
||||
type=openapi.TYPE_OBJECT,
|
||||
properties={
|
||||
'id': openapi.Schema(type=openapi.TYPE_STRING, title=_("Primary key id"),
|
||||
description=_("Primary key id")),
|
||||
'secret_key': openapi.Schema(type=openapi.TYPE_STRING, title=_("Secret key"),
|
||||
description=_("Secret key")),
|
||||
'is_active': openapi.Schema(type=openapi.TYPE_BOOLEAN, title=_("Is activation"),
|
||||
description=_("Is activation")),
|
||||
'application_id': openapi.Schema(type=openapi.TYPE_STRING, title=_("Application ID"),
|
||||
description=_("Application ID")),
|
||||
'allow_cross_domain': openapi.Schema(type=openapi.TYPE_BOOLEAN,
|
||||
title=_("Is cross-domain allowed"),
|
||||
description=_("Is cross-domain allowed")),
|
||||
'cross_domain_list': openapi.Schema(type=openapi.TYPE_ARRAY, title=_('Cross-domain list'),
|
||||
items=openapi.Schema(type=openapi.TYPE_STRING))
|
||||
}
|
||||
)
|
||||
|
||||
class AccessToken(ApiMixin):
|
||||
@staticmethod
|
||||
def get_request_params_api():
|
||||
|
|
@ -201,37 +171,6 @@ class ApplicationApi(ApiMixin):
|
|||
}
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_response_body_api():
|
||||
return openapi.Schema(
|
||||
type=openapi.TYPE_OBJECT,
|
||||
required=[],
|
||||
properties={
|
||||
'id': openapi.Schema(type=openapi.TYPE_STRING, title=_("Primary key id"),
|
||||
description=_("Primary key id")),
|
||||
'access_token': openapi.Schema(type=openapi.TYPE_STRING, title=_("Access Token"),
|
||||
description=_("Access Token")),
|
||||
'access_token_reset': openapi.Schema(type=openapi.TYPE_BOOLEAN, title=_("Reset Token"),
|
||||
description=_("Reset Token")),
|
||||
|
||||
'is_active': openapi.Schema(type=openapi.TYPE_BOOLEAN, title=_("Is activation"),
|
||||
description=_("Is activation")),
|
||||
'access_num': openapi.Schema(type=openapi.TYPE_NUMBER, title=_("Number of visits"),
|
||||
description=_("Number of visits")),
|
||||
'white_active': openapi.Schema(type=openapi.TYPE_BOOLEAN, title=_("Whether to enable whitelist"),
|
||||
description=_("Whether to enable whitelist")),
|
||||
'white_list': openapi.Schema(type=openapi.TYPE_ARRAY,
|
||||
items=openapi.Schema(type=openapi.TYPE_STRING), title=_("Whitelist"),
|
||||
description=_("Whitelist")),
|
||||
'show_source': openapi.Schema(type=openapi.TYPE_BOOLEAN,
|
||||
title=_("Whether to display knowledge sources"),
|
||||
description=_("Whether to display knowledge sources")),
|
||||
'language': openapi.Schema(type=openapi.TYPE_STRING,
|
||||
title=_("language"),
|
||||
description=_("language"))
|
||||
}
|
||||
)
|
||||
|
||||
class Edit(ApiMixin):
|
||||
@staticmethod
|
||||
def get_request_body_api():
|
||||
|
|
@ -295,7 +234,7 @@ class ApplicationApi(ApiMixin):
|
|||
default=[]),
|
||||
'edges': openapi.Schema(type=openapi.TYPE_ARRAY, items=openapi.Schema(type=openapi.TYPE_OBJECT),
|
||||
title=_('Connection List'), description=_("Connection List"),
|
||||
default=[]),
|
||||
default={}),
|
||||
|
||||
}
|
||||
)
|
||||
|
|
@ -385,8 +324,7 @@ class ApplicationApi(ApiMixin):
|
|||
return openapi.Schema(
|
||||
type=openapi.TYPE_OBJECT,
|
||||
required=['name', 'desc', 'model_id', 'dialogue_number', 'dataset_setting', 'model_setting',
|
||||
'problem_optimization', 'stt_model_enable', 'stt_model_enable', 'tts_type',
|
||||
'work_flow'],
|
||||
'problem_optimization', 'stt_model_enable', 'stt_model_enable', 'tts_type'],
|
||||
properties={
|
||||
'name': openapi.Schema(type=openapi.TYPE_STRING, title=_("Application Name"),
|
||||
description=_("Application Name")),
|
||||
|
|
@ -423,58 +361,7 @@ class ApplicationApi(ApiMixin):
|
|||
'tts_model_enable': openapi.Schema(type=openapi.TYPE_STRING, title=_("Is text-to-speech enabled"),
|
||||
description=_("Is text-to-speech enabled")),
|
||||
'tts_type': openapi.Schema(type=openapi.TYPE_STRING, title=_("Text-to-speech type"),
|
||||
description=_("Text-to-speech type")),
|
||||
'work_flow': ApplicationApi.WorkFlow.get_request_body_api(),
|
||||
}
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_response_body_api():
|
||||
return openapi.Schema(
|
||||
type=openapi.TYPE_OBJECT,
|
||||
required=['id', 'name', 'desc', 'model_id', 'dialogue_number', 'dataset_setting', 'model_setting',
|
||||
'problem_optimization', 'stt_model_enable', 'stt_model_enable', 'tts_type',
|
||||
'work_flow'],
|
||||
properties={
|
||||
'id': openapi.Schema(type=openapi.TYPE_STRING, title=_("Primary key id"),
|
||||
description=_("Primary key id")),
|
||||
'name': openapi.Schema(type=openapi.TYPE_STRING, title=_("Application Name"),
|
||||
description=_("Application Name")),
|
||||
'desc': openapi.Schema(type=openapi.TYPE_STRING, title=_("Application Description"),
|
||||
description=_("Application Description")),
|
||||
'model_id': openapi.Schema(type=openapi.TYPE_STRING, title=_("Model id"),
|
||||
description=_("Model id")),
|
||||
"dialogue_number": openapi.Schema(type=openapi.TYPE_NUMBER,
|
||||
title=_("Number of multi-round conversations"),
|
||||
description=_("Number of multi-round conversations")),
|
||||
'prologue': openapi.Schema(type=openapi.TYPE_STRING, title=_("Opening remarks"),
|
||||
description=_("Opening remarks")),
|
||||
'dataset_id_list': openapi.Schema(type=openapi.TYPE_ARRAY,
|
||||
items=openapi.Schema(type=openapi.TYPE_STRING),
|
||||
title=_("List of associated knowledge base IDs"),
|
||||
description=_("List of associated knowledge base IDs")),
|
||||
'dataset_setting': ApplicationApi.DatasetSetting.get_request_body_api(),
|
||||
'model_setting': ApplicationApi.ModelSetting.get_request_body_api(),
|
||||
'problem_optimization': openapi.Schema(type=openapi.TYPE_BOOLEAN, title=_("Problem Optimization"),
|
||||
description=_("Problem Optimization"), default=True),
|
||||
'type': openapi.Schema(type=openapi.TYPE_STRING, title=_("Application Type"),
|
||||
description=_("Application Type SIMPLE | WORK_FLOW")),
|
||||
'problem_optimization_prompt': openapi.Schema(type=openapi.TYPE_STRING,
|
||||
title=_('Question optimization tips'),
|
||||
description=_("Question optimization tips"),
|
||||
default=_(
|
||||
"() contains the user's question. Answer the guessed user's question based on the context ({question}) Requirement: Output a complete question and put it in the <data></data> tag")),
|
||||
'tts_model_id': openapi.Schema(type=openapi.TYPE_STRING, title=_("Text-to-speech model ID"),
|
||||
description=_("Text-to-speech model ID")),
|
||||
'stt_model_id': openapi.Schema(type=openapi.TYPE_STRING, title=_("Speech-to-text model id"),
|
||||
description=_("Speech-to-text model id")),
|
||||
'stt_model_enable': openapi.Schema(type=openapi.TYPE_STRING, title=_("Is speech-to-text enabled"),
|
||||
description=_("Is speech-to-text enabled")),
|
||||
'tts_model_enable': openapi.Schema(type=openapi.TYPE_STRING, title=_("Is text-to-speech enabled"),
|
||||
description=_("Is text-to-speech enabled")),
|
||||
'tts_type': openapi.Schema(type=openapi.TYPE_STRING, title=_("Text-to-speech type"),
|
||||
description=_("Text-to-speech type")),
|
||||
'work_flow': ApplicationApi.WorkFlow.get_request_body_api(),
|
||||
description=_("Text-to-speech type"))
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -545,4 +432,4 @@ class ApplicationApi(ApiMixin):
|
|||
'text': openapi.Schema(type=openapi.TYPE_STRING, title=_("Text"),
|
||||
description=_("Text")),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
|
@ -23,115 +23,8 @@ class ChatClientHistoryApi(ApiMixin):
|
|||
description=_('Application ID'))
|
||||
]
|
||||
|
||||
class Operate(ApiMixin):
|
||||
@staticmethod
|
||||
def get_request_params_api():
|
||||
return [openapi.Parameter(name='application_id',
|
||||
in_=openapi.IN_PATH,
|
||||
type=openapi.TYPE_STRING,
|
||||
required=True,
|
||||
description=_('Application ID')),
|
||||
openapi.Parameter(name='chat_id',
|
||||
in_=openapi.IN_PATH,
|
||||
type=openapi.TYPE_STRING,
|
||||
required=True,
|
||||
description=_('Conversation ID')),
|
||||
]
|
||||
|
||||
class ReAbstract(ApiMixin):
|
||||
@staticmethod
|
||||
def get_request_body_api():
|
||||
return openapi.Schema(
|
||||
type=openapi.TYPE_OBJECT,
|
||||
required=['abstract'],
|
||||
properties={
|
||||
'abstract': openapi.Schema(type=openapi.TYPE_STRING, title=_("abstract"),
|
||||
description=_("abstract"))
|
||||
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class OpenAIChatApi(ApiMixin):
|
||||
@staticmethod
|
||||
def get_response_body_api():
|
||||
return openapi.Responses(responses={
|
||||
200: openapi.Response(description=_('response parameters'),
|
||||
schema=openapi.Schema(type=openapi.TYPE_OBJECT,
|
||||
required=['id',
|
||||
'choices'],
|
||||
properties={
|
||||
'id': openapi.Schema(
|
||||
type=openapi.TYPE_STRING,
|
||||
title=_(
|
||||
"Conversation ID")),
|
||||
'choices': openapi.Schema(
|
||||
type=openapi.TYPE_ARRAY,
|
||||
items=openapi.Schema(
|
||||
type=openapi.TYPE_OBJECT,
|
||||
required=[
|
||||
'message'],
|
||||
properties={
|
||||
'finish_reason': openapi.Schema(
|
||||
type=openapi.TYPE_STRING, ),
|
||||
'index': openapi.Schema(
|
||||
type=openapi.TYPE_INTEGER),
|
||||
'answer_list': openapi.Schema(
|
||||
type=openapi.TYPE_ARRAY,
|
||||
items=openapi.Schema(
|
||||
type=openapi.TYPE_OBJECT,
|
||||
required=[
|
||||
'content'],
|
||||
properties={
|
||||
'content': openapi.Schema(
|
||||
type=openapi.TYPE_STRING),
|
||||
'view_type': openapi.Schema(
|
||||
type=openapi.TYPE_STRING),
|
||||
'runtime_node_id': openapi.Schema(
|
||||
type=openapi.TYPE_STRING),
|
||||
'chat_record_id': openapi.Schema(
|
||||
type=openapi.TYPE_STRING),
|
||||
'reasoning_content': openapi.Schema(
|
||||
type=openapi.TYPE_STRING),
|
||||
}
|
||||
)),
|
||||
'message': openapi.Schema(
|
||||
type=openapi.TYPE_OBJECT,
|
||||
required=[
|
||||
'content'],
|
||||
properties={
|
||||
'content': openapi.Schema(
|
||||
type=openapi.TYPE_STRING),
|
||||
'role': openapi.Schema(
|
||||
type=openapi.TYPE_STRING)
|
||||
|
||||
}),
|
||||
|
||||
}
|
||||
)),
|
||||
'created': openapi.Schema(
|
||||
type=openapi.TYPE_INTEGER),
|
||||
'model': openapi.Schema(
|
||||
type=openapi.TYPE_STRING),
|
||||
'object': openapi.Schema(
|
||||
type=openapi.TYPE_STRING),
|
||||
'usage': openapi.Schema(
|
||||
type=openapi.TYPE_OBJECT,
|
||||
required=[
|
||||
'completion_tokens',
|
||||
'prompt_tokens',
|
||||
'total_tokens'],
|
||||
properties={
|
||||
'completion_tokens': openapi.Schema(
|
||||
type=openapi.TYPE_INTEGER),
|
||||
'prompt_tokens': openapi.Schema(
|
||||
type=openapi.TYPE_INTEGER),
|
||||
'total_tokens': openapi.Schema(
|
||||
type=openapi.TYPE_INTEGER)
|
||||
})
|
||||
|
||||
}))})
|
||||
|
||||
@staticmethod
|
||||
def get_request_body_api():
|
||||
return openapi.Schema(type=openapi.TYPE_OBJECT,
|
||||
|
|
@ -155,8 +48,7 @@ class OpenAIChatApi(ApiMixin):
|
|||
'chat_id': openapi.Schema(type=openapi.TYPE_STRING, title=_("Conversation ID")),
|
||||
're_chat': openapi.Schema(type=openapi.TYPE_BOOLEAN, title=_("regenerate"),
|
||||
default=False),
|
||||
'stream': openapi.Schema(type=openapi.TYPE_BOOLEAN, title=_("Stream Output"),
|
||||
default=True)
|
||||
'stream': openapi.Schema(type=openapi.TYPE_BOOLEAN, title=_("Stream Output"), default=True)
|
||||
|
||||
})
|
||||
|
||||
|
|
@ -170,66 +62,7 @@ class ChatApi(ApiMixin):
|
|||
properties={
|
||||
'message': openapi.Schema(type=openapi.TYPE_STRING, title=_("problem"), description=_("problem")),
|
||||
're_chat': openapi.Schema(type=openapi.TYPE_BOOLEAN, title=_("regenerate"), default=False),
|
||||
'stream': openapi.Schema(type=openapi.TYPE_BOOLEAN, title=_("Is it streaming output"), default=True),
|
||||
|
||||
'form_data': openapi.Schema(type=openapi.TYPE_OBJECT, title=_("Form data"),
|
||||
description=_("Form data"),
|
||||
default={}),
|
||||
'image_list': openapi.Schema(
|
||||
type=openapi.TYPE_ARRAY,
|
||||
title=_("Image list"),
|
||||
description=_("Image list"),
|
||||
items=openapi.Schema(
|
||||
type=openapi.TYPE_OBJECT,
|
||||
properties={
|
||||
'name': openapi.Schema(type=openapi.TYPE_STRING,
|
||||
title=_("Image name")),
|
||||
'url': openapi.Schema(type=openapi.TYPE_STRING,
|
||||
title=_("Image URL")),
|
||||
'file_id': openapi.Schema(type=openapi.TYPE_STRING),
|
||||
}
|
||||
),
|
||||
default=[]
|
||||
),
|
||||
'document_list': openapi.Schema(type=openapi.TYPE_ARRAY, title=_("Document list"),
|
||||
description=_("Document list"),
|
||||
items=openapi.Schema(
|
||||
type=openapi.TYPE_OBJECT,
|
||||
properties={
|
||||
# 定义对象的具体属性
|
||||
'name': openapi.Schema(type=openapi.TYPE_STRING,
|
||||
title=_("Document name")),
|
||||
'url': openapi.Schema(type=openapi.TYPE_STRING,
|
||||
title=_("Document URL")),
|
||||
'file_id': openapi.Schema(type=openapi.TYPE_STRING),
|
||||
}
|
||||
),
|
||||
default=[]),
|
||||
'audio_list': openapi.Schema(type=openapi.TYPE_ARRAY, title=_("Audio list"),
|
||||
description=_("Audio list"),
|
||||
items=openapi.Schema(
|
||||
type=openapi.TYPE_OBJECT,
|
||||
properties={
|
||||
'name': openapi.Schema(type=openapi.TYPE_STRING,
|
||||
title=_("Audio name")),
|
||||
'url': openapi.Schema(type=openapi.TYPE_STRING,
|
||||
title=_("Audio URL")),
|
||||
'file_id': openapi.Schema(type=openapi.TYPE_STRING),
|
||||
}
|
||||
),
|
||||
default=[]),
|
||||
'runtime_node_id': openapi.Schema(type=openapi.TYPE_STRING, title=_("Runtime node id"),
|
||||
description=_("Runtime node id"),
|
||||
default=""),
|
||||
'node_data': openapi.Schema(type=openapi.TYPE_OBJECT, title=_("Node data"),
|
||||
description=_("Node data"),
|
||||
default={}),
|
||||
'chat_record_id': openapi.Schema(type=openapi.TYPE_STRING, title=_("Conversation record id"),
|
||||
description=_("Conversation record id"),
|
||||
default=""),
|
||||
'child_node': openapi.Schema(type=openapi.TYPE_STRING, title=_("Child node"),
|
||||
description=_("Child node"),
|
||||
default={}),
|
||||
'stream': openapi.Schema(type=openapi.TYPE_BOOLEAN, title=_("Is it streaming output"), default=True)
|
||||
|
||||
}
|
||||
)
|
||||
|
|
@ -299,35 +132,20 @@ class ChatApi(ApiMixin):
|
|||
'problem_optimization'],
|
||||
properties={
|
||||
'id': openapi.Schema(type=openapi.TYPE_STRING, title=_("Application ID"),
|
||||
description=_(
|
||||
"Application ID, pass when modifying, do not pass when creating")),
|
||||
'model_id': openapi.Schema(type=openapi.TYPE_STRING, title=_("Model ID"),
|
||||
description=_("Model ID")),
|
||||
description=_("Application ID, pass when modifying, do not pass when creating")),
|
||||
'model_id': openapi.Schema(type=openapi.TYPE_STRING, title=_("Model ID"), description=_("Model ID")),
|
||||
'dataset_id_list': openapi.Schema(type=openapi.TYPE_ARRAY,
|
||||
items=openapi.Schema(type=openapi.TYPE_STRING),
|
||||
title=_("List of associated knowledge base IDs"),
|
||||
description=_("List of associated knowledge base IDs")),
|
||||
'multiple_rounds_dialogue': openapi.Schema(type=openapi.TYPE_BOOLEAN,
|
||||
title=_("Do you want to initiate multiple sessions"),
|
||||
description=_(
|
||||
"Do you want to initiate multiple sessions")),
|
||||
title=_("List of associated knowledge base IDs"), description=_("List of associated knowledge base IDs")),
|
||||
'multiple_rounds_dialogue': openapi.Schema(type=openapi.TYPE_BOOLEAN, title=_("Do you want to initiate multiple sessions"),
|
||||
description=_("Do you want to initiate multiple sessions")),
|
||||
'dataset_setting': ApplicationApi.DatasetSetting.get_request_body_api(),
|
||||
'model_setting': ApplicationApi.ModelSetting.get_request_body_api(),
|
||||
'problem_optimization': openapi.Schema(type=openapi.TYPE_BOOLEAN, title=_("Problem optimization"),
|
||||
description=_("Do you want to enable problem optimization"),
|
||||
default=True)
|
||||
description=_("Do you want to enable problem optimization"), default=True)
|
||||
}
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_response_body_api():
|
||||
return openapi.Schema(
|
||||
type=openapi.TYPE_STRING,
|
||||
title=_("Conversation ID"),
|
||||
description=_("Conversation ID"),
|
||||
default="chat_id"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_request_params_api():
|
||||
return [openapi.Parameter(name='application_id',
|
||||
|
|
@ -347,15 +165,7 @@ class ChatApi(ApiMixin):
|
|||
openapi.Parameter(name='min_trample', in_=openapi.IN_QUERY, type=openapi.TYPE_INTEGER, required=False,
|
||||
description=_("Minimum number of clicks")),
|
||||
openapi.Parameter(name='comparer', in_=openapi.IN_QUERY, type=openapi.TYPE_STRING, required=False,
|
||||
description=_("or|and comparator")),
|
||||
openapi.Parameter(name='start_time', in_=openapi.IN_QUERY,
|
||||
type=openapi.TYPE_STRING,
|
||||
required=True,
|
||||
description=_('start time')),
|
||||
openapi.Parameter(name='end_time', in_=openapi.IN_QUERY,
|
||||
type=openapi.TYPE_STRING,
|
||||
required=True,
|
||||
description=_('End time')),
|
||||
description=_("or|and comparator"))
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -372,11 +182,6 @@ class ChatRecordApi(ApiMixin):
|
|||
type=openapi.TYPE_STRING,
|
||||
required=True,
|
||||
description=_('Conversation ID')),
|
||||
openapi.Parameter(name='order_asc',
|
||||
in_=openapi.IN_QUERY,
|
||||
type=openapi.TYPE_BOOLEAN,
|
||||
required=False,
|
||||
description=_('Is it ascending order')),
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -401,19 +206,14 @@ class ChatRecordApi(ApiMixin):
|
|||
description=_("Resource ID"), default=1),
|
||||
'source_type': openapi.Schema(type=openapi.TYPE_STRING, title=_("Resource Type"),
|
||||
description=_("Resource Type"), default='xxx'),
|
||||
'message_tokens': openapi.Schema(type=openapi.TYPE_INTEGER,
|
||||
title=_("Number of tokens consumed by the question"),
|
||||
'message_tokens': openapi.Schema(type=openapi.TYPE_INTEGER, title=_("Number of tokens consumed by the question"),
|
||||
description=_("Number of tokens consumed by the question"), default=0),
|
||||
'answer_tokens': openapi.Schema(type=openapi.TYPE_INTEGER,
|
||||
title=_("The number of tokens consumed by the answer"),
|
||||
description=_("The number of tokens consumed by the answer"),
|
||||
default=0),
|
||||
'improve_paragraph_id_list': openapi.Schema(type=openapi.TYPE_STRING,
|
||||
title=_("Improved annotation list"),
|
||||
'answer_tokens': openapi.Schema(type=openapi.TYPE_INTEGER, title=_("The number of tokens consumed by the answer"),
|
||||
description=_("The number of tokens consumed by the answer"), default=0),
|
||||
'improve_paragraph_id_list': openapi.Schema(type=openapi.TYPE_STRING, title=_("Improved annotation list"),
|
||||
description=_("Improved annotation list"),
|
||||
default=[]),
|
||||
'index': openapi.Schema(type=openapi.TYPE_STRING,
|
||||
title=_("Corresponding session Corresponding subscript"),
|
||||
'index': openapi.Schema(type=openapi.TYPE_STRING, title=_("Corresponding session Corresponding subscript"),
|
||||
description=_("Corresponding session id corresponding subscript"),
|
||||
default=0
|
||||
),
|
||||
|
|
@ -572,8 +372,7 @@ class ChatRecordImproveApi(ApiMixin):
|
|||
description=_("Paragraph content"), default=_('Paragraph content')),
|
||||
'title': openapi.Schema(type=openapi.TYPE_STRING, title=_("title"),
|
||||
description=_("title"), default=_("Description of xxx")),
|
||||
'hit_num': openapi.Schema(type=openapi.TYPE_INTEGER, title=_("Number of hits"),
|
||||
description=_("Number of hits"),
|
||||
'hit_num': openapi.Schema(type=openapi.TYPE_INTEGER, title=_("Number of hits"), description=_("Number of hits"),
|
||||
default=1),
|
||||
'star_num': openapi.Schema(type=openapi.TYPE_INTEGER, title=_("Number of Likes"),
|
||||
description=_("Number of Likes"), default=1),
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ urlpatterns = [
|
|||
path('application/profile', views.Application.Profile.as_view(), name='application/profile'),
|
||||
path('application/embed', views.Application.Embed.as_view()),
|
||||
path('application/authentication', views.Application.Authentication.as_view()),
|
||||
path('application/mcp_servers', views.Application.McpServers.as_view()),
|
||||
path('application/<str:application_id>/publish', views.Application.Publish.as_view()),
|
||||
path('application/<str:application_id>/edit_icon', views.Application.EditIcon.as_view()),
|
||||
path('application/<str:application_id>/export', views.Application.Export.as_view()),
|
||||
|
|
|
|||
|
|
@ -13,11 +13,9 @@ from rest_framework.views import APIView
|
|||
|
||||
from application.serializers.application_version_serializers import ApplicationVersionSerializer
|
||||
from application.swagger_api.application_version_api import ApplicationVersionApi
|
||||
from application.views import get_application_operation_object
|
||||
from common.auth import has_permissions, TokenAuth
|
||||
from common.constants.permission_constants import PermissionConstants, CompareConstants, ViewPermission, RoleConstants, \
|
||||
Permission, Group, Operate
|
||||
from common.log.log import log
|
||||
from common.response import result
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
|
|
@ -84,8 +82,6 @@ class ApplicationVersionView(APIView):
|
|||
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.MANAGE,
|
||||
dynamic_tag=keywords.get('application_id'))],
|
||||
compare=CompareConstants.AND))
|
||||
@log(menu='Application', operate="Modify application version information",
|
||||
get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')))
|
||||
def put(self, request: Request, application_id: str, work_flow_version_id: str):
|
||||
return result.success(
|
||||
ApplicationVersionSerializer.Operate(
|
||||
|
|
|
|||
|
|
@ -9,9 +9,8 @@
|
|||
|
||||
from django.core import cache
|
||||
from django.http import HttpResponse
|
||||
from django.utils.translation import gettext_lazy as _, gettext
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
from drf_yasg.utils import swagger_auto_schema
|
||||
from langchain_core.prompts import PromptTemplate
|
||||
from rest_framework.decorators import action
|
||||
from rest_framework.parsers import MultiPartParser
|
||||
from rest_framework.request import Request
|
||||
|
|
@ -21,12 +20,10 @@ from application.serializers.application_serializers import ApplicationSerialize
|
|||
from application.serializers.application_statistics_serializers import ApplicationStatisticsSerializer
|
||||
from application.swagger_api.application_api import ApplicationApi
|
||||
from application.swagger_api.application_statistics_api import ApplicationStatisticsApi
|
||||
from application.views.common import get_application_operation_object
|
||||
from common.auth import TokenAuth, has_permissions
|
||||
from common.constants.permission_constants import CompareConstants, PermissionConstants, Permission, Group, Operate, \
|
||||
ViewPermission, RoleConstants
|
||||
from common.exception.app_exception import AppAuthenticationFailed
|
||||
from common.log.log import log
|
||||
from common.response import result
|
||||
from common.swagger_api.common_api import CommonApi
|
||||
from common.util.common import query_params_to_single_dict
|
||||
|
|
@ -155,8 +152,6 @@ class Application(APIView):
|
|||
dynamic_tag=keywords.get('application_id'))],
|
||||
compare=CompareConstants.AND), PermissionConstants.APPLICATION_EDIT,
|
||||
compare=CompareConstants.AND)
|
||||
@log(menu='Application', operate="Modify application icon",
|
||||
get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')))
|
||||
def put(self, request: Request, application_id: str):
|
||||
return result.success(
|
||||
ApplicationSerializer.IconOperate(
|
||||
|
|
@ -173,7 +168,6 @@ class Application(APIView):
|
|||
tags=[_("Application")]
|
||||
)
|
||||
@has_permissions(RoleConstants.ADMIN, RoleConstants.USER)
|
||||
@log(menu='Application', operate="Import Application")
|
||||
def post(self, request: Request):
|
||||
return result.success(ApplicationSerializer.Import(
|
||||
data={'user_id': request.user.id, 'file': request.FILES.get('file')}).import_())
|
||||
|
|
@ -188,8 +182,6 @@ class Application(APIView):
|
|||
)
|
||||
@has_permissions(lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.MANAGE,
|
||||
dynamic_tag=keywords.get('application_id')))
|
||||
@log(menu='Application', operate="Export Application",
|
||||
get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')))
|
||||
def get(self, request: Request, application_id: str):
|
||||
return ApplicationSerializer.Operate(
|
||||
data={'application_id': application_id, 'user_id': request.user.id}).export()
|
||||
|
|
@ -343,8 +335,6 @@ class Application(APIView):
|
|||
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.MANAGE,
|
||||
dynamic_tag=keywords.get('application_id'))],
|
||||
compare=CompareConstants.AND))
|
||||
@log(menu='Application', operate="Add ApiKey",
|
||||
get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')))
|
||||
def post(self, request: Request, application_id: str):
|
||||
return result.success(
|
||||
ApplicationSerializer.ApplicationKeySerializer(
|
||||
|
|
@ -373,16 +363,13 @@ class Application(APIView):
|
|||
operation_id=_("Modify application API_KEY"),
|
||||
tags=[_('Application/API_KEY')],
|
||||
manual_parameters=ApplicationApi.ApiKey.Operate.get_request_params_api(),
|
||||
request_body=ApplicationApi.ApiKey.Operate.get_request_body_api(),
|
||||
responses=result.get_api_response(ApplicationApi.ApiKey.Operate.get_response_body_api()))
|
||||
request_body=ApplicationApi.ApiKey.Operate.get_request_body_api())
|
||||
@has_permissions(ViewPermission(
|
||||
[RoleConstants.ADMIN, RoleConstants.USER],
|
||||
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.MANAGE,
|
||||
dynamic_tag=keywords.get('application_id'))],
|
||||
compare=CompareConstants.AND), PermissionConstants.APPLICATION_EDIT,
|
||||
compare=CompareConstants.AND)
|
||||
@log(menu='Application', operate="Modify application API_KEY",
|
||||
get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')))
|
||||
def put(self, request: Request, application_id: str, api_key_id: str):
|
||||
return result.success(
|
||||
ApplicationSerializer.ApplicationKeySerializer.Operate(
|
||||
|
|
@ -400,8 +387,6 @@ class Application(APIView):
|
|||
dynamic_tag=keywords.get('application_id'))],
|
||||
compare=CompareConstants.AND), PermissionConstants.APPLICATION_DELETE,
|
||||
compare=CompareConstants.AND)
|
||||
@log(menu='Application', operate="Delete Application API_KEY",
|
||||
get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')))
|
||||
def delete(self, request: Request, application_id: str, api_key_id: str):
|
||||
return result.success(
|
||||
ApplicationSerializer.ApplicationKeySerializer.Operate(
|
||||
|
|
@ -416,15 +401,12 @@ class Application(APIView):
|
|||
operation_id=_("Modify Application AccessToken"),
|
||||
tags=[_('Application/Public Access')],
|
||||
manual_parameters=ApplicationApi.AccessToken.get_request_params_api(),
|
||||
request_body=ApplicationApi.AccessToken.get_request_body_api(),
|
||||
responses=result.get_api_response(ApplicationApi.AccessToken.get_response_body_api()))
|
||||
request_body=ApplicationApi.AccessToken.get_request_body_api())
|
||||
@has_permissions(ViewPermission(
|
||||
[RoleConstants.ADMIN, RoleConstants.USER],
|
||||
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.MANAGE,
|
||||
dynamic_tag=keywords.get('application_id'))],
|
||||
compare=CompareConstants.AND))
|
||||
@log(menu='Application', operate="Modify Application AccessToken",
|
||||
get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')))
|
||||
def put(self, request: Request, application_id: str):
|
||||
return result.success(
|
||||
ApplicationSerializer.AccessTokenSerializer(data={'application_id': application_id}).edit(
|
||||
|
|
@ -457,7 +439,6 @@ class Application(APIView):
|
|||
@swagger_auto_schema(operation_summary=_("Application Certification"),
|
||||
operation_id=_("Application Certification"),
|
||||
request_body=ApplicationApi.Authentication.get_request_body_api(),
|
||||
responses=result.get_api_response(ApplicationApi.Authentication.get_response_body_api()),
|
||||
tags=[_("Application/Certification")],
|
||||
security=[])
|
||||
def post(self, request: Request):
|
||||
|
|
@ -475,11 +456,8 @@ class Application(APIView):
|
|||
@swagger_auto_schema(operation_summary=_("Create an application"),
|
||||
operation_id=_("Create an application"),
|
||||
request_body=ApplicationApi.Create.get_request_body_api(),
|
||||
responses=result.get_api_response(ApplicationApi.Create.get_response_body_api()),
|
||||
tags=[_('Application')])
|
||||
@has_permissions(PermissionConstants.APPLICATION_CREATE, compare=CompareConstants.AND)
|
||||
@log(menu='Application', operate="Create an application",
|
||||
get_operation_object=lambda r, k: {'name': r.data.get('name')})
|
||||
def post(self, request: Request):
|
||||
return result.success(ApplicationSerializer.Create(data={'user_id': request.user.id}).insert(request.data))
|
||||
|
||||
|
|
@ -534,8 +512,6 @@ class Application(APIView):
|
|||
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.MANAGE,
|
||||
dynamic_tag=keywords.get('application_id'))],
|
||||
compare=CompareConstants.AND))
|
||||
@log(menu='Application', operate="Publishing an application",
|
||||
get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')))
|
||||
def put(self, request: Request, application_id: str):
|
||||
return result.success(
|
||||
ApplicationSerializer.Operate(
|
||||
|
|
@ -557,8 +533,6 @@ class Application(APIView):
|
|||
compare=CompareConstants.AND),
|
||||
lambda r, k: Permission(group=Group.APPLICATION, operate=Operate.DELETE,
|
||||
dynamic_tag=k.get('application_id')), compare=CompareConstants.AND)
|
||||
@log(menu='Application', operate="Deleting application",
|
||||
get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')))
|
||||
def delete(self, request: Request, application_id: str):
|
||||
return result.success(ApplicationSerializer.Operate(
|
||||
data={'application_id': application_id, 'user_id': request.user.id}).delete(
|
||||
|
|
@ -576,8 +550,6 @@ class Application(APIView):
|
|||
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.MANAGE,
|
||||
dynamic_tag=keywords.get('application_id'))],
|
||||
compare=CompareConstants.AND))
|
||||
@log(menu='Application', operate="Modify the application",
|
||||
get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')))
|
||||
def put(self, request: Request, application_id: str):
|
||||
return result.success(
|
||||
ApplicationSerializer.Operate(
|
||||
|
|
@ -688,19 +660,8 @@ class Application(APIView):
|
|||
dynamic_tag=keywords.get(
|
||||
'application_id'))],
|
||||
compare=CompareConstants.AND))
|
||||
@log(menu='Application', operate="trial listening",
|
||||
get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')))
|
||||
def post(self, request: Request, application_id: str):
|
||||
byte_data = ApplicationSerializer.Operate(
|
||||
data={'application_id': application_id, 'user_id': request.user.id}).play_demo_text(request.data)
|
||||
return HttpResponse(byte_data, status=200, headers={'Content-Type': 'audio/mp3',
|
||||
'Content-Disposition': 'attachment; filename="abc.mp3"'})
|
||||
|
||||
class McpServers(APIView):
|
||||
authentication_classes = [TokenAuth]
|
||||
|
||||
@action(methods=['GET'], detail=False)
|
||||
@has_permissions(PermissionConstants.APPLICATION_READ, compare=CompareConstants.AND)
|
||||
def get(self, request: Request):
|
||||
return result.success(ApplicationSerializer.McpServers(
|
||||
data={'mcp_servers': request.query_params.get('mcp_servers')}).get_mcp_servers())
|
||||
|
|
|
|||
|
|
@ -8,10 +8,8 @@
|
|||
"""
|
||||
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
from drf_yasg import openapi
|
||||
from drf_yasg.utils import swagger_auto_schema
|
||||
from rest_framework.decorators import action
|
||||
from rest_framework.parsers import MultiPartParser
|
||||
from rest_framework.request import Request
|
||||
from rest_framework.views import APIView
|
||||
|
||||
|
|
@ -19,12 +17,10 @@ from application.serializers.chat_message_serializers import ChatMessageSerializ
|
|||
from application.serializers.chat_serializers import ChatSerializers, ChatRecordSerializer
|
||||
from application.swagger_api.chat_api import ChatApi, VoteApi, ChatRecordApi, ImproveApi, ChatRecordImproveApi, \
|
||||
ChatClientHistoryApi, OpenAIChatApi
|
||||
from application.views import get_application_operation_object
|
||||
from common.auth import TokenAuth, has_permissions, OpenAIKeyAuth
|
||||
from common.constants.authentication_type import AuthenticationType
|
||||
from common.constants.permission_constants import Permission, Group, Operate, \
|
||||
RoleConstants, ViewPermission, CompareConstants
|
||||
from common.log.log import log
|
||||
from common.response import result
|
||||
from common.util.common import query_params_to_single_dict
|
||||
from dataset.serializers.file_serializers import FileSerializer
|
||||
|
|
@ -37,7 +33,6 @@ class Openai(APIView):
|
|||
@swagger_auto_schema(operation_summary=_("OpenAI Interface Dialogue"),
|
||||
operation_id=_("OpenAI Interface Dialogue"),
|
||||
request_body=OpenAIChatApi.get_request_body_api(),
|
||||
responses=OpenAIChatApi.get_response_body_api(),
|
||||
tags=[_("OpenAI Dialogue")])
|
||||
def post(self, request: Request, application_id: str):
|
||||
return OpenAIChatSerializer(data={'application_id': application_id, 'client_id': request.auth.client_id,
|
||||
|
|
@ -61,8 +56,6 @@ class ChatView(APIView):
|
|||
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.USE,
|
||||
dynamic_tag=keywords.get('application_id'))])
|
||||
)
|
||||
@log(menu='Conversation Log', operate="Export conversation",
|
||||
get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')))
|
||||
def post(self, request: Request, application_id: str):
|
||||
return ChatSerializers.Query(
|
||||
data={**query_params_to_single_dict(request.query_params), 'application_id': application_id,
|
||||
|
|
@ -94,11 +87,10 @@ class ChatView(APIView):
|
|||
@swagger_auto_schema(operation_summary=_("Get the workflow temporary session id"),
|
||||
operation_id=_("Get the workflow temporary session id"),
|
||||
request_body=ChatApi.OpenWorkFlowTemp.get_request_body_api(),
|
||||
responses=result.get_api_response(ChatApi.OpenTempChat.get_response_body_api()),
|
||||
tags=[_("Application/Chat")])
|
||||
def post(self, request: Request):
|
||||
return result.success(ChatSerializers.OpenWorkFlowChat(
|
||||
data={'user_id': request.user.id, **request.data}).open())
|
||||
data={**request.data, 'user_id': request.user.id}).open())
|
||||
|
||||
class OpenTemp(APIView):
|
||||
authentication_classes = [TokenAuth]
|
||||
|
|
@ -107,7 +99,6 @@ class ChatView(APIView):
|
|||
@swagger_auto_schema(operation_summary=_("Get a temporary session id"),
|
||||
operation_id=_("Get a temporary session id"),
|
||||
request_body=ChatApi.OpenTempChat.get_request_body_api(),
|
||||
responses=result.get_api_response(ChatApi.OpenTempChat.get_response_body_api()),
|
||||
tags=[_("Application/Chat")])
|
||||
@has_permissions(RoleConstants.ADMIN, RoleConstants.USER)
|
||||
def post(self, request: Request):
|
||||
|
|
@ -146,8 +137,6 @@ class ChatView(APIView):
|
|||
'document_list') if 'document_list' in request.data else [],
|
||||
'audio_list': request.data.get(
|
||||
'audio_list') if 'audio_list' in request.data else [],
|
||||
'other_list': request.data.get(
|
||||
'other_list') if 'other_list' in request.data else [],
|
||||
'client_type': request.auth.client_type,
|
||||
'node_id': request.data.get('node_id', None),
|
||||
'runtime_node_id': request.data.get('runtime_node_id', None),
|
||||
|
|
@ -161,7 +150,7 @@ class ChatView(APIView):
|
|||
operation_id=_("Get the conversation list"),
|
||||
manual_parameters=ChatApi.get_request_params_api(),
|
||||
responses=result.get_api_array_response(ChatApi.get_response_body_api()),
|
||||
tags=[_("Application/Conversation Log")]
|
||||
tags=[_("Application/Conversation Log")]
|
||||
)
|
||||
@has_permissions(
|
||||
ViewPermission([RoleConstants.ADMIN, RoleConstants.USER, RoleConstants.APPLICATION_KEY],
|
||||
|
|
@ -186,8 +175,6 @@ class ChatView(APIView):
|
|||
dynamic_tag=keywords.get('application_id'))],
|
||||
compare=CompareConstants.AND),
|
||||
compare=CompareConstants.AND)
|
||||
@log(menu='Conversation Log', operate="Delete a conversation",
|
||||
get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')))
|
||||
def delete(self, request: Request, application_id: str, chat_id: str):
|
||||
return result.success(
|
||||
ChatSerializers.Operate(
|
||||
|
|
@ -229,34 +216,12 @@ class ChatView(APIView):
|
|||
dynamic_tag=keywords.get('application_id'))],
|
||||
compare=CompareConstants.AND),
|
||||
compare=CompareConstants.AND)
|
||||
@log(menu='Conversation Log', operate="Client deletes conversation",
|
||||
get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')))
|
||||
def delete(self, request: Request, application_id: str, chat_id: str):
|
||||
return result.success(
|
||||
ChatSerializers.Operate(
|
||||
data={'application_id': application_id, 'user_id': request.user.id,
|
||||
'chat_id': chat_id}).logic_delete())
|
||||
|
||||
@action(methods=['PUT'], detail=False)
|
||||
@swagger_auto_schema(operation_summary=_("Client modifies dialogue summary"),
|
||||
operation_id=_("Client modifies dialogue summary"),
|
||||
request_body=ChatClientHistoryApi.Operate.ReAbstract.get_request_body_api(),
|
||||
responses=result.get_default_response(),
|
||||
tags=[_("Application/Conversation Log")])
|
||||
@has_permissions(ViewPermission(
|
||||
[RoleConstants.APPLICATION_ACCESS_TOKEN, RoleConstants.ADMIN, RoleConstants.USER],
|
||||
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.USE,
|
||||
dynamic_tag=keywords.get('application_id'))],
|
||||
compare=CompareConstants.AND),
|
||||
compare=CompareConstants.AND)
|
||||
@log(menu='Conversation Log', operate="Client modifies dialogue summary",
|
||||
get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')))
|
||||
def put(self, request: Request, application_id: str, chat_id: str):
|
||||
return result.success(
|
||||
ChatSerializers.Operate(
|
||||
data={'application_id': application_id, 'user_id': request.user.id,
|
||||
'chat_id': chat_id}).re_abstract(request.data))
|
||||
|
||||
class Page(APIView):
|
||||
authentication_classes = [TokenAuth]
|
||||
|
||||
|
|
@ -359,8 +324,6 @@ class ChatView(APIView):
|
|||
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.USE,
|
||||
dynamic_tag=keywords.get('application_id'))])
|
||||
)
|
||||
@log(menu='Conversation Log', operate="Like, Dislike",
|
||||
get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')))
|
||||
def put(self, request: Request, application_id: str, chat_id: str, chat_record_id: str):
|
||||
return result.success(ChatRecordSerializer.Vote(
|
||||
data={'vote_status': request.data.get('vote_status'), 'chat_id': chat_id,
|
||||
|
|
@ -408,8 +371,6 @@ class ChatView(APIView):
|
|||
'dataset_id'))],
|
||||
compare=CompareConstants.AND
|
||||
), compare=CompareConstants.AND)
|
||||
@log(menu='Conversation Log', operate="Annotation",
|
||||
get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')))
|
||||
def put(self, request: Request, application_id: str, chat_id: str, chat_record_id: str, dataset_id: str,
|
||||
document_id: str):
|
||||
return result.success(ChatRecordSerializer.Improve(
|
||||
|
|
@ -421,7 +382,6 @@ class ChatView(APIView):
|
|||
operation_id=_("Add to Knowledge Base"),
|
||||
manual_parameters=ImproveApi.get_request_params_api_post(),
|
||||
request_body=ImproveApi.get_request_body_api_post(),
|
||||
responses=result.get_default_response(),
|
||||
tags=[_("Application/Conversation Log/Add to Knowledge Base")]
|
||||
)
|
||||
@has_permissions(
|
||||
|
|
@ -436,8 +396,6 @@ class ChatView(APIView):
|
|||
'dataset_id'))],
|
||||
compare=CompareConstants.AND
|
||||
), compare=CompareConstants.AND)
|
||||
@log(menu='Conversation Log', operate="Add to Knowledge Base",
|
||||
get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')))
|
||||
def post(self, request: Request, application_id: str, dataset_id: str):
|
||||
return result.success(ChatRecordSerializer.PostImprove().post_improve(request.data))
|
||||
|
||||
|
|
@ -463,8 +421,6 @@ class ChatView(APIView):
|
|||
'dataset_id'))],
|
||||
compare=CompareConstants.AND
|
||||
), compare=CompareConstants.AND)
|
||||
@log(menu='Conversation Log', operate="Delete a Annotation",
|
||||
get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')))
|
||||
def delete(self, request: Request, application_id: str, chat_id: str, chat_record_id: str,
|
||||
dataset_id: str,
|
||||
document_id: str, paragraph_id: str):
|
||||
|
|
@ -475,28 +431,11 @@ class ChatView(APIView):
|
|||
|
||||
class UploadFile(APIView):
|
||||
authentication_classes = [TokenAuth]
|
||||
parser_classes = [MultiPartParser]
|
||||
|
||||
@action(methods=['POST'], detail=False)
|
||||
@swagger_auto_schema(operation_summary=_("Upload files"),
|
||||
operation_id=_("Upload files"),
|
||||
manual_parameters=[
|
||||
openapi.Parameter(name='application_id',
|
||||
in_=openapi.IN_PATH,
|
||||
type=openapi.TYPE_STRING,
|
||||
required=True,
|
||||
description=_('Application ID')),
|
||||
openapi.Parameter(name='chat_id',
|
||||
in_=openapi.IN_PATH,
|
||||
type=openapi.TYPE_STRING,
|
||||
required=True,
|
||||
description=_('Conversation ID')),
|
||||
openapi.Parameter(name='file',
|
||||
in_=openapi.IN_FORM,
|
||||
type=openapi.TYPE_FILE,
|
||||
required=True,
|
||||
description=_('Upload file'))
|
||||
],
|
||||
manual_parameters=ChatRecordApi.get_request_params_api(),
|
||||
tags=[_("Application/Conversation Log")]
|
||||
)
|
||||
@has_permissions(
|
||||
|
|
|
|||
|
|
@ -1,21 +0,0 @@
|
|||
# coding=utf-8
|
||||
"""
|
||||
@project: MaxKB
|
||||
@Author：虎
|
||||
@file： common.py
|
||||
@date：2025/3/25 16:56
|
||||
@desc:
|
||||
"""
|
||||
|
||||
from django.db.models import QuerySet
|
||||
|
||||
from application.models import Application
|
||||
|
||||
|
||||
def get_application_operation_object(application_id):
|
||||
application_model = QuerySet(model=Application).filter(id=application_id).first()
|
||||
if application_model is not None:
|
||||
return {
|
||||
"name": application_model.name
|
||||
}
|
||||
return {}
|
||||
|
|
@ -11,50 +11,35 @@ import time
|
|||
|
||||
from common.cache.mem_cache import MemCache
|
||||
|
||||
_lock = threading.Lock()
|
||||
locks = {}
|
||||
lock = threading.Lock()
|
||||
|
||||
|
||||
class ModelManage:
|
||||
cache = MemCache('model', {})
|
||||
up_clear_time = time.time()
|
||||
|
||||
@staticmethod
|
||||
def _get_lock(_id):
|
||||
lock = locks.get(_id)
|
||||
if lock is None:
|
||||
with _lock:
|
||||
lock = locks.get(_id)
|
||||
if lock is None:
|
||||
lock = threading.Lock()
|
||||
locks[_id] = lock
|
||||
|
||||
return lock
|
||||
|
||||
@staticmethod
|
||||
def get_model(_id, get_model):
|
||||
model_instance = ModelManage.cache.get(_id)
|
||||
if model_instance is None:
|
||||
lock = ModelManage._get_lock(_id)
|
||||
with lock:
|
||||
model_instance = ModelManage.cache.get(_id)
|
||||
if model_instance is None:
|
||||
model_instance = get_model(_id)
|
||||
ModelManage.cache.set(_id, model_instance, timeout=60 * 60 * 8)
|
||||
else:
|
||||
if model_instance.is_cache_model():
|
||||
ModelManage.cache.touch(_id, timeout=60 * 60 * 8)
|
||||
else:
|
||||
# 获取锁
|
||||
lock.acquire()
|
||||
try:
|
||||
model_instance = ModelManage.cache.get(_id)
|
||||
if model_instance is None or not model_instance.is_cache_model():
|
||||
model_instance = get_model(_id)
|
||||
ModelManage.cache.set(_id, model_instance, timeout=60 * 60 * 8)
|
||||
ModelManage.clear_timeout_cache()
|
||||
return model_instance
|
||||
ModelManage.cache.set(_id, model_instance, timeout=60 * 30)
|
||||
return model_instance
|
||||
# 续期
|
||||
ModelManage.cache.touch(_id, timeout=60 * 30)
|
||||
ModelManage.clear_timeout_cache()
|
||||
return model_instance
|
||||
finally:
|
||||
# 释放锁
|
||||
lock.release()
|
||||
|
||||
@staticmethod
|
||||
def clear_timeout_cache():
|
||||
if time.time() - ModelManage.up_clear_time > 60 * 60:
|
||||
threading.Thread(target=lambda: ModelManage.cache.clear_timeout_data()).start()
|
||||
ModelManage.up_clear_time = time.time()
|
||||
if time.time() - ModelManage.up_clear_time > 60:
|
||||
ModelManage.cache.clear_timeout_data()
|
||||
|
||||
@staticmethod
|
||||
def delete_key(_id):
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
@date：2023/9/5 14:01
|
||||
@desc: 用于swagger 分组
|
||||
"""
|
||||
from drf_yasg.generators import OpenAPISchemaGenerator
|
||||
|
||||
from drf_yasg.inspectors import SwaggerAutoSchema
|
||||
|
||||
tags_dict = {
|
||||
|
|
@ -20,10 +20,10 @@ class CustomSwaggerAutoSchema(SwaggerAutoSchema):
|
|||
if "api" in tags and operation_keys:
|
||||
return [tags_dict.get(operation_keys[1]) if operation_keys[1] in tags_dict else operation_keys[1]]
|
||||
return tags
|
||||
|
||||
|
||||
class CustomOpenAPISchemaGenerator(OpenAPISchemaGenerator):
|
||||
def get_schema(self, request=None, public=False):
|
||||
schema = super().get_schema(request, public)
|
||||
schema.schemes = ['https', 'http']
|
||||
return schema
|
||||
if request.is_secure():
|
||||
schema.schemes = ['https']
|
||||
else:
|
||||
schema.schemes = ['http']
|
||||
return schema
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
# coding=utf-8
|
||||
"""
|
||||
@project: MaxKB
|
||||
@Author：虎
|
||||
@file： SystemEncoder.py
|
||||
@date：2025/3/17 16:38
|
||||
@desc:
|
||||
"""
|
||||
import datetime
|
||||
import decimal
|
||||
import json
|
||||
import uuid
|
||||
|
||||
from django.core.files.uploadedfile import InMemoryUploadedFile, TemporaryUploadedFile
|
||||
|
||||
|
||||
class SystemEncoder(json.JSONEncoder):
|
||||
def default(self, obj):
|
||||
if isinstance(obj, uuid.UUID):
|
||||
return str(obj)
|
||||
if isinstance(obj, datetime.datetime):
|
||||
return obj.strftime("%Y-%m-%d %H:%M:%S")
|
||||
if isinstance(obj, decimal.Decimal):
|
||||
return float(obj)
|
||||
if isinstance(obj, InMemoryUploadedFile):
|
||||
return {'name': obj.name, 'size': obj.size}
|
||||
if isinstance(obj, TemporaryUploadedFile):
|
||||
return {'name': obj.name, 'size': obj.size}
|
||||
else:
|
||||
return json.JSONEncoder.default(self, obj)
|
||||
|
|
@ -12,22 +12,15 @@ from .listener_manage import *
|
|||
from django.utils.translation import gettext as _
|
||||
|
||||
from ..db.sql_execute import update_execute
|
||||
from common.lock.impl.file_lock import FileLock
|
||||
|
||||
lock = FileLock()
|
||||
update_document_status_sql = """
|
||||
UPDATE "public"."document"
|
||||
SET status ="replace"("replace"("replace"(status, '1', '3'), '0', '3'), '4', '3')
|
||||
WHERE status ~ '1|0|4'
|
||||
"""
|
||||
|
||||
|
||||
def run():
|
||||
if lock.try_lock('event_init', 30 * 30):
|
||||
try:
|
||||
QuerySet(Model).filter(status=setting.models.Status.DOWNLOAD).update(status=setting.models.Status.ERROR,
|
||||
meta={'message': _(
|
||||
'The download process was interrupted, please try again')})
|
||||
update_execute(update_document_status_sql, [])
|
||||
finally:
|
||||
lock.un_lock('event_init')
|
||||
# QuerySet(Document).filter(status__in=[Status.embedding, Status.queue_up]).update(**{'status': Status.error})
|
||||
QuerySet(Model).filter(status=setting.models.Status.DOWNLOAD).update(status=setting.models.Status.ERROR,
|
||||
meta={'message': _('The download process was interrupted, please try again')})
|
||||
update_execute(update_document_status_sql, [])
|
||||
|
|
|
|||
|
|
@ -238,8 +238,11 @@ class ListenerManagement:
|
|||
for key in params_dict:
|
||||
_value_ = params_dict[key]
|
||||
exec_sql = exec_sql.replace(key, str(_value_))
|
||||
with lock:
|
||||
lock.acquire()
|
||||
try:
|
||||
native_update(query_set, exec_sql)
|
||||
finally:
|
||||
lock.release()
|
||||
|
||||
@staticmethod
|
||||
def embedding_by_document(document_id, embedding_model: Embeddings, state_list=None):
|
||||
|
|
@ -269,6 +272,9 @@ class ListenerManagement:
|
|||
ListenerManagement.update_status(QuerySet(Document).filter(id=document_id), TaskType.EMBEDDING,
|
||||
State.STARTED)
|
||||
|
||||
# 删除文档向量数据
|
||||
VectorStore.get_embedding_vector().delete_by_document_id(document_id)
|
||||
|
||||
# 根据段落进行向量化处理
|
||||
page_desc(QuerySet(Paragraph)
|
||||
.annotate(
|
||||
|
|
|
|||
|
|
@ -22,4 +22,3 @@ from .table_checkbox import *
|
|||
from .radio_card_field import *
|
||||
from .label import *
|
||||
from .slider_field import *
|
||||
from .switch_field import *
|
||||
|
|
|
|||
|
|
@ -28,6 +28,6 @@ class SwitchField(BaseField):
|
|||
@param props_info:
|
||||
"""
|
||||
|
||||
super().__init__('SwitchInput', label, required, default_value, relation_show_field_dict,
|
||||
super().__init__('Switch', label, required, default_value, relation_show_field_dict,
|
||||
{},
|
||||
TriggerType.OPTION_LIST, attrs, props_info)
|
||||
|
|
|
|||
|
|
@ -110,47 +110,24 @@ def get_image_id_func():
|
|||
return get_image_id
|
||||
|
||||
|
||||
title_font_list = [
|
||||
[36, 100],
|
||||
[30, 36]
|
||||
]
|
||||
|
||||
|
||||
def get_title_level(paragraph: Paragraph):
|
||||
try:
|
||||
if paragraph.style is not None:
|
||||
psn = paragraph.style.name
|
||||
if psn.startswith('Heading') or psn.startswith('TOC 标题') or psn.startswith('标题'):
|
||||
return int(psn.replace("Heading ", '').replace('TOC 标题', '').replace('标题',
|
||||
''))
|
||||
if len(paragraph.runs) == 1:
|
||||
font_size = paragraph.runs[0].font.size
|
||||
pt = font_size.pt
|
||||
if pt >= 30:
|
||||
for _value, index in zip(title_font_list, range(len(title_font_list))):
|
||||
if pt >= _value[0] and pt < _value[1]:
|
||||
return index + 1
|
||||
except Exception as e:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
class DocSplitHandle(BaseSplitHandle):
|
||||
@staticmethod
|
||||
def paragraph_to_md(paragraph: Paragraph, doc: Document, images_list, get_image_id):
|
||||
try:
|
||||
title_level = get_title_level(paragraph)
|
||||
if title_level is not None:
|
||||
title = "".join(["#" for i in range(title_level)]) + " " + paragraph.text
|
||||
psn = paragraph.style.name
|
||||
if psn.startswith('Heading') or psn.startswith('TOC 标题') or psn.startswith('标题'):
|
||||
title = "".join(["#" for i in range(
|
||||
int(psn.replace("Heading ", '').replace('TOC 标题', '').replace('标题',
|
||||
'')))]) + " " + paragraph.text
|
||||
images = reduce(lambda x, y: [*x, *y],
|
||||
[get_paragraph_element_images(e, doc, images_list, get_image_id) for e in
|
||||
paragraph._element],
|
||||
[])
|
||||
|
||||
if len(images) > 0:
|
||||
return title + '\n' + images_to_string(images, doc, images_list, get_image_id) if len(
|
||||
paragraph.text) > 0 else images_to_string(images, doc, images_list, get_image_id)
|
||||
return title
|
||||
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
return paragraph.text
|
||||
|
|
|
|||
|
|
@ -173,15 +173,14 @@ class PdfSplitHandle(BaseSplitHandle):
|
|||
|
||||
# Null characters are not allowed.
|
||||
chapter_text = chapter_text.replace('\0', '')
|
||||
# 限制标题长度
|
||||
real_chapter_title = chapter_title[:256]
|
||||
|
||||
# 限制章节内容长度
|
||||
if 0 < limit < len(chapter_text):
|
||||
split_text = PdfSplitHandle.split_text(chapter_text, limit)
|
||||
for text in split_text:
|
||||
chapters.append({"title": real_chapter_title, "content": text})
|
||||
chapters.append({"title": chapter_title, "content": text})
|
||||
else:
|
||||
chapters.append({"title": real_chapter_title, "content": chapter_text if chapter_text else real_chapter_title})
|
||||
chapters.append({"title": chapter_title, "content": chapter_text if chapter_text else chapter_title})
|
||||
# 保存章节内容和章节标题
|
||||
return chapters
|
||||
|
||||
|
|
|
|||
|
|
@ -132,8 +132,7 @@ class ZipParseQAHandle(BaseParseQAHandle):
|
|||
files = zip_ref.namelist()
|
||||
# 读取压缩包中的文件内容
|
||||
for file in files:
|
||||
# 跳过 macOS 特有的元数据目录和文件
|
||||
if file.endswith('/') or file.startswith('__MACOSX'):
|
||||
if file.endswith('/'):
|
||||
continue
|
||||
with zip_ref.open(file) as f:
|
||||
# 对文件内容进行处理
|
||||
|
|
|
|||
|
|
@ -22,11 +22,8 @@ class OpenaiToResponse(BaseToResponse):
|
|||
def to_block_response(self, chat_id, chat_record_id, content, is_end, completion_tokens, prompt_tokens,
|
||||
other_params: dict = None,
|
||||
_status=status.HTTP_200_OK):
|
||||
if other_params is None:
|
||||
other_params = {}
|
||||
data = ChatCompletion(id=chat_record_id, choices=[
|
||||
BlockChoice(finish_reason='stop', index=0, chat_id=chat_id,
|
||||
answer_list=other_params.get('answer_list', ""),
|
||||
message=ChatCompletionMessage(role='assistant', content=content))],
|
||||
created=datetime.datetime.now().second, model='', object='chat.completion',
|
||||
usage=CompletionUsage(completion_tokens=completion_tokens,
|
||||
|
|
@ -35,16 +32,11 @@ class OpenaiToResponse(BaseToResponse):
|
|||
).dict()
|
||||
return JsonResponse(data=data, status=_status)
|
||||
|
||||
def to_stream_chunk_response(self, chat_id, chat_record_id, node_id, up_node_id_list, content, is_end,
|
||||
completion_tokens,
|
||||
def to_stream_chunk_response(self, chat_id, chat_record_id, node_id, up_node_id_list, content, is_end, completion_tokens,
|
||||
prompt_tokens, other_params: dict = None):
|
||||
if other_params is None:
|
||||
other_params = {}
|
||||
chunk = ChatCompletionChunk(id=chat_record_id, model='', object='chat.completion.chunk',
|
||||
created=datetime.datetime.now().second, choices=[
|
||||
Choice(delta=ChoiceDelta(content=content, reasoning_content=other_params.get('reasoning_content', ""),
|
||||
chat_id=chat_id),
|
||||
finish_reason='stop' if is_end else None,
|
||||
created=datetime.datetime.now().second,choices=[
|
||||
Choice(delta=ChoiceDelta(content=content, chat_id=chat_id), finish_reason='stop' if is_end else None,
|
||||
index=0)],
|
||||
usage=CompletionUsage(completion_tokens=completion_tokens,
|
||||
prompt_tokens=prompt_tokens,
|
||||
|
|
|
|||
|
|
@ -82,10 +82,7 @@ class XlsSplitHandle(BaseParseTableHandle):
|
|||
for row in data:
|
||||
# 将每个单元格中的内容替换换行符为 <br> 以保留原始格式
|
||||
md_table += '| ' + ' | '.join(
|
||||
[str(cell)
|
||||
.replace('\r\n', '<br>')
|
||||
.replace('\n', '<br>')
|
||||
if cell else '' for cell in row]) + ' |\n'
|
||||
[str(cell).replace('\n', '<br>') if cell else '' for cell in row]) + ' |\n'
|
||||
md_tables += md_table + '\n\n'
|
||||
|
||||
return md_tables
|
||||
|
|
|
|||
|
|
@ -19,24 +19,31 @@ class XlsxSplitHandle(BaseParseTableHandle):
|
|||
|
||||
def fill_merged_cells(self, sheet, image_dict):
|
||||
data = []
|
||||
|
||||
# 获取第一行作为标题行
|
||||
headers = [cell.value for cell in sheet[1]]
|
||||
|
||||
# 从第二行开始遍历每一行
|
||||
for row in sheet.iter_rows(values_only=False):
|
||||
row_data = []
|
||||
for row in sheet.iter_rows(min_row=2, values_only=False):
|
||||
row_data = {}
|
||||
for col_idx, cell in enumerate(row):
|
||||
cell_value = cell.value
|
||||
|
||||
# 如果单元格为空，并且该单元格在合并单元格内，获取合并单元格的值
|
||||
if cell_value is None:
|
||||
for merged_range in sheet.merged_cells.ranges:
|
||||
if cell.coordinate in merged_range:
|
||||
cell_value = sheet[merged_range.min_row][merged_range.min_col - 1].value
|
||||
break
|
||||
|
||||
image = image_dict.get(cell_value, None)
|
||||
if image is not None:
|
||||
cell_value = f''
|
||||
|
||||
# 使用标题作为键，单元格的值作为值存入字典
|
||||
row_data.insert(col_idx, cell_value)
|
||||
row_data[headers[col_idx]] = cell_value
|
||||
data.append(row_data)
|
||||
|
||||
for merged_range in sheet.merged_cells.ranges:
|
||||
cell_value = data[merged_range.min_row - 1][merged_range.min_col - 1]
|
||||
for row_index in range(merged_range.min_row, merged_range.max_row + 1):
|
||||
for col_index in range(merged_range.min_col, merged_range.max_col + 1):
|
||||
data[row_index - 1][col_index - 1] = cell_value
|
||||
return data
|
||||
|
||||
def handle(self, file, get_buffer, save_image):
|
||||
|
|
@ -53,13 +60,11 @@ class XlsxSplitHandle(BaseParseTableHandle):
|
|||
paragraphs = []
|
||||
ws = wb[sheetname]
|
||||
data = self.fill_merged_cells(ws, image_dict)
|
||||
if len(data) >= 2:
|
||||
head_list = data[0]
|
||||
for row_index in range(1, len(data)):
|
||||
row_output = "; ".join(
|
||||
[f"{head_list[col_index]}: {data[row_index][col_index]}" for col_index in
|
||||
range(0, len(data[row_index]))])
|
||||
paragraphs.append({'title': '', 'content': row_output})
|
||||
|
||||
for row in data:
|
||||
row_output = "; ".join([f"{key}: {value}" for key, value in row.items()])
|
||||
# print(row_output)
|
||||
paragraphs.append({'title': '', 'content': row_output})
|
||||
|
||||
result.append({'name': sheetname, 'paragraphs': paragraphs})
|
||||
|
||||
|
|
@ -68,6 +73,7 @@ class XlsxSplitHandle(BaseParseTableHandle):
|
|||
return [{'name': file.name, 'paragraphs': []}]
|
||||
return result
|
||||
|
||||
|
||||
def get_content(self, file, save_image):
|
||||
try:
|
||||
# 加载 Excel 文件
|
||||
|
|
@ -83,18 +89,18 @@ class XlsxSplitHandle(BaseParseTableHandle):
|
|||
# 如果未指定 sheet_name，则使用第一个工作表
|
||||
for sheetname in workbook.sheetnames:
|
||||
sheet = workbook[sheetname] if sheetname else workbook.active
|
||||
data = self.fill_merged_cells(sheet, image_dict)
|
||||
if len(data) == 0:
|
||||
rows = self.fill_merged_cells(sheet, image_dict)
|
||||
if len(rows) == 0:
|
||||
continue
|
||||
# 提取表头和内容
|
||||
|
||||
headers = [f"{value}" for value in data[0]]
|
||||
headers = [f"{key}" for key, value in rows[0].items()]
|
||||
|
||||
# 构建 Markdown 表格
|
||||
md_table = '| ' + ' | '.join(headers) + ' |\n'
|
||||
md_table += '| ' + ' | '.join(['---'] * len(headers)) + ' |\n'
|
||||
for row_index in range(1, len(data)):
|
||||
r = [f'{value}' for value in data[row_index]]
|
||||
for row in rows:
|
||||
r = [f'{value}' for key, value in row.items()]
|
||||
md_table += '| ' + ' | '.join(
|
||||
[str(cell).replace('\n', '<br>') if cell is not None else '' for cell in r]) + ' |\n'
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from common.handle.base_split_handle import BaseSplitHandle
|
|||
|
||||
|
||||
def post_cell(cell_value):
|
||||
return cell_value.replace('\r\n', '<br>').replace('\n', '<br>').replace('|', '|')
|
||||
return cell_value.replace('\n', '<br>').replace('|', '|')
|
||||
|
||||
|
||||
def row_to_md(row):
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ import zipfile
|
|||
from typing import List
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from charset_normalizer import detect
|
||||
from django.db.models import QuerySet
|
||||
|
||||
from common.handle.base_split_handle import BaseSplitHandle
|
||||
|
|
@ -29,7 +28,6 @@ from common.util.common import parse_md_image
|
|||
from dataset.models import Image
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
|
||||
class FileBufferHandle:
|
||||
buffer = None
|
||||
|
||||
|
|
@ -77,7 +75,6 @@ def get_image_list(result_list: list, zip_files: List[str]):
|
|||
if search:
|
||||
new_image_id = str(uuid.uuid1())
|
||||
source_image_path = search.group().replace('(', '').replace(')', '')
|
||||
source_image_path = source_image_path.strip().split(" ")[0]
|
||||
image_path = urljoin(result.get('name'), '.' + source_image_path if source_image_path.startswith(
|
||||
'/') else source_image_path)
|
||||
if not zip_files.__contains__(image_path):
|
||||
|
|
@ -101,15 +98,6 @@ def get_image_list(result_list: list, zip_files: List[str]):
|
|||
return image_file_list
|
||||
|
||||
|
||||
def get_file_name(file_name):
|
||||
try:
|
||||
file_name_code = file_name.encode('cp437')
|
||||
charset = detect(file_name_code)['encoding']
|
||||
return file_name_code.decode(charset)
|
||||
except Exception as e:
|
||||
return file_name
|
||||
|
||||
|
||||
def filter_image_file(result_list: list, image_list):
|
||||
image_source_file_list = [image.get('source_file') for image in image_list]
|
||||
return [r for r in result_list if not image_source_file_list.__contains__(r.get('name', ''))]
|
||||
|
|
@ -126,13 +114,11 @@ class ZipSplitHandle(BaseSplitHandle):
|
|||
files = zip_ref.namelist()
|
||||
# 读取压缩包中的文件内容
|
||||
for file in files:
|
||||
if file.endswith('/') or file.startswith('__MACOSX'):
|
||||
if file.endswith('/'):
|
||||
continue
|
||||
with zip_ref.open(file) as f:
|
||||
# 对文件内容进行处理
|
||||
try:
|
||||
# 处理一下文件名
|
||||
f.name = get_file_name(f.name)
|
||||
value = file_to_paragraph(f, pattern_list, with_filter, limit)
|
||||
if isinstance(value, list):
|
||||
result = [*result, *value]
|
||||
|
|
|
|||
|
|
@ -1,100 +0,0 @@
|
|||
# coding=utf-8
|
||||
"""
|
||||
@project: MaxKB
|
||||
@Author：虎
|
||||
@file： log.py
|
||||
@date：2025/3/14 16:09
|
||||
@desc:
|
||||
"""
|
||||
from gettext import gettext
|
||||
|
||||
from setting.models.log_management import Log
|
||||
|
||||
|
||||
def _get_ip_address(request):
|
||||
"""
|
||||
获取ip地址
|
||||
@param request:
|
||||
@return:
|
||||
"""
|
||||
x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
|
||||
if x_forwarded_for:
|
||||
ip = x_forwarded_for.split(',')[0]
|
||||
else:
|
||||
ip = request.META.get('REMOTE_ADDR')
|
||||
return ip
|
||||
|
||||
|
||||
def _get_user(request):
|
||||
"""
|
||||
获取用户
|
||||
@param request:
|
||||
@return:
|
||||
"""
|
||||
user = request.user
|
||||
if user is None:
|
||||
return {
|
||||
|
||||
}
|
||||
return {
|
||||
"id": str(user.id),
|
||||
"email": user.email,
|
||||
"phone": user.phone,
|
||||
"nick_name": user.nick_name,
|
||||
"username": user.username,
|
||||
"role": user.role,
|
||||
}
|
||||
|
||||
|
||||
def _get_details(request):
|
||||
path = request.path
|
||||
body = request.data
|
||||
query = request.query_params
|
||||
return {
|
||||
'path': path,
|
||||
'body': body,
|
||||
'query': query
|
||||
}
|
||||
|
||||
|
||||
def log(menu: str, operate, get_user=_get_user, get_ip_address=_get_ip_address, get_details=_get_details,
|
||||
get_operation_object=None):
|
||||
"""
|
||||
记录审计日志
|
||||
@param menu: 操作菜单 str
|
||||
@param operate: 操作 str|func 如果是一个函数 入参将是一个request 响应为str def operate(request): return "操作菜单"
|
||||
@param get_user: 获取用户
|
||||
@param get_ip_address:获取IP地址
|
||||
@param get_details: 获取执行详情
|
||||
@param get_operation_object: 获取操作对象
|
||||
@return:
|
||||
"""
|
||||
|
||||
def inner(func):
|
||||
def run(view, request, **kwargs):
|
||||
status = 200
|
||||
operation_object = {}
|
||||
try:
|
||||
if get_operation_object is not None:
|
||||
operation_object = get_operation_object(request, kwargs)
|
||||
except Exception as e:
|
||||
pass
|
||||
try:
|
||||
return func(view, request, **kwargs)
|
||||
except Exception as e:
|
||||
status = 500
|
||||
raise e
|
||||
finally:
|
||||
ip = get_ip_address(request)
|
||||
user = get_user(request)
|
||||
details = get_details(request)
|
||||
_operate = operate
|
||||
if callable(operate):
|
||||
_operate = operate(request)
|
||||
# 插入审计日志
|
||||
Log(menu=menu, operate=_operate, user=user, status=status, ip_address=ip, details=details,
|
||||
operation_object=operation_object).save()
|
||||
|
||||
return run
|
||||
|
||||
return inner
|
||||
|
|
@ -24,13 +24,12 @@ class GunicornLocalModelService(BaseService):
|
|||
os.environ.setdefault('SERVER_NAME', 'local_model')
|
||||
log_format = '%(h)s %(t)s %(L)ss "%(r)s" %(s)s %(b)s '
|
||||
bind = f'{CONFIG.get("LOCAL_MODEL_HOST")}:{CONFIG.get("LOCAL_MODEL_PORT")}'
|
||||
worker = CONFIG.get("LOCAL_MODEL_HOST_WORKER", 1)
|
||||
cmd = [
|
||||
'gunicorn', 'smartdoc.wsgi:application',
|
||||
'-b', bind,
|
||||
'-k', 'gthread',
|
||||
'--threads', '200',
|
||||
'-w', str(worker),
|
||||
'-w', "1",
|
||||
'--max-requests', '10240',
|
||||
'--max-requests-jitter', '2048',
|
||||
'--access-logformat', log_format,
|
||||
|
|
|
|||
|
|
@ -1,62 +0,0 @@
|
|||
# coding=utf-8
|
||||
"""
|
||||
@project: maxkb
|
||||
@Author：虎
|
||||
@file： static_headers_middleware.py
|
||||
@date：2024/3/13 18:26
|
||||
@desc:
|
||||
"""
|
||||
from django.http import HttpResponse
|
||||
from django.utils.deprecation import MiddlewareMixin
|
||||
|
||||
content = """
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Document</title>
|
||||
<script>
|
||||
window.onload = () => {
|
||||
var xhr = new XMLHttpRequest()
|
||||
xhr.open('GET', '/api/user', true)
|
||||
|
||||
xhr.setRequestHeader('Content-Type', 'application/json')
|
||||
const token = localStorage.getItem('token')
|
||||
const pathname = window.location.pathname
|
||||
if (token) {
|
||||
xhr.setRequestHeader('Authorization', token)
|
||||
xhr.onreadystatechange = function () {
|
||||
if (xhr.readyState === 4) {
|
||||
if (xhr.status === 200) {
|
||||
window.location.href = pathname
|
||||
}
|
||||
if (xhr.status === 401) {
|
||||
window.location.href = '/ui/login'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
xhr.send()
|
||||
} else {
|
||||
window.location.href = '/ui/login'
|
||||
}
|
||||
}
|
||||
</script>
|
||||
</head>
|
||||
<body></body>
|
||||
</html>
|
||||
|
||||
"""
|
||||
|
||||
|
||||
class DocHeadersMiddleware(MiddlewareMixin):
|
||||
def process_response(self, request, response):
|
||||
if request.path.startswith('/doc/') or request.path.startswith('/doc/chat/'):
|
||||
HTTP_REFERER = request.META.get('HTTP_REFERER')
|
||||
if HTTP_REFERER is None:
|
||||
return HttpResponse(content)
|
||||
if HTTP_REFERER == request._current_scheme_host + request.path:
|
||||
return response
|
||||
return response
|
||||
|
|
@ -1,84 +0,0 @@
|
|||
# coding=utf-8
|
||||
"""
|
||||
@project: MaxKB
|
||||
@Author：虎
|
||||
@file： gzip.py
|
||||
@date：2025/2/27 10:03
|
||||
@desc:
|
||||
"""
|
||||
from django.utils.cache import patch_vary_headers
|
||||
from django.utils.deprecation import MiddlewareMixin
|
||||
from django.utils.regex_helper import _lazy_re_compile
|
||||
from django.utils.text import compress_sequence, compress_string
|
||||
|
||||
re_accepts_gzip = _lazy_re_compile(r"\bgzip\b")
|
||||
|
||||
|
||||
class GZipMiddleware(MiddlewareMixin):
|
||||
"""
|
||||
Compress content if the browser allows gzip compression.
|
||||
Set the Vary header accordingly, so that caches will base their storage
|
||||
on the Accept-Encoding header.
|
||||
"""
|
||||
|
||||
max_random_bytes = 100
|
||||
|
||||
def process_response(self, request, response):
|
||||
if request.method != 'GET' or request.path.startswith('/api'):
|
||||
return response
|
||||
# It's not worth attempting to compress really short responses.
|
||||
if not response.streaming and len(response.content) < 200:
|
||||
return response
|
||||
|
||||
# Avoid gzipping if we've already got a content-encoding.
|
||||
if response.has_header("Content-Encoding"):
|
||||
return response
|
||||
|
||||
patch_vary_headers(response, ("Accept-Encoding",))
|
||||
|
||||
ae = request.META.get("HTTP_ACCEPT_ENCODING", "")
|
||||
if not re_accepts_gzip.search(ae):
|
||||
return response
|
||||
|
||||
if response.streaming:
|
||||
if response.is_async:
|
||||
# pull to lexical scope to capture fixed reference in case
|
||||
# streaming_content is set again later.
|
||||
original_iterator = response.streaming_content
|
||||
|
||||
async def gzip_wrapper():
|
||||
async for chunk in original_iterator:
|
||||
yield compress_string(
|
||||
chunk,
|
||||
max_random_bytes=self.max_random_bytes,
|
||||
)
|
||||
|
||||
response.streaming_content = gzip_wrapper()
|
||||
else:
|
||||
response.streaming_content = compress_sequence(
|
||||
response.streaming_content,
|
||||
max_random_bytes=self.max_random_bytes,
|
||||
)
|
||||
# Delete the `Content-Length` header for streaming content, because
|
||||
# we won't know the compressed size until we stream it.
|
||||
del response.headers["Content-Length"]
|
||||
else:
|
||||
# Return the compressed content only if it's actually shorter.
|
||||
compressed_content = compress_string(
|
||||
response.content,
|
||||
max_random_bytes=self.max_random_bytes,
|
||||
)
|
||||
if len(compressed_content) >= len(response.content):
|
||||
return response
|
||||
response.content = compressed_content
|
||||
response.headers["Content-Length"] = str(len(response.content))
|
||||
|
||||
# If there is a strong ETag, make it weak to fulfill the requirements
|
||||
# of RFC 9110 Section 8.8.1 while also allowing conditional request
|
||||
# matches on ETags.
|
||||
etag = response.get("ETag")
|
||||
if etag and etag.startswith('"'):
|
||||
response.headers["ETag"] = "W/" + etag
|
||||
response.headers["Content-Encoding"] = "gzip"
|
||||
|
||||
return response
|
||||
|
|
@ -10,8 +10,6 @@ import hashlib
|
|||
import importlib
|
||||
import io
|
||||
import mimetypes
|
||||
import pickle
|
||||
import random
|
||||
import re
|
||||
import shutil
|
||||
from functools import reduce
|
||||
|
|
@ -25,51 +23,6 @@ from pydub import AudioSegment
|
|||
from ..exception.app_exception import AppApiException
|
||||
from ..models.db_model_manage import DBModelManage
|
||||
|
||||
safe_builtins = {
|
||||
'MKInstance'
|
||||
}
|
||||
|
||||
ALLOWED_CLASSES = {
|
||||
("builtins", "dict"),
|
||||
('uuid', 'UUID'),
|
||||
("application.serializers.application_serializers", "MKInstance"),
|
||||
("function_lib.serializers.function_lib_serializer", "FlibInstance")
|
||||
}
|
||||
|
||||
|
||||
class RestrictedUnpickler(pickle.Unpickler):
|
||||
|
||||
def find_class(self, module, name):
|
||||
if (module, name) in ALLOWED_CLASSES:
|
||||
return super().find_class(module, name)
|
||||
raise pickle.UnpicklingError("global '%s.%s' is forbidden" %
|
||||
(module, name))
|
||||
|
||||
|
||||
def restricted_loads(s):
|
||||
"""Helper function analogous to pickle.loads()."""
|
||||
return RestrictedUnpickler(io.BytesIO(s)).load()
|
||||
|
||||
|
||||
def encryption(message: str):
|
||||
"""
|
||||
加密敏感字段数据 加密方式是 如果密码是 1234567890 那么给前端则是 123******890
|
||||
:param message:
|
||||
:return:
|
||||
"""
|
||||
max_pre_len = 8
|
||||
max_post_len = 4
|
||||
message_len = len(message)
|
||||
pre_len = int(message_len / 5 * 2)
|
||||
post_len = int(message_len / 5 * 1)
|
||||
pre_str = "".join([message[index] for index in
|
||||
range(0, max_pre_len if pre_len > max_pre_len else 1 if pre_len <= 0 else int(pre_len))])
|
||||
end_str = "".join(
|
||||
[message[index] for index in
|
||||
range(message_len - (int(post_len) if pre_len < max_post_len else max_post_len), message_len)])
|
||||
content = "***************"
|
||||
return pre_str + content + end_str
|
||||
|
||||
|
||||
def sub_array(array: List, item_num=10):
|
||||
result = []
|
||||
|
|
@ -298,14 +251,3 @@ def markdown_to_plain_text(md: str) -> str:
|
|||
# 去除首尾空格
|
||||
text = text.strip()
|
||||
return text
|
||||
|
||||
|
||||
SAFE_CHAR_SET = (
|
||||
[chr(i) for i in range(65, 91) if chr(i) not in {'I', 'O'}] + # 大写字母 A-H, J-N, P-Z
|
||||
[chr(i) for i in range(97, 123) if chr(i) not in {'i', 'l', 'o'}] + # 小写字母 a-h, j-n, p-z
|
||||
[str(i) for i in range(10) if str(i) not in {'0', '1', '7'}] # 数字 2-6, 8-9
|
||||
)
|
||||
|
||||
|
||||
def get_random_chars(number=4):
|
||||
return ''.join(random.choices(SAFE_CHAR_SET, k=number))
|
||||
|
|
|
|||
|
|
@ -7,12 +7,13 @@
|
|||
@desc:
|
||||
"""
|
||||
import os
|
||||
import pickle
|
||||
import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
from textwrap import dedent
|
||||
|
||||
from diskcache import Cache
|
||||
|
||||
from smartdoc.const import BASE_DIR
|
||||
from smartdoc.const import PROJECT_DIR
|
||||
|
||||
|
|
@ -30,14 +31,12 @@ class FunctionExecutor:
|
|||
self.user = None
|
||||
self._createdir()
|
||||
if self.sandbox:
|
||||
os.system(f"chown -R {self.user}:root {self.sandbox_path}")
|
||||
os.system(f"chown -R {self.user}:{self.user} {self.sandbox_path}")
|
||||
|
||||
def _createdir(self):
|
||||
old_mask = os.umask(0o077)
|
||||
try:
|
||||
os.makedirs(self.sandbox_path, 0o700, exist_ok=True)
|
||||
os.makedirs(os.path.join(self.sandbox_path, 'execute'), 0o700, exist_ok=True)
|
||||
os.makedirs(os.path.join(self.sandbox_path, 'result'), 0o700, exist_ok=True)
|
||||
finally:
|
||||
os.umask(old_mask)
|
||||
|
||||
|
|
@ -45,11 +44,10 @@ class FunctionExecutor:
|
|||
_id = str(uuid.uuid1())
|
||||
success = '{"code":200,"msg":"成功","data":exec_result}'
|
||||
err = '{"code":500,"msg":str(e),"data":None}'
|
||||
result_path = f'{self.sandbox_path}/result/{_id}.result'
|
||||
path = r'' + self.sandbox_path + ''
|
||||
_exec_code = f"""
|
||||
try:
|
||||
import os
|
||||
import pickle
|
||||
env = dict(os.environ)
|
||||
for key in list(env.keys()):
|
||||
if key in os.environ and (key.startswith('MAXKB') or key.startswith('POSTGRES') or key.startswith('PG')):
|
||||
|
|
@ -62,11 +60,13 @@ try:
|
|||
for local in locals_v:
|
||||
globals_v[local] = locals_v[local]
|
||||
exec_result=f(**keywords)
|
||||
with open({result_path!a}, 'wb') as file:
|
||||
file.write(pickle.dumps({success}))
|
||||
from diskcache import Cache
|
||||
cache = Cache({path!a})
|
||||
cache.set({_id!a},{success})
|
||||
except Exception as e:
|
||||
with open({result_path!a}, 'wb') as file:
|
||||
file.write(pickle.dumps({err}))
|
||||
from diskcache import Cache
|
||||
cache = Cache({path!a})
|
||||
cache.set({_id!a},{err})
|
||||
"""
|
||||
if self.sandbox:
|
||||
subprocess_result = self._exec_sandbox(_exec_code, _id)
|
||||
|
|
@ -74,21 +74,21 @@ except Exception as e:
|
|||
subprocess_result = self._exec(_exec_code)
|
||||
if subprocess_result.returncode == 1:
|
||||
raise Exception(subprocess_result.stderr)
|
||||
with open(result_path, 'rb') as file:
|
||||
result = pickle.loads(file.read())
|
||||
os.remove(result_path)
|
||||
cache = Cache(self.sandbox_path)
|
||||
result = cache.get(_id)
|
||||
cache.delete(_id)
|
||||
if result.get('code') == 200:
|
||||
return result.get('data')
|
||||
raise Exception(result.get('msg'))
|
||||
|
||||
def _exec_sandbox(self, _code, _id):
|
||||
exec_python_file = f'{self.sandbox_path}/execute/{_id}.py'
|
||||
exec_python_file = f'{self.sandbox_path}/{_id}.py'
|
||||
with open(exec_python_file, 'w') as file:
|
||||
file.write(_code)
|
||||
os.system(f"chown {self.user}:root {exec_python_file}")
|
||||
os.system(f"chown {self.user}:{self.user} {exec_python_file}")
|
||||
kwargs = {'cwd': BASE_DIR}
|
||||
subprocess_result = subprocess.run(
|
||||
['su', '-s', python_directory, '-c', "exec(open('" + exec_python_file + "').read())", self.user],
|
||||
['su', '-c', python_directory + ' ' + exec_python_file, self.user],
|
||||
text=True,
|
||||
capture_output=True, **kwargs)
|
||||
os.remove(exec_python_file)
|
||||
|
|
|
|||
|
|
@ -40,12 +40,15 @@ def generate():
|
|||
def get_key_pair():
|
||||
rsa_value = rsa_cache.get(cache_key)
|
||||
if rsa_value is None:
|
||||
with lock:
|
||||
rsa_value = rsa_cache.get(cache_key)
|
||||
if rsa_value is not None:
|
||||
return rsa_value
|
||||
lock.acquire()
|
||||
rsa_value = rsa_cache.get(cache_key)
|
||||
if rsa_value is not None:
|
||||
return rsa_value
|
||||
try:
|
||||
rsa_value = get_key_pair_by_sql()
|
||||
rsa_cache.set(cache_key, rsa_value)
|
||||
finally:
|
||||
lock.release()
|
||||
return rsa_value
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -339,14 +339,13 @@ class SplitModel:
|
|||
for e in result:
|
||||
if len(e['content']) > 4096:
|
||||
pass
|
||||
title_list = list(set([row.get('title') for row in result]))
|
||||
return [item for item in [self.post_reset_paragraph(row, title_list) for row in result] if
|
||||
return [item for item in [self.post_reset_paragraph(row) for row in result] if
|
||||
'content' in item and len(item.get('content').strip()) > 0]
|
||||
|
||||
def post_reset_paragraph(self, paragraph: Dict, title_list: List[str]):
|
||||
result = self.content_is_null(paragraph, title_list)
|
||||
result = self.filter_title_special_characters(result)
|
||||
def post_reset_paragraph(self, paragraph: Dict):
|
||||
result = self.filter_title_special_characters(paragraph)
|
||||
result = self.sub_title(result)
|
||||
result = self.content_is_null(result)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -358,14 +357,11 @@ class SplitModel:
|
|||
return paragraph
|
||||
|
||||
@staticmethod
|
||||
def content_is_null(paragraph: Dict, title_list: List[str]):
|
||||
def content_is_null(paragraph: Dict):
|
||||
if 'title' in paragraph:
|
||||
title = paragraph.get('title')
|
||||
content = paragraph.get('content')
|
||||
if (content is None or len(content.strip()) == 0) and (title is not None and len(title) > 0):
|
||||
find = [t for t in title_list if t.__contains__(title) and t != title]
|
||||
if find:
|
||||
return {'title': '', 'content': ''}
|
||||
return {'title': '', 'content': title}
|
||||
return paragraph
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ from django.core import signing
|
|||
from django.core.cache import cache
|
||||
|
||||
# alg使用的算法
|
||||
HEADER = {'type': 'JWP', 'alg': 'default'}
|
||||
HEADER = {'typ': 'JWP', 'alg': 'default'}
|
||||
TOKEN_KEY = 'solomon_world_token'
|
||||
TOKEN_SALT = 'solomonwanc@gmail.com'
|
||||
TIME_OUT = 30 * 60
|
||||
|
|
|
|||
|
|
@ -12,6 +12,9 @@ from typing import List
|
|||
|
||||
import jieba
|
||||
import jieba.posseg
|
||||
from jieba import analyse
|
||||
|
||||
from common.util.split_model import group_by
|
||||
|
||||
jieba_word_list_cache = [chr(item) for item in range(38, 84)]
|
||||
|
||||
|
|
@ -77,12 +80,37 @@ def get_key_by_word_dict(key, word_dict):
|
|||
|
||||
|
||||
def to_ts_vector(text: str):
|
||||
# 获取不分词的数据
|
||||
word_list = get_word_list(text)
|
||||
# 获取关键词关系
|
||||
word_dict = to_word_dict(word_list, text)
|
||||
# 替换字符串
|
||||
text = replace_word(word_dict, text)
|
||||
# 分词
|
||||
result = jieba.lcut(text, cut_all=True)
|
||||
return " ".join(result)
|
||||
filter_word = jieba.analyse.extract_tags(text, topK=100)
|
||||
result = jieba.lcut(text, HMM=True, use_paddle=True)
|
||||
# 过滤标点符号
|
||||
result = [item for item in result if filter_word.__contains__(item) and len(item) < 10]
|
||||
result_ = [{'word': get_key_by_word_dict(result[index], word_dict), 'index': index} for index in
|
||||
range(len(result))]
|
||||
result_group = group_by(result_, lambda r: r['word'])
|
||||
return " ".join(
|
||||
[f"{key.lower()}:{','.join([str(item['index'] + 1) for item in result_group[key]][:20])}" for key in
|
||||
result_group if
|
||||
not remove_chars.__contains__(key) and len(key.strip()) >= 0])
|
||||
|
||||
|
||||
def to_query(text: str):
|
||||
extract_tags = jieba.lcut(text, cut_all=True)
|
||||
result = " ".join(extract_tags)
|
||||
# 获取不分词的数据
|
||||
word_list = get_word_list(text)
|
||||
# 获取关键词关系
|
||||
word_dict = to_word_dict(word_list, text)
|
||||
# 替换字符串
|
||||
text = replace_word(word_dict, text)
|
||||
extract_tags = analyse.extract_tags(text, topK=5, withWeight=True, allowPOS=('ns', 'n', 'vn', 'v', 'eng'))
|
||||
result = " ".join([get_key_by_word_dict(word, word_dict) for word, score in extract_tags if
|
||||
not remove_chars.__contains__(word)])
|
||||
# 删除词库
|
||||
for word in word_list:
|
||||
jieba.del_word(word)
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -82,9 +82,6 @@ class Type(models.TextChoices):
|
|||
|
||||
web = 1, 'web站点类型'
|
||||
|
||||
lark = 2, '飞书类型'
|
||||
yuque = 3, '语雀类型'
|
||||
|
||||
|
||||
class HitHandlingMethod(models.TextChoices):
|
||||
optimization = 'optimization', '模型优化'
|
||||
|
|
|
|||
|
|
@ -40,14 +40,6 @@ def zip_dir(zip_path, output=None):
|
|||
zip.close()
|
||||
|
||||
|
||||
def is_valid_uuid(s):
|
||||
try:
|
||||
uuid.UUID(s)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def write_image(zip_path: str, image_list: List[str]):
|
||||
for image in image_list:
|
||||
search = re.search("\(.*\)", image)
|
||||
|
|
@ -55,9 +47,6 @@ def write_image(zip_path: str, image_list: List[str]):
|
|||
text = search.group()
|
||||
if text.startswith('(/api/file/'):
|
||||
r = text.replace('(/api/file/', '').replace(')', '')
|
||||
r = r.strip().split(" ")[0]
|
||||
if not is_valid_uuid(r):
|
||||
break
|
||||
file = QuerySet(File).filter(id=r).first()
|
||||
if file is None:
|
||||
break
|
||||
|
|
@ -69,9 +58,6 @@ def write_image(zip_path: str, image_list: List[str]):
|
|||
f.write(file.get_byte())
|
||||
else:
|
||||
r = text.replace('(/api/image/', '').replace(')', '')
|
||||
r = r.strip().split(" ")[0]
|
||||
if not is_valid_uuid(r):
|
||||
break
|
||||
image_model = QuerySet(Image).filter(id=r).first()
|
||||
if image_model is None:
|
||||
break
|
||||
|
|
@ -222,26 +208,3 @@ def get_embedding_model_id_by_dataset_id_list(dataset_id_list: List):
|
|||
if len(dataset_list) == 0:
|
||||
raise Exception(_('Knowledge base setting error, please reset the knowledge base'))
|
||||
return str(dataset_list[0].embedding_mode_id)
|
||||
|
||||
|
||||
class GenerateRelatedSerializer(ApiMixin, serializers.Serializer):
|
||||
model_id = serializers.UUIDField(required=True, error_messages=ErrMessage.uuid(_('Model id')))
|
||||
prompt = serializers.CharField(required=True, error_messages=ErrMessage.uuid(_('Prompt word')))
|
||||
state_list = serializers.ListField(required=False, child=serializers.CharField(required=True),
|
||||
error_messages=ErrMessage.list("state list"))
|
||||
|
||||
@staticmethod
|
||||
def get_request_body_api():
|
||||
return openapi.Schema(
|
||||
type=openapi.TYPE_OBJECT,
|
||||
properties={
|
||||
'model_id': openapi.Schema(type=openapi.TYPE_STRING,
|
||||
title=_('Model id'),
|
||||
description=_('Model id')),
|
||||
'prompt': openapi.Schema(type=openapi.TYPE_STRING, title=_('Prompt word'),
|
||||
description=_("Prompt word")),
|
||||
'state_list': openapi.Schema(type=openapi.TYPE_ARRAY,
|
||||
items=openapi.Schema(type=openapi.TYPE_STRING),
|
||||
title=_('state list'))
|
||||
}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ from django.contrib.postgres.fields import ArrayField
|
|||
from django.core import validators
|
||||
from django.db import transaction, models
|
||||
from django.db.models import QuerySet
|
||||
from django.db.models.functions import Reverse, Substr
|
||||
from django.http import HttpResponse
|
||||
from drf_yasg import openapi
|
||||
from rest_framework import serializers
|
||||
|
|
@ -43,10 +42,9 @@ from common.util.split_model import get_split_model
|
|||
from dataset.models.data_set import DataSet, Document, Paragraph, Problem, Type, ProblemParagraphMapping, TaskType, \
|
||||
State, File, Image
|
||||
from dataset.serializers.common_serializers import list_paragraph, MetaSerializer, ProblemParagraphManage, \
|
||||
get_embedding_model_by_dataset_id, get_embedding_model_id_by_dataset_id, write_image, zip_dir, \
|
||||
GenerateRelatedSerializer
|
||||
get_embedding_model_by_dataset_id, get_embedding_model_id_by_dataset_id, write_image, zip_dir
|
||||
from dataset.serializers.document_serializers import DocumentSerializers, DocumentInstanceSerializer
|
||||
from dataset.task import sync_web_dataset, sync_replace_web_dataset, generate_related_by_dataset_id
|
||||
from dataset.task import sync_web_dataset, sync_replace_web_dataset
|
||||
from embedding.models import SearchMode
|
||||
from embedding.task import embedding_by_dataset, delete_embedding_by_dataset
|
||||
from setting.models import AuthOperate, Model
|
||||
|
|
@ -455,13 +453,11 @@ class DataSetSerializers(serializers.ModelSerializer):
|
|||
# 批量插入关联问题
|
||||
QuerySet(ProblemParagraphMapping).bulk_create(problem_paragraph_mapping_list) if len(
|
||||
problem_paragraph_mapping_list) > 0 else None
|
||||
|
||||
# 响应数据
|
||||
return {**DataSetSerializers(dataset).data,
|
||||
'user_id': user_id,
|
||||
'document_list': document_model_list,
|
||||
"document_count": len(document_model_list),
|
||||
"char_length": reduce(lambda x, y: x + y, [d.char_length for d in document_model_list],
|
||||
0)}, dataset_id
|
||||
'document_list': DocumentSerializers.Query(data={'dataset_id': dataset_id}).list(
|
||||
with_valid=True)}, dataset_id
|
||||
|
||||
@staticmethod
|
||||
def get_last_url_path(url):
|
||||
|
|
@ -571,13 +567,13 @@ class DataSetSerializers(serializers.ModelSerializer):
|
|||
id = serializers.CharField(required=True, error_messages=ErrMessage.char("id"))
|
||||
user_id = serializers.UUIDField(required=False, error_messages=ErrMessage.char(_('user id')))
|
||||
query_text = serializers.CharField(required=True, error_messages=ErrMessage.char(_('query text')))
|
||||
top_number = serializers.IntegerField(required=True, max_value=10000, min_value=1,
|
||||
top_number = serializers.IntegerField(required=True, max_value=100, min_value=1,
|
||||
error_messages=ErrMessage.char("top number"))
|
||||
similarity = serializers.FloatField(required=True, max_value=2, min_value=0,
|
||||
error_messages=ErrMessage.char(_('similarity')))
|
||||
search_mode = serializers.CharField(required=True, validators=[
|
||||
validators.RegexValidator(regex=re.compile("^embedding|keywords|blend$"),
|
||||
message=_('The type only supports embedding|keywords|blend'), code=500)
|
||||
message=_('The type only supports register|reset_password'), code=500)
|
||||
], error_messages=ErrMessage.char(_('search mode')))
|
||||
|
||||
def is_valid(self, *, raise_exception=True):
|
||||
|
|
@ -818,31 +814,6 @@ class DataSetSerializers(serializers.ModelSerializer):
|
|||
except AlreadyQueued as e:
|
||||
raise AppApiException(500, _('Failed to send the vectorization task, please try again later!'))
|
||||
|
||||
def generate_related(self, instance: Dict, with_valid=True):
|
||||
if with_valid:
|
||||
self.is_valid(raise_exception=True)
|
||||
GenerateRelatedSerializer(data=instance).is_valid(raise_exception=True)
|
||||
dataset_id = self.data.get('id')
|
||||
model_id = instance.get("model_id")
|
||||
prompt = instance.get("prompt")
|
||||
state_list = instance.get('state_list')
|
||||
ListenerManagement.update_status(QuerySet(Document).filter(dataset_id=dataset_id),
|
||||
TaskType.GENERATE_PROBLEM,
|
||||
State.PENDING)
|
||||
ListenerManagement.update_status(QuerySet(Paragraph).annotate(
|
||||
reversed_status=Reverse('status'),
|
||||
task_type_status=Substr('reversed_status', TaskType.GENERATE_PROBLEM.value,
|
||||
1),
|
||||
).filter(task_type_status__in=state_list, dataset_id=dataset_id)
|
||||
.values('id'),
|
||||
TaskType.GENERATE_PROBLEM,
|
||||
State.PENDING)
|
||||
ListenerManagement.get_aggregation_document_status_by_dataset_id(dataset_id)()
|
||||
try:
|
||||
generate_related_by_dataset_id.delay(dataset_id, model_id, prompt, state_list)
|
||||
except AlreadyQueued as e:
|
||||
raise AppApiException(500, _('Failed to send the vectorization task, please try again later!'))
|
||||
|
||||
def list_application(self, with_valid=True):
|
||||
if with_valid:
|
||||
self.is_valid(raise_exception=True)
|
||||
|
|
|
|||
|
|
@ -19,18 +19,16 @@ from typing import List, Dict
|
|||
import openpyxl
|
||||
from celery_once import AlreadyQueued
|
||||
from django.core import validators
|
||||
from django.db import transaction, models
|
||||
from django.db import transaction
|
||||
from django.db.models import QuerySet, Count
|
||||
from django.db.models.functions import Substr, Reverse
|
||||
from django.http import HttpResponse
|
||||
from django.utils.translation import get_language
|
||||
from django.utils.translation import gettext_lazy as _, gettext, to_locale
|
||||
from drf_yasg import openapi
|
||||
from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE
|
||||
from rest_framework import serializers
|
||||
from xlwt import Utils
|
||||
|
||||
from common.db.search import native_search, native_page_search, get_dynamics_model
|
||||
from common.db.search import native_search, native_page_search
|
||||
from common.event import ListenerManagement
|
||||
from common.event.common import work_thread_pool
|
||||
from common.exception.app_exception import AppApiException
|
||||
|
|
@ -66,6 +64,8 @@ from embedding.task.embedding import embedding_by_document, delete_embedding_by_
|
|||
embedding_by_document_list
|
||||
from setting.models import Model
|
||||
from smartdoc.conf import PROJECT_DIR
|
||||
from django.utils.translation import gettext_lazy as _, gettext, to_locale
|
||||
from django.utils.translation import get_language
|
||||
|
||||
parse_qa_handle_list = [XlsParseQAHandle(), CsvParseQAHandle(), XlsxParseQAHandle(), ZipParseQAHandle()]
|
||||
parse_table_handle_list = [CsvSplitTableHandle(), XlsSplitTableHandle(), XlsxSplitTableHandle()]
|
||||
|
|
@ -243,16 +243,13 @@ class DocumentSerializers(ApiMixin, serializers.Serializer):
|
|||
self.is_valid(raise_exception=True)
|
||||
language = get_language()
|
||||
if self.data.get('type') == 'csv':
|
||||
file = open(
|
||||
os.path.join(PROJECT_DIR, "apps", "dataset", 'template', f'csv_template_{to_locale(language)}.csv'),
|
||||
"rb")
|
||||
file = open(os.path.join(PROJECT_DIR, "apps", "dataset", 'template', f'csv_template_{to_locale(language)}.csv'), "rb")
|
||||
content = file.read()
|
||||
file.close()
|
||||
return HttpResponse(content, status=200, headers={'Content-Type': 'text/csv',
|
||||
return HttpResponse(content, status=200, headers={'Content-Type': 'text/cxv',
|
||||
'Content-Disposition': 'attachment; filename="csv_template.csv"'})
|
||||
elif self.data.get('type') == 'excel':
|
||||
file = open(os.path.join(PROJECT_DIR, "apps", "dataset", 'template',
|
||||
f'excel_template_{to_locale(language)}.xlsx'), "rb")
|
||||
file = open(os.path.join(PROJECT_DIR, "apps", "dataset", 'template', f'excel_template_{to_locale(language)}.xlsx'), "rb")
|
||||
content = file.read()
|
||||
file.close()
|
||||
return HttpResponse(content, status=200, headers={'Content-Type': 'application/vnd.ms-excel',
|
||||
|
|
@ -264,8 +261,7 @@ class DocumentSerializers(ApiMixin, serializers.Serializer):
|
|||
language = get_language()
|
||||
if self.data.get('type') == 'csv':
|
||||
file = open(
|
||||
os.path.join(PROJECT_DIR, "apps", "dataset", 'template',
|
||||
f'table_template_{to_locale(language)}.csv'),
|
||||
os.path.join(PROJECT_DIR, "apps", "dataset", 'template', f'table_template_{to_locale(language)}.csv'),
|
||||
"rb")
|
||||
content = file.read()
|
||||
file.close()
|
||||
|
|
@ -415,7 +411,6 @@ class DocumentSerializers(ApiMixin, serializers.Serializer):
|
|||
is_active = serializers.BooleanField(required=False, error_messages=ErrMessage.boolean(_('document is active')))
|
||||
task_type = serializers.IntegerField(required=False, error_messages=ErrMessage.integer(_('task type')))
|
||||
status = serializers.CharField(required=False, error_messages=ErrMessage.char(_('status')))
|
||||
order_by = serializers.CharField(required=False, error_messages=ErrMessage.char(_('order by')))
|
||||
|
||||
def get_query_set(self):
|
||||
query_set = QuerySet(model=Document)
|
||||
|
|
@ -442,18 +437,8 @@ class DocumentSerializers(ApiMixin, serializers.Serializer):
|
|||
query_set = query_set.filter(status__icontains=status)
|
||||
else:
|
||||
query_set = query_set.filter(status__iregex='^[2n]*$')
|
||||
order_by = self.data.get('order_by', '')
|
||||
order_by_query_set = QuerySet(model=get_dynamics_model(
|
||||
{'char_length': models.CharField(), 'paragraph_count': models.IntegerField(),
|
||||
"update_time": models.IntegerField(), 'create_time': models.DateTimeField()}))
|
||||
if order_by:
|
||||
order_by_query_set = order_by_query_set.order_by(order_by)
|
||||
else:
|
||||
order_by_query_set = order_by_query_set.order_by('-create_time', 'id')
|
||||
return {
|
||||
'document_custom_sql': query_set,
|
||||
'order_by_query': order_by_query_set
|
||||
}
|
||||
query_set = query_set.order_by('-create_time', 'id')
|
||||
return query_set
|
||||
|
||||
def list(self, with_valid=False):
|
||||
if with_valid:
|
||||
|
|
@ -661,8 +646,6 @@ class DocumentSerializers(ApiMixin, serializers.Serializer):
|
|||
cell = worksheet.cell(row=row_idx + 1, column=col_idx + 1)
|
||||
if isinstance(col, str):
|
||||
col = re.sub(ILLEGAL_CHARACTERS_RE, '', col)
|
||||
if col.startswith(('=', '+', '-', '@')):
|
||||
col = '\ufeff' + col
|
||||
cell.value = col
|
||||
# 创建HttpResponse对象返回Excel文件
|
||||
return workbook
|
||||
|
|
@ -705,8 +688,6 @@ class DocumentSerializers(ApiMixin, serializers.Serializer):
|
|||
|
||||
@staticmethod
|
||||
def reset_document_name(document_name):
|
||||
if document_name is not None:
|
||||
document_name = document_name.strip()[0:29]
|
||||
if document_name is None or not Utils.valid_sheet_name(document_name):
|
||||
return "Sheet"
|
||||
return document_name.strip()
|
||||
|
|
@ -716,10 +697,7 @@ class DocumentSerializers(ApiMixin, serializers.Serializer):
|
|||
self.is_valid(raise_exception=True)
|
||||
query_set = QuerySet(model=Document)
|
||||
query_set = query_set.filter(**{'id': self.data.get("document_id")})
|
||||
return native_search({
|
||||
'document_custom_sql': query_set,
|
||||
'order_by_query': QuerySet(Document).order_by('-create_time', 'id')
|
||||
}, select_string=get_file_content(
|
||||
return native_search(query_set, select_string=get_file_content(
|
||||
os.path.join(PROJECT_DIR, "apps", "dataset", 'sql', 'list_document.sql')), with_search_one=True)
|
||||
|
||||
def edit(self, instance: Dict, with_valid=False):
|
||||
|
|
@ -1105,12 +1083,9 @@ class DocumentSerializers(ApiMixin, serializers.Serializer):
|
|||
if len(document_model_list) == 0:
|
||||
return [], dataset_id
|
||||
query_set = query_set.filter(**{'id__in': [d.id for d in document_model_list]})
|
||||
return native_search({
|
||||
'document_custom_sql': query_set,
|
||||
'order_by_query': QuerySet(Document).order_by('-create_time', 'id')
|
||||
}, select_string=get_file_content(
|
||||
return native_search(query_set, select_string=get_file_content(
|
||||
os.path.join(PROJECT_DIR, "apps", "dataset", 'sql', 'list_document.sql')),
|
||||
with_search_one=False), dataset_id
|
||||
with_search_one=False), dataset_id
|
||||
|
||||
@staticmethod
|
||||
def _batch_sync(document_id_list: List[str]):
|
||||
|
|
@ -1200,7 +1175,7 @@ class DocumentSerializers(ApiMixin, serializers.Serializer):
|
|||
if not QuerySet(Document).filter(id=document_id).exists():
|
||||
raise AppApiException(500, _('document id not exist'))
|
||||
|
||||
def generate_related(self, model_id, prompt, state_list=None, with_valid=True):
|
||||
def generate_related(self, model_id, prompt, with_valid=True):
|
||||
if with_valid:
|
||||
self.is_valid(raise_exception=True)
|
||||
document_id = self.data.get('document_id')
|
||||
|
|
@ -1212,7 +1187,7 @@ class DocumentSerializers(ApiMixin, serializers.Serializer):
|
|||
State.PENDING)
|
||||
ListenerManagement.get_aggregation_document_status(document_id)()
|
||||
try:
|
||||
generate_related_by_document_id.delay(document_id, model_id, prompt, state_list)
|
||||
generate_related_by_document_id.delay(document_id, model_id, prompt)
|
||||
except AlreadyQueued as e:
|
||||
raise AppApiException(500, _('The task is being executed, please do not send it again.'))
|
||||
|
||||
|
|
@ -1225,23 +1200,17 @@ class DocumentSerializers(ApiMixin, serializers.Serializer):
|
|||
document_id_list = instance.get("document_id_list")
|
||||
model_id = instance.get("model_id")
|
||||
prompt = instance.get("prompt")
|
||||
state_list = instance.get('state_list')
|
||||
ListenerManagement.update_status(QuerySet(Document).filter(id__in=document_id_list),
|
||||
TaskType.GENERATE_PROBLEM,
|
||||
State.PENDING)
|
||||
ListenerManagement.update_status(QuerySet(Paragraph).annotate(
|
||||
reversed_status=Reverse('status'),
|
||||
task_type_status=Substr('reversed_status', TaskType.GENERATE_PROBLEM.value,
|
||||
1),
|
||||
).filter(task_type_status__in=state_list, document_id__in=document_id_list)
|
||||
.values('id'),
|
||||
ListenerManagement.update_status(QuerySet(Paragraph).filter(document_id__in=document_id_list),
|
||||
TaskType.GENERATE_PROBLEM,
|
||||
State.PENDING)
|
||||
ListenerManagement.get_aggregation_document_status_by_query_set(
|
||||
QuerySet(Document).filter(id__in=document_id_list))()
|
||||
try:
|
||||
for document_id in document_id_list:
|
||||
generate_related_by_document_id.delay(document_id, model_id, prompt, state_list)
|
||||
generate_related_by_document_id.delay(document_id, model_id, prompt)
|
||||
except AlreadyQueued as e:
|
||||
pass
|
||||
|
||||
|
|
@ -1267,7 +1236,6 @@ def save_image(image_list):
|
|||
exist_image_list = [str(i.get('id')) for i in
|
||||
QuerySet(Image).filter(id__in=[i.id for i in image_list]).values('id')]
|
||||
save_image_list = [image for image in image_list if not exist_image_list.__contains__(str(image.id))]
|
||||
save_image_list = list({img.id: img for img in save_image_list}.values())
|
||||
if len(save_image_list) > 0:
|
||||
QuerySet(Image).bulk_create(save_image_list)
|
||||
|
||||
|
|
|
|||
|
|
@ -28,9 +28,6 @@ mime_types = {"html": "text/html", "htm": "text/html", "shtml": "text/html", "cs
|
|||
"woff2": "font/woff2", "jar": "application/java-archive", "war": "application/java-archive",
|
||||
"ear": "application/java-archive", "json": "application/json", "hqx": "application/mac-binhex40",
|
||||
"doc": "application/msword", "pdf": "application/pdf", "ps": "application/postscript",
|
||||
"docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
"eps": "application/postscript", "ai": "application/postscript", "rtf": "application/rtf",
|
||||
"m3u8": "application/vnd.apple.mpegurl", "kml": "application/vnd.google-earth.kml+xml",
|
||||
"kmz": "application/vnd.google-earth.kmz", "xls": "application/vnd.ms-excel",
|
||||
|
|
@ -60,14 +57,12 @@ mime_types = {"html": "text/html", "htm": "text/html", "shtml": "text/html", "cs
|
|||
|
||||
class FileSerializer(serializers.Serializer):
|
||||
file = UploadedFileField(required=True, error_messages=ErrMessage.image(_('file')))
|
||||
meta = serializers.JSONField(required=False, allow_null=True)
|
||||
meta = serializers.JSONField(required=False)
|
||||
|
||||
def upload(self, with_valid=True):
|
||||
if with_valid:
|
||||
self.is_valid(raise_exception=True)
|
||||
meta = self.data.get('meta', None)
|
||||
if not meta:
|
||||
meta = {'debug': True}
|
||||
meta = self.data.get('meta')
|
||||
file_id = meta.get('file_id', uuid.uuid1())
|
||||
file = File(id=file_id, file_name=self.data.get('file').name, meta=meta)
|
||||
file.save(self.data.get('file').read())
|
||||
|
|
@ -90,4 +85,4 @@ class FileSerializer(serializers.Serializer):
|
|||
'Content-Disposition': 'attachment; filename="{}"'.format(
|
||||
file.file_name)})
|
||||
return HttpResponse(file.get_byte(), status=200,
|
||||
headers={'Content-Type': mime_types.get(file_type, 'text/plain')})
|
||||
headers={'Content-Type': mime_types.get(file.file_name.split(".")[-1], 'text/plain')})
|
||||
|
|
|
|||
|
|
@ -663,7 +663,6 @@ class ParagraphSerializers(ApiMixin, serializers.Serializer):
|
|||
**{'title__icontains': self.data.get('title')})
|
||||
if 'content' in self.data:
|
||||
query_set = query_set.filter(**{'content__icontains': self.data.get('content')})
|
||||
query_set.order_by('-create_time', 'id')
|
||||
return query_set
|
||||
|
||||
def list(self):
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
SELECT * from (
|
||||
SELECT
|
||||
"document".* ,
|
||||
to_json("document"."meta") as meta,
|
||||
|
|
@ -6,6 +5,3 @@ SELECT
|
|||
(SELECT "count"("id") FROM "paragraph" WHERE document_id="document"."id") as "paragraph_count"
|
||||
FROM
|
||||
"document" "document"
|
||||
${document_custom_sql}
|
||||
) temp
|
||||
${order_by_query}
|
||||
|
|
@ -2,7 +2,6 @@ UPDATE "document"
|
|||
SET "char_length" = ( SELECT CASE WHEN
|
||||
"sum" ( "char_length" ( "content" ) ) IS NULL THEN
|
||||
0 ELSE "sum" ( "char_length" ( "content" ) )
|
||||
END FROM paragraph WHERE "document_id" = %s ),
|
||||
"update_time" = CURRENT_TIMESTAMP
|
||||
END FROM paragraph WHERE "document_id" = %s )
|
||||
WHERE
|
||||
"id" = %s
|
||||
|
|
@ -3,12 +3,11 @@ import traceback
|
|||
|
||||
from celery_once import QueueOnce
|
||||
from django.db.models import QuerySet
|
||||
from django.db.models.functions import Reverse, Substr
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
from common.config.embedding_config import ModelManage
|
||||
from common.event import ListenerManagement
|
||||
from common.util.page_utils import page, page_desc
|
||||
from common.util.page_utils import page
|
||||
from dataset.models import Paragraph, Document, Status, TaskType, State
|
||||
from dataset.task.tools import save_problem
|
||||
from ops import celery_app
|
||||
|
|
@ -29,8 +28,7 @@ def generate_problem_by_paragraph(paragraph, llm_model, prompt):
|
|||
try:
|
||||
ListenerManagement.update_status(QuerySet(Paragraph).filter(id=paragraph.id), TaskType.GENERATE_PROBLEM,
|
||||
State.STARTED)
|
||||
res = llm_model.invoke(
|
||||
[HumanMessage(content=prompt.replace('{data}', paragraph.content).replace('{title}', paragraph.title))])
|
||||
res = llm_model.invoke([HumanMessage(content=prompt.replace('{data}', paragraph.content))])
|
||||
if (res.content is None) or (len(res.content) == 0):
|
||||
return
|
||||
problems = res.content.split('\n')
|
||||
|
|
@ -64,24 +62,9 @@ def get_is_the_task_interrupted(document_id):
|
|||
return is_the_task_interrupted
|
||||
|
||||
|
||||
@celery_app.task(base=QueueOnce, once={'keys': ['dataset_id']},
|
||||
name='celery:generate_related_by_dataset')
|
||||
def generate_related_by_dataset_id(dataset_id, model_id, prompt, state_list=None):
|
||||
document_list = QuerySet(Document).filter(dataset_id=dataset_id)
|
||||
for document in document_list:
|
||||
try:
|
||||
generate_related_by_document_id.delay(document.id, model_id, prompt, state_list)
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
|
||||
@celery_app.task(base=QueueOnce, once={'keys': ['document_id']},
|
||||
name='celery:generate_related_by_document')
|
||||
def generate_related_by_document_id(document_id, model_id, prompt, state_list=None):
|
||||
if state_list is None:
|
||||
state_list = [State.PENDING.value, State.STARTED.value, State.SUCCESS.value, State.FAILURE.value,
|
||||
State.REVOKE.value,
|
||||
State.REVOKED.value, State.IGNORED.value]
|
||||
def generate_related_by_document_id(document_id, model_id, prompt):
|
||||
try:
|
||||
is_the_task_interrupted = get_is_the_task_interrupted(document_id)
|
||||
if is_the_task_interrupted():
|
||||
|
|
@ -95,12 +78,7 @@ def generate_related_by_document_id(document_id, model_id, prompt, state_list=No
|
|||
generate_problem = get_generate_problem(llm_model, prompt,
|
||||
ListenerManagement.get_aggregation_document_status(
|
||||
document_id), is_the_task_interrupted)
|
||||
query_set = QuerySet(Paragraph).annotate(
|
||||
reversed_status=Reverse('status'),
|
||||
task_type_status=Substr('reversed_status', TaskType.GENERATE_PROBLEM.value,
|
||||
1),
|
||||
).filter(task_type_status__in=state_list, document_id=document_id)
|
||||
page_desc(query_set, 10, generate_problem, is_the_task_interrupted)
|
||||
page(QuerySet(Paragraph).filter(document_id=document_id), 10, generate_problem, is_the_task_interrupted)
|
||||
except Exception as e:
|
||||
max_kb_error.error(f'根据文档生成问题:{document_id}出现错误{str(e)}{traceback.format_exc()}')
|
||||
max_kb_error.error(_('Generate issue based on document: {document_id} error {error}{traceback}').format(
|
||||
|
|
|
|||
|
|
@ -54,8 +54,5 @@ def sync_replace_web_dataset(dataset_id: str, url: str, selector: str):
|
|||
def sync_web_document(dataset_id, source_url_list: List[str], selector: str):
|
||||
handler = get_sync_web_document_handler(dataset_id)
|
||||
for source_url in source_url_list:
|
||||
try:
|
||||
result = Fork(base_fork_url=source_url, selector_list=selector.split(' ')).fork()
|
||||
handler(source_url, selector, result)
|
||||
except Exception as e:
|
||||
pass
|
||||
result = Fork(base_fork_url=source_url, selector_list=selector.split(' ')).fork()
|
||||
handler(source_url, selector, result)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
Section title (optional), Section content (required，question answer), Question (optional，one per line in the cell)
|
||||
MaxKB product introduction,"MaxKB is a knowledge base question-answering system based on the LLM large language model. MaxKB = Max Knowledge Base，aims to become the most powerful brain of the enterprise。Out-of-the-box: supports direct document upload、automatic crawling of online documents、automatic text splitting and vectorization、and good intelligent question-answering interactive experience；Seamless embedding: supports zero-coding and rapid embedding into third-party business systems；Multi-model support: supports docking with mainstream large models，including Ollama local private large models (such as Llama 2、Llama 3、qwen)、Tongyi Qianwen、OpenAI、Azure OpenAI、Kimi、Zhipu AI、iFlytek Spark and Baidu Qianfan large models、etc.","What is MaxKB?
|
||||
Section title (optional), Section content (required, question answer, no more than 4096 characters)), question (optional, one per line in the cell)
|
||||
MaxKB product introduction, "MaxKB is a knowledge base question and answer system based on the LLM large language model. MaxKB = Max Knowledge Base aims to become the most powerful brain of the enterprise.
|
||||
Out-of-the-box: Supports direct uploading of documents, automatic crawling of online documents, automatic text splitting and vectorization, and a good intelligent Q&A interactive experience;
|
||||
Seamless embedding: Supports rapid embedding into third-party business systems with zero coding;
|
||||
Multi-model support: Supports docking with mainstream large models, including Ollama local private large models (such as Llama 2, Llama 3, qwen), Tongyi Qianwen, OpenAI, Azure OpenAI, Kimi, Zhipu AI, iFlytek Spark and Baidu Qianfan Large models etc. ","What is MaxKB?
|
||||
MaxKB product introduction
|
||||
Large language model supported by MaxKB
|
||||
MaxKB advantages"
|
||||
|
Can't render this file because it has a wrong number of fields in line 2.
|
|
|
@ -1,4 +1,4 @@
|
|||
分段标题（选填）,分段内容（必填，问题答案））,问题（选填，单元格内一行一个）
|
||||
分段标题（选填）,分段内容（必填，问题答案，最长不超过4096个字符））,问题（选填，单元格内一行一个）
|
||||
MaxKB产品介绍,"MaxKB 是一款基于 LLM 大语言模型的知识库问答系统。MaxKB = Max Knowledge Base，旨在成为企业的最强大脑。
|
||||
开箱即用：支持直接上传文档、自动爬取在线文档，支持文本自动拆分、向量化，智能问答交互体验好；
|
||||
无缝嵌入：支持零编码快速嵌入到第三方业务系统；
|
||||
|
|
|
|||
|
|
|
@ -1,4 +1,4 @@
|
|||
分段標題（選填）,分段內容（必填，問題答案））,問題（選填，單元格內一行一個）
|
||||
分段標題（選填）,分段內容（必填，問題答案，最長不超過4096個字元））,問題（選填，單元格內一行一個）
|
||||
MaxKB產品介紹,"MaxKB 是一款基於 LLM 大語言模型的知識庫問答系統。MaxKB = Max Knowledge Base，旨在成為企業的最強大大腦。
|
||||
開箱即用：支援直接上傳文檔、自動爬取線上文檔，支援文字自動分割、向量化，智慧問答互動體驗好；
|
||||
無縫嵌入：支援零編碼快速嵌入到第三方業務系統；
|
||||
|
|
|
|||
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -11,8 +11,6 @@ urlpatterns = [
|
|||
path('dataset/<str:dataset_id>/export', views.Dataset.Export.as_view(), name="export"),
|
||||
path('dataset/<str:dataset_id>/export_zip', views.Dataset.ExportZip.as_view(), name="export_zip"),
|
||||
path('dataset/<str:dataset_id>/re_embedding', views.Dataset.Embedding.as_view(), name="dataset_key"),
|
||||
path('dataset/<str:dataset_id>/generate_related', views.Dataset.GenerateRelated.as_view(),
|
||||
name="dataset_generate_related"),
|
||||
path('dataset/<str:dataset_id>/application', views.Dataset.Application.as_view()),
|
||||
path('dataset/<int:current_page>/<int:page_size>', views.Dataset.Page.as_view(), name="dataset"),
|
||||
path('dataset/<str:dataset_id>/sync_web', views.Dataset.SyncWeb.as_view()),
|
||||
|
|
|
|||
|
|
@ -1,56 +0,0 @@
|
|||
# coding=utf-8
|
||||
"""
|
||||
@project: MaxKB
|
||||
@Author：虎
|
||||
@file： common.py.py
|
||||
@date：2025/3/25 15:43
|
||||
@desc:
|
||||
"""
|
||||
from django.db.models import QuerySet
|
||||
|
||||
from dataset.models import DataSet, Document
|
||||
|
||||
|
||||
def get_dataset_operation_object(dataset_id: str):
|
||||
dataset_model = QuerySet(model=DataSet).filter(id=dataset_id).first()
|
||||
if dataset_model is not None:
|
||||
return {
|
||||
"name": dataset_model.name,
|
||||
"desc": dataset_model.desc,
|
||||
"type": dataset_model.type,
|
||||
"create_time": dataset_model.create_time,
|
||||
"update_time": dataset_model.update_time
|
||||
}
|
||||
return {}
|
||||
|
||||
|
||||
def get_document_operation_object(document_id: str):
|
||||
document_model = QuerySet(model=Document).filter(id=document_id).first()
|
||||
if document_model is not None:
|
||||
return {
|
||||
"name": document_model.name,
|
||||
"type": document_model.type,
|
||||
}
|
||||
return {}
|
||||
|
||||
|
||||
def get_document_operation_object_batch(document_id_list: str):
|
||||
document_model_list = QuerySet(model=Document).filter(id__in=document_id_list)
|
||||
if document_model_list is not None:
|
||||
return {
|
||||
"name": f'[{",".join([document_model.name for document_model in document_model_list])}]',
|
||||
'document_list': [{'name': document_model.name, 'type': document_model.type} for document_model in
|
||||
document_model_list]
|
||||
}
|
||||
return {}
|
||||
|
||||
|
||||
def get_dataset_document_operation_object(dataset_dict: dict, document_dict: dict):
|
||||
return {
|
||||
'name': f'{dataset_dict.get("name", "")}/{document_dict.get("name", "")}',
|
||||
'dataset_name': dataset_dict.get("name", ""),
|
||||
'dataset_desc': dataset_dict.get("desc", ""),
|
||||
'dataset_type': dataset_dict.get("type", ""),
|
||||
'document_name': document_dict.get("name", ""),
|
||||
'document_type': document_dict.get("type", ""),
|
||||
}
|
||||
|
|
@ -13,17 +13,13 @@ from rest_framework.parsers import MultiPartParser
|
|||
from rest_framework.views import APIView
|
||||
from rest_framework.views import Request
|
||||
|
||||
import dataset.models
|
||||
from common.auth import TokenAuth, has_permissions
|
||||
from common.constants.permission_constants import PermissionConstants, CompareConstants, Permission, Group, Operate, \
|
||||
ViewPermission, RoleConstants
|
||||
from common.log.log import log
|
||||
from common.response import result
|
||||
from common.response.result import get_page_request_params, get_page_api_response, get_api_response
|
||||
from common.swagger_api.common_api import CommonApi
|
||||
from dataset.serializers.common_serializers import GenerateRelatedSerializer
|
||||
from dataset.serializers.dataset_serializers import DataSetSerializers
|
||||
from dataset.views.common import get_dataset_operation_object
|
||||
from setting.serializers.provider_serializers import ModelSerializer
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
|
|
@ -35,8 +31,8 @@ class Dataset(APIView):
|
|||
authentication_classes = [TokenAuth]
|
||||
|
||||
@action(methods=['PUT'], detail=False)
|
||||
@swagger_auto_schema(operation_summary=_("Synchronize the knowledge base of the website"),
|
||||
operation_id=_("Synchronize the knowledge base of the website"),
|
||||
@swagger_auto_schema(operation_summary="同步Web站点知识库",
|
||||
operation_id="同步Web站点知识库",
|
||||
manual_parameters=DataSetSerializers.SyncWeb.get_request_params_api(),
|
||||
responses=result.get_default_response(),
|
||||
tags=[_('Knowledge Base')])
|
||||
|
|
@ -46,8 +42,6 @@ class Dataset(APIView):
|
|||
dynamic_tag=keywords.get('dataset_id'))],
|
||||
compare=CompareConstants.AND), PermissionConstants.DATASET_EDIT,
|
||||
compare=CompareConstants.AND)
|
||||
@log(menu='Knowledge Base', operate="Synchronize the knowledge base of the website",
|
||||
get_operation_object=lambda r, keywords: get_dataset_operation_object(keywords.get('dataset_id')))
|
||||
def put(self, request: Request, dataset_id: str):
|
||||
return result.success(DataSetSerializers.SyncWeb(
|
||||
data={'sync_type': request.query_params.get('sync_type'), 'id': dataset_id,
|
||||
|
|
@ -58,17 +52,14 @@ class Dataset(APIView):
|
|||
parser_classes = [MultiPartParser]
|
||||
|
||||
@action(methods=['POST'], detail=False)
|
||||
@swagger_auto_schema(operation_summary=_("Create QA knowledge base"),
|
||||
operation_id=_("Create QA knowledge base"),
|
||||
@swagger_auto_schema(operation_summary="创建QA知识库",
|
||||
operation_id="创建QA知识库",
|
||||
manual_parameters=DataSetSerializers.Create.CreateQASerializers.get_request_params_api(),
|
||||
responses=get_api_response(
|
||||
DataSetSerializers.Create.CreateQASerializers.get_response_body_api()),
|
||||
tags=[_('Knowledge Base')]
|
||||
)
|
||||
@has_permissions(PermissionConstants.DATASET_CREATE, compare=CompareConstants.AND)
|
||||
@log(menu='Knowledge Base', operate="Create QA knowledge base",
|
||||
get_operation_object=lambda r, keywords: {'name': r.data.get('name'), 'desc': r.data.get('desc'),
|
||||
'file_list': r.FILES.getlist('file')})
|
||||
def post(self, request: Request):
|
||||
return result.success(DataSetSerializers.Create(data={'user_id': request.user.id}).save_qa({
|
||||
'file_list': request.FILES.getlist('file'),
|
||||
|
|
@ -88,13 +79,6 @@ class Dataset(APIView):
|
|||
tags=[_('Knowledge Base')]
|
||||
)
|
||||
@has_permissions(PermissionConstants.DATASET_CREATE, compare=CompareConstants.AND)
|
||||
@log(menu='Knowledge Base', operate="Create a web site knowledge base",
|
||||
get_operation_object=lambda r, keywords: {'name': r.data.get('name'), 'desc': r.data.get('desc'),
|
||||
'file_list': r.FILES.getlist('file'),
|
||||
'meta': {'source_url': r.data.get('source_url'),
|
||||
'selector': r.data.get('selector'),
|
||||
'embedding_mode_id': r.data.get('embedding_mode_id')}}
|
||||
)
|
||||
def post(self, request: Request):
|
||||
return result.success(DataSetSerializers.Create(data={'user_id': request.user.id}).save_web(request.data))
|
||||
|
||||
|
|
@ -133,8 +117,6 @@ class Dataset(APIView):
|
|||
tags=[_('Knowledge Base')]
|
||||
)
|
||||
@has_permissions(PermissionConstants.DATASET_CREATE, compare=CompareConstants.AND)
|
||||
@log(menu='Knowledge Base', operate="Create a knowledge base",
|
||||
get_operation_object=lambda r, keywords: {'name': r.data.get('name'), 'desc': r.data.get('desc')})
|
||||
def post(self, request: Request):
|
||||
return result.success(DataSetSerializers.Create(data={'user_id': request.user.id}).save(request.data))
|
||||
|
||||
|
|
@ -168,30 +150,10 @@ class Dataset(APIView):
|
|||
)
|
||||
@has_permissions(lambda r, keywords: Permission(group=Group.DATASET, operate=Operate.MANAGE,
|
||||
dynamic_tag=keywords.get('dataset_id')))
|
||||
@log(menu='Knowledge Base', operate="Re-vectorize",
|
||||
get_operation_object=lambda r, keywords: get_dataset_operation_object(keywords.get('dataset_id')))
|
||||
def put(self, request: Request, dataset_id: str):
|
||||
return result.success(
|
||||
DataSetSerializers.Operate(data={'id': dataset_id, 'user_id': request.user.id}).re_embedding())
|
||||
|
||||
class GenerateRelated(APIView):
|
||||
authentication_classes = [TokenAuth]
|
||||
|
||||
@action(methods=['PUT'], detail=False)
|
||||
@swagger_auto_schema(operation_summary=_('Generate related'), operation_id=_('Generate related'),
|
||||
manual_parameters=DataSetSerializers.Operate.get_request_params_api(),
|
||||
request_body=GenerateRelatedSerializer.get_request_body_api(),
|
||||
responses=result.get_default_response(),
|
||||
tags=[_('Knowledge Base')]
|
||||
)
|
||||
@log(menu='document', operate="Generate related documents",
|
||||
get_operation_object=lambda r, keywords: get_dataset_operation_object(keywords.get('dataset_id'))
|
||||
)
|
||||
def put(self, request: Request, dataset_id: str):
|
||||
return result.success(
|
||||
DataSetSerializers.Operate(data={'id': dataset_id, 'user_id': request.user.id}).generate_related(
|
||||
request.data))
|
||||
|
||||
class Export(APIView):
|
||||
authentication_classes = [TokenAuth]
|
||||
|
||||
|
|
@ -202,8 +164,6 @@ class Dataset(APIView):
|
|||
)
|
||||
@has_permissions(lambda r, keywords: Permission(group=Group.DATASET, operate=Operate.MANAGE,
|
||||
dynamic_tag=keywords.get('dataset_id')))
|
||||
@log(menu='Knowledge Base', operate="Export knowledge base",
|
||||
get_operation_object=lambda r, keywords: get_dataset_operation_object(keywords.get('dataset_id')))
|
||||
def get(self, request: Request, dataset_id: str):
|
||||
return DataSetSerializers.Operate(data={'id': dataset_id, 'user_id': request.user.id}).export_excel()
|
||||
|
||||
|
|
@ -218,8 +178,6 @@ class Dataset(APIView):
|
|||
)
|
||||
@has_permissions(lambda r, keywords: Permission(group=Group.DATASET, operate=Operate.MANAGE,
|
||||
dynamic_tag=keywords.get('dataset_id')))
|
||||
@log(menu='Knowledge Base', operate="Export knowledge base containing images",
|
||||
get_operation_object=lambda r, keywords: get_dataset_operation_object(keywords.get('dataset_id')))
|
||||
def get(self, request: Request, dataset_id: str):
|
||||
return DataSetSerializers.Operate(data={'id': dataset_id, 'user_id': request.user.id}).export_zip()
|
||||
|
||||
|
|
@ -235,8 +193,6 @@ class Dataset(APIView):
|
|||
dynamic_tag=keywords.get('dataset_id')),
|
||||
lambda r, k: Permission(group=Group.DATASET, operate=Operate.DELETE,
|
||||
dynamic_tag=k.get('dataset_id')), compare=CompareConstants.AND)
|
||||
@log(menu='Knowledge Base', operate="Delete knowledge base",
|
||||
get_operation_object=lambda r, keywords: get_dataset_operation_object(keywords.get('dataset_id')))
|
||||
def delete(self, request: Request, dataset_id: str):
|
||||
operate = DataSetSerializers.Operate(data={'id': dataset_id})
|
||||
return result.success(operate.delete())
|
||||
|
|
@ -263,8 +219,6 @@ class Dataset(APIView):
|
|||
)
|
||||
@has_permissions(lambda r, keywords: Permission(group=Group.DATASET, operate=Operate.MANAGE,
|
||||
dynamic_tag=keywords.get('dataset_id')))
|
||||
@log(menu='Knowledge Base', operate="Modify knowledge base information",
|
||||
get_operation_object=lambda r, keywords: get_dataset_operation_object(keywords.get('dataset_id')))
|
||||
def put(self, request: Request, dataset_id: str):
|
||||
return result.success(
|
||||
DataSetSerializers.Operate(data={'id': dataset_id, 'user_id': request.user.id}).edit(request.data,
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue