diff --git a/package.json b/package.json index 55cbc3957..2f85071f4 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "nprogress": "^0.2.0", "openai": "^3.2.1", "papaparse": "^5.4.1", + "pg": "^8.10.0", "react": "18.2.0", "react-dom": "18.2.0", "react-hook-form": "^7.43.1", @@ -63,11 +64,11 @@ "@types/node": "18.14.0", "@types/nodemailer": "^6.4.7", "@types/papaparse": "^5.3.7", + "@types/pg": "^8.6.6", "@types/react": "18.0.28", "@types/react-dom": "18.0.11", "@types/react-syntax-highlighter": "^15.5.6", "@types/tunnel": "^0.0.3", - "@types/uuid": "^9.0.1", "eslint": "8.34.0", "eslint-config-next": "13.1.6", "husky": "^8.0.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e368f8cb3..5f6308341 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -18,11 +18,11 @@ specifiers: '@types/nodemailer': ^6.4.7 '@types/nprogress': ^0.2.0 '@types/papaparse': ^5.3.7 + '@types/pg': ^8.6.6 '@types/react': 18.0.28 '@types/react-dom': 18.0.11 '@types/react-syntax-highlighter': ^15.5.6 '@types/tunnel': ^0.0.3 - '@types/uuid': ^9.0.1 axios: ^1.3.3 crypto: ^1.0.1 dayjs: ^1.11.7 @@ -46,6 +46,7 @@ specifiers: nprogress: ^0.2.0 openai: ^3.2.1 papaparse: ^5.4.1 + pg: ^8.10.0 prettier: ^2.8.4 react: 18.2.0 react-dom: 18.2.0 @@ -93,6 +94,7 @@ dependencies: nprogress: registry.npmmirror.com/nprogress/0.2.0 openai: registry.npmmirror.com/openai/3.2.1 papaparse: registry.npmmirror.com/papaparse/5.4.1 + pg: registry.npmmirror.com/pg/8.10.0 react: registry.npmmirror.com/react/18.2.0 react-dom: registry.npmmirror.com/react-dom/18.2.0_react@18.2.0 react-hook-form: registry.npmmirror.com/react-hook-form/7.43.1_react@18.2.0 @@ -116,11 +118,11 @@ devDependencies: '@types/node': registry.npmmirror.com/@types/node/18.14.0 '@types/nodemailer': registry.npmmirror.com/@types/nodemailer/6.4.7 '@types/papaparse': registry.npmmirror.com/@types/papaparse/5.3.7 + '@types/pg': registry.npmmirror.com/@types/pg/8.6.6 '@types/react': registry.npmmirror.com/@types/react/18.0.28 '@types/react-dom': registry.npmmirror.com/@types/react-dom/18.0.11 '@types/react-syntax-highlighter': registry.npmmirror.com/@types/react-syntax-highlighter/15.5.6 '@types/tunnel': registry.npmmirror.com/@types/tunnel/0.0.3 - '@types/uuid': registry.npmmirror.com/@types/uuid/9.0.1 eslint: registry.npmmirror.com/eslint/8.34.0 eslint-config-next: registry.npmmirror.com/eslint-config-next/13.1.6_7kw3g6rralp5ps6mg3uyzz6azm husky: registry.npmmirror.com/husky/8.0.3 @@ -5192,6 +5194,16 @@ packages: name: '@types/parse-json' version: 4.0.0 + registry.npmmirror.com/@types/pg/8.6.6: + resolution: {integrity: sha512-O2xNmXebtwVekJDD+02udOncjVcMZQuTEQEMpKJ0ZRf5E7/9JJX3izhKUcUifBkyKpljyUM6BTgy2trmviKlpw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/pg/-/pg-8.6.6.tgz} + name: '@types/pg' + version: 8.6.6 + dependencies: + '@types/node': registry.npmmirror.com/@types/node/18.14.0 + pg-protocol: registry.npmmirror.com/pg-protocol/1.6.0 + pg-types: registry.npmmirror.com/pg-types/2.2.0 + dev: true + registry.npmmirror.com/@types/prop-types/15.7.5: resolution: {integrity: sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/prop-types/-/prop-types-15.7.5.tgz} name: '@types/prop-types' @@ -5241,12 +5253,6 @@ packages: version: 2.0.6 dev: false - registry.npmmirror.com/@types/uuid/9.0.1: - resolution: {integrity: sha512-rFT3ak0/2trgvp4yYZo5iKFEPsET7vKydKF+VRCxlQ9bpheehyAJH89dAkaLEq/j/RZXJIqcgsmPJKUP1Z28HA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/uuid/-/uuid-9.0.1.tgz} - name: '@types/uuid' - version: 9.0.1 - dev: true - registry.npmmirror.com/@types/webidl-conversions/7.0.0: resolution: {integrity: sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/webidl-conversions/-/webidl-conversions-7.0.0.tgz} name: '@types/webidl-conversions' @@ -5816,6 +5822,13 @@ packages: version: 1.0.1 dev: false + registry.npmmirror.com/buffer-writer/2.0.0: + resolution: {integrity: sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/buffer-writer/-/buffer-writer-2.0.0.tgz} + name: buffer-writer + version: 2.0.0 + engines: {node: '>=4'} + dev: false + registry.npmmirror.com/buffer/5.7.1: resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/buffer/-/buffer-5.7.1.tgz} name: buffer @@ -9733,6 +9746,12 @@ packages: netmask: registry.npmmirror.com/netmask/2.0.2 dev: false + registry.npmmirror.com/packet-reader/1.0.0: + resolution: {integrity: sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/packet-reader/-/packet-reader-1.0.0.tgz} + name: packet-reader + version: 1.0.0 + dev: false + registry.npmmirror.com/pako/1.0.11: resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pako/-/pako-1.0.11.tgz} name: pako @@ -9829,6 +9848,74 @@ packages: through: registry.npmmirror.com/through/2.3.8 dev: false + registry.npmmirror.com/pg-connection-string/2.5.0: + resolution: {integrity: sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz} + name: pg-connection-string + version: 2.5.0 + dev: false + + registry.npmmirror.com/pg-int8/1.0.1: + resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pg-int8/-/pg-int8-1.0.1.tgz} + name: pg-int8 + version: 1.0.1 + engines: {node: '>=4.0.0'} + + registry.npmmirror.com/pg-pool/3.6.0_pg@8.10.0: + resolution: {integrity: sha512-clFRf2ksqd+F497kWFyM21tMjeikn60oGDmqMT8UBrynEwVEX/5R5xd2sdvdo1cZCFlguORNpVuqxIj+aK4cfQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pg-pool/-/pg-pool-3.6.0.tgz} + id: registry.npmmirror.com/pg-pool/3.6.0 + name: pg-pool + version: 3.6.0 + peerDependencies: + pg: '>=8.0' + dependencies: + pg: registry.npmmirror.com/pg/8.10.0 + dev: false + + registry.npmmirror.com/pg-protocol/1.6.0: + resolution: {integrity: sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pg-protocol/-/pg-protocol-1.6.0.tgz} + name: pg-protocol + version: 1.6.0 + + registry.npmmirror.com/pg-types/2.2.0: + resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pg-types/-/pg-types-2.2.0.tgz} + name: pg-types + version: 2.2.0 + engines: {node: '>=4'} + dependencies: + pg-int8: registry.npmmirror.com/pg-int8/1.0.1 + postgres-array: registry.npmmirror.com/postgres-array/2.0.0 + postgres-bytea: registry.npmmirror.com/postgres-bytea/1.0.0 + postgres-date: registry.npmmirror.com/postgres-date/1.0.7 + postgres-interval: registry.npmmirror.com/postgres-interval/1.2.0 + + registry.npmmirror.com/pg/8.10.0: + resolution: {integrity: sha512-ke7o7qSTMb47iwzOSaZMfeR7xToFdkE71ifIipOAAaLIM0DYzfOAXlgFFmYUIE2BcJtvnVlGCID84ZzCegE8CQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pg/-/pg-8.10.0.tgz} + name: pg + version: 8.10.0 + engines: {node: '>= 8.0.0'} + peerDependencies: + pg-native: '>=3.0.1' + peerDependenciesMeta: + pg-native: + optional: true + dependencies: + buffer-writer: registry.npmmirror.com/buffer-writer/2.0.0 + packet-reader: registry.npmmirror.com/packet-reader/1.0.0 + pg-connection-string: registry.npmmirror.com/pg-connection-string/2.5.0 + pg-pool: registry.npmmirror.com/pg-pool/3.6.0_pg@8.10.0 + pg-protocol: registry.npmmirror.com/pg-protocol/1.6.0 + pg-types: registry.npmmirror.com/pg-types/2.2.0 + pgpass: registry.npmmirror.com/pgpass/1.0.5 + dev: false + + registry.npmmirror.com/pgpass/1.0.5: + resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pgpass/-/pgpass-1.0.5.tgz} + name: pgpass + version: 1.0.5 + dependencies: + split2: registry.npmmirror.com/split2/4.2.0 + dev: false + registry.npmmirror.com/picocolors/1.0.0: resolution: {integrity: sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/picocolors/-/picocolors-1.0.0.tgz} name: picocolors @@ -9859,6 +9946,32 @@ packages: source-map-js: registry.npmmirror.com/source-map-js/1.0.2 dev: false + registry.npmmirror.com/postgres-array/2.0.0: + resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/postgres-array/-/postgres-array-2.0.0.tgz} + name: postgres-array + version: 2.0.0 + engines: {node: '>=4'} + + registry.npmmirror.com/postgres-bytea/1.0.0: + resolution: {integrity: sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/postgres-bytea/-/postgres-bytea-1.0.0.tgz} + name: postgres-bytea + version: 1.0.0 + engines: {node: '>=0.10.0'} + + registry.npmmirror.com/postgres-date/1.0.7: + resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/postgres-date/-/postgres-date-1.0.7.tgz} + name: postgres-date + version: 1.0.7 + engines: {node: '>=0.10.0'} + + registry.npmmirror.com/postgres-interval/1.2.0: + resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/postgres-interval/-/postgres-interval-1.2.0.tgz} + name: postgres-interval + version: 1.2.0 + engines: {node: '>=0.10.0'} + dependencies: + xtend: registry.npmmirror.com/xtend/4.0.2 + registry.npmmirror.com/prebuild-install/7.1.1: resolution: {integrity: sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/prebuild-install/-/prebuild-install-7.1.1.tgz} name: prebuild-install @@ -10831,6 +10944,13 @@ packages: dev: false optional: true + registry.npmmirror.com/split2/4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/split2/-/split2-4.2.0.tgz} + name: split2 + version: 4.2.0 + engines: {node: '>= 10.x'} + dev: false + registry.npmmirror.com/sprintf-js/1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/sprintf-js/-/sprintf-js-1.0.3.tgz} name: sprintf-js @@ -11815,7 +11935,6 @@ packages: name: xtend version: 4.0.2 engines: {node: '>=0.4'} - dev: false registry.npmmirror.com/yallist/3.1.1: resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/yallist/-/yallist-3.1.1.tgz} diff --git a/src/api/model.ts b/src/api/model.ts index 3ce2e3a03..6bef501ab 100644 --- a/src/api/model.ts +++ b/src/api/model.ts @@ -79,7 +79,7 @@ export const getWebContent = (url: string) => POST(`/model/data/fetching */ export const postModelDataInput = (data: { modelId: string; - data: { text: ModelDataSchema['text']; q: ModelDataSchema['q'] }[]; + data: { a: ModelDataSchema['a']; q: ModelDataSchema['q'] }[]; }) => POST(`/model/data/pushModelDataInput`, data); /** @@ -97,7 +97,7 @@ export const postModelDataCsvData = (modelId: string, data: string[][]) => /** * 更新模型数据 */ -export const putModelDataById = (data: { dataId: string; text: string; q?: string }) => +export const putModelDataById = (data: { dataId: string; a: string; q?: string }) => PUT('/model/data/putModelData', data); /** * 删除一条模型数据 diff --git a/src/constants/model.ts b/src/constants/model.ts index 837b287d5..669be9850 100644 --- a/src/constants/model.ts +++ b/src/constants/model.ts @@ -1,5 +1,9 @@ import type { ServiceName, ModelDataType, ModelSchema } from '@/types/mongoSchema'; -import type { RedisModelDataItemType } from '@/types/redis'; + +export enum ModelDataStatusEnum { + ready = 'ready', + waiting = 'waiting' +} export enum ChatModelNameEnum { GPT35 = 'gpt-3.5-turbo', @@ -80,7 +84,7 @@ export const formatModelStatus = { } }; -export const ModelDataStatusMap: Record = { +export const ModelDataStatusMap: Record<`${ModelDataStatusEnum}`, string> = { ready: '训练完成', waiting: '训练中' }; diff --git a/src/pages/api/model/data/delModelDataById.ts b/src/pages/api/model/data/delModelDataById.ts index aa4059c07..d3e04ec09 100644 --- a/src/pages/api/model/data/delModelDataById.ts +++ b/src/pages/api/model/data/delModelDataById.ts @@ -1,7 +1,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@/service/response'; import { authToken } from '@/service/utils/tools'; -import { connectRedis } from '@/service/redis'; +import { connectPg } from '@/service/pg'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -21,15 +21,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< // 凭证校验 const userId = await authToken(authorization); - const redis = await connectRedis(); + const pg = await connectPg(); + await pg.query(`DELETE FROM modelData WHERE user_id = '${userId}' AND id = '${dataId}'`); - // 校验是否为该用户的数据 - const dataItemUserId = await redis.hGet(dataId, 'userId'); - if (dataItemUserId !== userId) { - throw new Error('无权操作'); - } - // 删除 - await redis.del(dataId); jsonRes(res); } catch (err) { console.log(err); diff --git a/src/pages/api/model/data/getModelData.ts b/src/pages/api/model/data/getModelData.ts index 5cb6976d3..f917386ed 100644 --- a/src/pages/api/model/data/getModelData.ts +++ b/src/pages/api/model/data/getModelData.ts @@ -2,8 +2,8 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@/service/response'; import { connectToDatabase } from '@/service/mongo'; import { authToken } from '@/service/utils/tools'; -import { connectRedis } from '@/service/redis'; -import { VecModelDataIdx } from '@/constants/redis'; +import { connectPg } from '@/service/pg'; +import type { PgModelDataItemType } from '@/types/pg'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -35,34 +35,20 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< const userId = await authToken(authorization); await connectToDatabase(); - const redis = await connectRedis(); + const pg = await connectPg(); - // 从 redis 中获取数据 - const searchRes = await redis.ft.search( - VecModelDataIdx, - `@modelId:{${modelId}} @userId:{${userId}} ${searchText ? `*${searchText}*` : ''}`, - { - RETURN: ['q', 'text', 'status'], - LIMIT: { - from: (pageNum - 1) * pageSize, - size: pageSize - }, - SORTBY: { - BY: 'modelId', - DIRECTION: 'DESC' - } - } - ); + const searchRes = await pg.query(`SELECT id, q, a, status + FROM modelData + WHERE user_id='${userId}' AND model_id='${modelId}' + LIMIT ${pageSize} OFFSET ${pageSize * (pageNum - 1)} + `); jsonRes(res, { data: { pageNum, pageSize, - data: searchRes.documents.map((item) => ({ - id: item.id, - ...item.value - })), - total: searchRes.total + data: searchRes.rows, + total: searchRes.rowCount } }); } catch (err) { diff --git a/src/pages/api/model/data/pushModelDataInput.ts b/src/pages/api/model/data/pushModelDataInput.ts index 00de78edb..6fab7572a 100644 --- a/src/pages/api/model/data/pushModelDataInput.ts +++ b/src/pages/api/model/data/pushModelDataInput.ts @@ -4,14 +4,13 @@ import { connectToDatabase, Model } from '@/service/mongo'; import { authToken } from '@/service/utils/tools'; import { ModelDataSchema } from '@/types/mongoSchema'; import { generateVector } from '@/service/events/generateVector'; -import { connectRedis } from '@/service/redis'; -import { VecModelDataPrefix, ModelDataStatusEnum } from '@/constants/redis'; +import { connectPg } from '@/service/pg'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { const { modelId, data } = req.body as { modelId: string; - data: { text: ModelDataSchema['text']; q: ModelDataSchema['q'] }[]; + data: { a: ModelDataSchema['a']; q: ModelDataSchema['q'] }[]; }; const { authorization } = req.headers; @@ -27,7 +26,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< const userId = await authToken(authorization); await connectToDatabase(); - const redis = await connectRedis(); + const pg = await connectPg(); // 验证是否是该用户的 model const model = await Model.findOne({ @@ -39,29 +38,23 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< throw new Error('无权操作该模型'); } - const insertRes = await Promise.allSettled( - data.map((item) => { - return redis.sendCommand([ - 'HMSET', - `${VecModelDataPrefix}:${item.q.id}`, - 'userId', - userId, - 'modelId', - modelId, - 'q', - item.q.text, - 'text', - item.text, - 'status', - ModelDataStatusEnum.waiting - ]); - }) + // 插入记录 + await pg.query( + `INSERT INTO modelData (user_id, model_id, q, a, status) VALUES ${data + .map( + (item) => + `('${userId}', '${modelId}', '${item.q.replace(/\'/g, '"')}', '${item.a.replace( + /\'/g, + '"' + )}', 'waiting')` + ) + .join(',')}` ); generateVector(); jsonRes(res, { - data: insertRes.filter((item) => item.status === 'rejected').length + data: 0 }); } catch (err) { jsonRes(res, { diff --git a/src/pages/model/detail/components/InputDataModal.tsx b/src/pages/model/detail/components/InputDataModal.tsx index 8dba1497a..dc89dd51e 100644 --- a/src/pages/model/detail/components/InputDataModal.tsx +++ b/src/pages/model/detail/components/InputDataModal.tsx @@ -16,14 +16,14 @@ import { useToast } from '@/hooks/useToast'; import { customAlphabet } from 'nanoid'; const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12); -export type FormData = { dataId?: string; text: string; q: string }; +export type FormData = { dataId?: string; a: string; q: string }; const InputDataModal = ({ onClose, onSuccess, modelId, defaultValues = { - text: '', + a: '', q: '' } }: { @@ -51,11 +51,8 @@ const InputDataModal = ({ modelId: modelId, data: [ { - text: e.text, - q: { - id: nanoid(), - text: e.q - } + a: e.a, + q: e.q } ] }); @@ -65,7 +62,7 @@ const InputDataModal = ({ status: res === 0 ? 'success' : 'warning' }); reset({ - text: '', + a: '', q: '' }); onSuccess(); @@ -81,10 +78,10 @@ const InputDataModal = ({ async (e: FormData) => { if (!e.dataId) return; - if (e.text !== defaultValues.text || e.q !== defaultValues.q) { + if (e.a !== defaultValues.a || e.q !== defaultValues.q) { await putModelDataById({ dataId: e.dataId, - text: e.text, + a: e.a, q: e.q === defaultValues.q ? '' : e.q }); onSuccess(); @@ -144,7 +141,7 @@ const InputDataModal = ({ maxLength={1000} resize={'none'} h={'calc(100% - 30px)'} - {...register(`text`, { + {...register(`a`, { required: '知识点' })} /> diff --git a/src/pages/model/detail/components/ModelDataCard.tsx b/src/pages/model/detail/components/ModelDataCard.tsx index a62ecf37c..87eb457d1 100644 --- a/src/pages/model/detail/components/ModelDataCard.tsx +++ b/src/pages/model/detail/components/ModelDataCard.tsx @@ -19,7 +19,7 @@ import { Input } from '@chakra-ui/react'; import type { ModelSchema } from '@/types/mongoSchema'; -import type { RedisModelDataItemType } from '@/types/redis'; +import type { ModelDataItemType } from '@/types/model'; import { ModelDataStatusMap } from '@/constants/model'; import { usePagination } from '@/hooks/usePagination'; import { @@ -53,9 +53,9 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => { total, getData, pageNum - } = usePagination({ + } = usePagination({ api: getModelDataList, - pageSize: 8, + pageSize: 10, params: { modelId: model._id, searchText @@ -149,7 +149,7 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => { setEditInputData({ - text: '', + a: '', q: '' }) } @@ -216,7 +216,7 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => { maxH={'250px'} overflowY={'auto'} > - {item.text} + {item.a} {ModelDataStatusMap[item.status]} @@ -231,7 +231,7 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => { setEditInputData({ dataId: item.id, q: item.q, - text: item.text + a: item.a }) } /> diff --git a/src/service/events/generateQA.ts b/src/service/events/generateQA.ts index eaac435a4..8d0826d71 100644 --- a/src/service/events/generateQA.ts +++ b/src/service/events/generateQA.ts @@ -6,12 +6,9 @@ import type { ChatCompletionRequestMessage } from 'openai'; import { ChatModelNameEnum } from '@/constants/model'; import { pushSplitDataBill } from '@/service/events/pushBill'; import { generateVector } from './generateVector'; -import { connectRedis } from '../redis'; -import { VecModelDataPrefix } from '@/constants/redis'; -import { customAlphabet } from 'nanoid'; import { openaiError2 } from '../errorCode'; +import { connectPg } from '@/service/pg'; import { ModelSplitDataSchema } from '@/types/mongoSchema'; -const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12); export async function generateQA(next = false): Promise { if (process.env.queueTask !== '1') { @@ -25,7 +22,7 @@ export async function generateQA(next = false): Promise { let dataId = null; try { - const redis = await connectRedis(); + const pg = await connectPg(); // 找出一个需要生成的 dataItem const data = await SplitData.aggregate([ { $match: { textList: { $exists: true, $ne: [] } } }, @@ -139,23 +136,17 @@ export async function generateQA(next = false): Promise { SplitData.findByIdAndUpdate(dataItem._id, { textList: dataItem.textList.slice(0, -5) }), // 删掉后5个数据 - ...resultList.map((item) => { - // 插入 redis - return redis.sendCommand([ - 'HMSET', - `${VecModelDataPrefix}:${nanoid()}`, - 'userId', - String(dataItem.userId), - 'modelId', - String(dataItem.modelId), - 'q', - item.q, - 'text', - item.a, - 'status', - 'waiting' - ]); - }) + // 生成的内容插入 pg + pg.query(`INSERT INTO modelData (user_id, model_id, q, a, status) VALUES ${resultList + .map( + (item) => + `('${String(dataItem.userId)}', '${String(dataItem.modelId)}', '${item.q.replace( + /\'/g, + '"' + )}', '${item.a.replace(/\'/g, '"')}', 'waiting')` + ) + .join(',')} + `) ]); console.log('生成QA成功，time:', `${(Date.now() - startTime) / 1000}s`); diff --git a/src/service/events/generateVector.ts b/src/service/events/generateVector.ts index 776b3e3c1..58b6f3c51 100644 --- a/src/service/events/generateVector.ts +++ b/src/service/events/generateVector.ts @@ -1,9 +1,8 @@ import { connectRedis } from '../redis'; -import { VecModelDataIdx } from '@/constants/redis'; -import { vectorToBuffer } from '@/utils/tools'; -import { ModelDataStatusEnum } from '@/constants/redis'; import { openaiCreateEmbedding, getOpenApiKey } from '../utils/openai'; import { openaiError2 } from '../errorCode'; +import { connectPg } from '@/service/pg'; +import type { PgModelDataItemType } from '@/types/pg'; export async function generateVector(next = false): Promise { if (process.env.queueTask !== '1') { @@ -15,32 +14,27 @@ export async function generateVector(next = false): Promise { global.generatingVector = true; let dataId = null; + try { - const redis = await connectRedis(); + const pg = await connectPg(); // 从找出一个 status = waiting 的数据 - const searchRes = await redis.ft.search( - VecModelDataIdx, - `@status:{${ModelDataStatusEnum.waiting}}`, - { - RETURN: ['q', 'userId'], - LIMIT: { - from: 0, - size: 1 - } - } - ); + const searchRes = await pg.query(`SELECT id, q, user_id + FROM modelData + WHERE status='waiting' + LIMIT 1 + `); - if (searchRes.total === 0) { + if (searchRes.rowCount === 0) { console.log('没有需要生成 【向量】 的数据'); global.generatingVector = false; return; } const dataItem: { id: string; q: string; userId: string } = { - id: searchRes.documents[0].id, - q: String(searchRes.documents[0]?.value?.q || ''), - userId: String(searchRes.documents[0]?.value?.userId || '') + id: searchRes.rows[0].id, + q: searchRes.rows[0].q, + userId: searchRes.rows[0].user_id }; dataId = dataItem.id; @@ -53,7 +47,7 @@ export async function generateVector(next = false): Promise { systemKey = res.systemKey; } catch (error: any) { if (error?.code === 501) { - await redis.del(dataItem.id); + await pg.query(`DELETE FROM modelData WHERE id = '${dataId}'`); generateVector(true); return; } @@ -69,15 +63,10 @@ export async function generateVector(next = false): Promise { apiKey: userApiKey || systemKey }); - // 更新 redis 向量和状态数据 - await redis.sendCommand([ - 'HMSET', - dataItem.id, - 'vector', - vectorToBuffer(vector), - 'status', - ModelDataStatusEnum.ready - ]); + // 更新 pg 向量和状态数据 + await pg.query( + `UPDATE modelData SET vector = '[${vector}]', status = 'ready' WHERE id = ${dataId}` + ); console.log(`生成向量成功: ${dataItem.id}`); diff --git a/src/service/pg.ts b/src/service/pg.ts new file mode 100644 index 000000000..02b349226 --- /dev/null +++ b/src/service/pg.ts @@ -0,0 +1,34 @@ +import { Pool } from 'pg'; + +export const connectPg = async () => { + if (global.pgClient) { + return global.pgClient; + } + + global.pgClient = new Pool({ + host: process.env.PG_HOST, + port: process.env.PG_PORT ? +process.env.PG_PORT : 5432, + user: process.env.PG_USER, + password: process.env.PG_PASSWORD, + database: process.env.PG_DB_NAME, + max: 20, + idleTimeoutMillis: 30000, + connectionTimeoutMillis: 2000 + }); + + global.pgClient.on('connect', () => { + console.log('pg connected'); + }); + global.pgClient.on('error', (err) => { + console.log(err); + global.pgClient = null; + }); + + try { + await global.pgClient.connect(); + return global.pgClient; + } catch (error) { + global.pgClient = null; + return Promise.reject(error); + } +}; diff --git a/src/types/index.d.ts b/src/types/index.d.ts index 8bbb57ee8..637bc51e5 100644 --- a/src/types/index.d.ts +++ b/src/types/index.d.ts @@ -1,10 +1,12 @@ import type { Mongoose } from 'mongoose'; import type { RedisClientType } from 'redis'; import type { Agent } from 'http'; +import type { Pool } from 'pg'; declare global { var mongodb: Mongoose | string | null; var redisClient: RedisClientType | null; + var pgClient: Pool | null; var generatingQA: boolean; var generatingAbstract: boolean; var generatingVector: boolean; diff --git a/src/types/model.d.ts b/src/types/model.d.ts index 5830cadec..b62bce5c4 100644 --- a/src/types/model.d.ts +++ b/src/types/model.d.ts @@ -12,7 +12,7 @@ export interface ModelUpdateParams { export interface ModelDataItemType { id: string; - status: 0 | 1; // 1代表向量生成完毕 + status: 'waiting' | 'ready'; q: string; // 提问词 a: string; // 原文 modelId: string; diff --git a/src/types/mongoSchema.d.ts b/src/types/mongoSchema.d.ts index bd44c84b7..3ee2b7673 100644 --- a/src/types/mongoSchema.d.ts +++ b/src/types/mongoSchema.d.ts @@ -66,11 +66,8 @@ export interface ModelDataSchema { _id: string; modelId: string; userId: string; - text: string; - q: { - id: string; - text: string; - }; + a: string; + q: string; status: ModelDataType; } diff --git a/src/types/pg.d.ts b/src/types/pg.d.ts new file mode 100644 index 000000000..ff710e084 --- /dev/null +++ b/src/types/pg.d.ts @@ -0,0 +1,10 @@ +import { ModelDataStatusEnum } from '@/constants/model'; + +export interface PgModelDataItemType { + id: string; + q: string; + a: string; + status: `${ModelDataStatusEnum}`; + model_id: string; + user_id: string; +} diff --git a/src/types/redis.d.ts b/src/types/redis.d.ts deleted file mode 100644 index dba1175e1..000000000 --- a/src/types/redis.d.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { ModelDataStatusEnum } from '@/constants/redis'; -export interface RedisModelDataItemType { - id: string; - q: string; - text: string; - status: `${ModelDataStatusEnum}`; -}