perf: Get redis kes function (#6112)

* perf: replace redis KEYS with SCAN (#6101)

* perf: replace redis KEYS with SCAN

* test: add redis scan mock to fix unit tests

* Fix formatting in redis.ts mock functions

* fix comment word

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* perf: get redis keys function

* replace prefix code

* add pipeline delete keys

---------

Co-authored-by: lgphone <inboxcvt@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

packages/service/common/cache/index.ts

@@ -1,5 +1,5 @@
 import './init';
-import { getGlobalRedisConnection } from '../../common/redis';
+import { getAllKeysByPrefix, getGlobalRedisConnection } from '../../common/redis';
 import type { SystemCacheKeyEnum } from './type';
 import { randomUUID } from 'node:crypto';
 import { initCache } from './init';
@@ -18,11 +18,16 @@ export const refreshVersionKey = async (key: `${SystemCacheKeyEnum}`, id?: strin
 
   const val = randomUUID();
   const versionKey = id ? `${cachePrefix}${key}:${id}` : `${cachePrefix}${key}`;
+
   if (id === '*') {
-    const pattern = `${cachePrefix}${key}:*`;
-    const keys = await redis.keys(pattern);
+    const pattern = `${cachePrefix}${key}`;
+    const keys = await getAllKeysByPrefix(pattern);
     if (keys.length > 0) {
-      await redis.del(keys);
+      const pipeline = redis.pipeline();
+      for (const k of keys) {
+        pipeline.del(k);
+      }
+      await pipeline.exec();
     }
   } else {
     await redis.set(versionKey, val);

packages/service/common/redis/index.ts

@@ -44,9 +44,24 @@ export const getGlobalRedisConnection = () => {
 };
 
 export const getAllKeysByPrefix = async (key: string) => {
+  if (!key) return [];
+
   const redis = getGlobalRedisConnection();
-  const keys = (await redis.keys(`${FASTGPT_REDIS_PREFIX}${key}:*`)).map((key) =>
-    key.replace(FASTGPT_REDIS_PREFIX, '')
-  );
-  return keys;
+  const prefix = FASTGPT_REDIS_PREFIX;
+  const pattern = `${prefix}${key}:*`;
+
+  let cursor = '0';
+  const batchSize = 1000; // SCAN 每次取多少
+  const results: string[] = [];
+
+  do {
+    const [nextCursor, keys] = await redis.scan(cursor, 'MATCH', pattern, 'COUNT', batchSize);
+    cursor = nextCursor;
+
+    for (const k of keys) {
+      results.push(k.replace(FASTGPT_REDIS_PREFIX, ''));
+    }
+  } while (cursor !== '0');
+
+  return results;
 };

test/mocks/common/redis.ts

@@ -17,6 +17,12 @@ const createMockRedisClient = () => ({
   del: vi.fn().mockResolvedValue(1),
   exists: vi.fn().mockResolvedValue(0),
   keys: vi.fn().mockResolvedValue([]),
+  scan: vi.fn().mockImplementation((cursor) => {
+    // 模拟多次迭代的场景
+    if (cursor === '0') return ['100', ['key1', 'key2']];
+    if (cursor === '100') return ['0', ['key3']];
+    return ['0', []];
+  }),
 
   // Hash operations
   hget: vi.fn().mockResolvedValue(null),
@@ -53,7 +59,14 @@ const createMockRedisClient = () => ({
   sadd: vi.fn().mockResolvedValue(1),
   srem: vi.fn().mockResolvedValue(1),
   smembers: vi.fn().mockResolvedValue([]),
-  sismember: vi.fn().mockResolvedValue(0)
+  sismember: vi.fn().mockResolvedValue(0),
+
+  // pipeline
+  pipeline: vi.fn(() => ({
+    del: vi.fn().mockReturnThis(),
+    unlink: vi.fn().mockReturnThis(),
+    exec: vi.fn().mockResolvedValue([])
+  }))
 });
 
 // Mock Redis connections to prevent connection errors in tests