diff --git a/apps/common/job/clean_chat_job.py b/apps/common/job/clean_chat_job.py index 349ebdf85..50b467c8b 100644 --- a/apps/common/job/clean_chat_job.py +++ b/apps/common/job/clean_chat_job.py @@ -9,7 +9,7 @@ from apscheduler.schedulers.background import BackgroundScheduler from django_apscheduler.jobstores import DjangoJobStore from application.models import Application, Chat, ChatRecord from django.db.models import Q, Max -from common.utils.lock import try_lock, un_lock +from common.utils.lock import try_lock, un_lock, lock from common.utils.logger import maxkb_logger from knowledge.models import File @@ -19,6 +19,11 @@ scheduler.add_jobstore(DjangoJobStore(), "default") def clean_chat_log_job(): + clean_chat_log_job_lock() + + +@lock(lock_key='clean_chat_log_job', timeout=30) +def clean_chat_log_job_lock(): from django.utils.translation import gettext_lazy as _ maxkb_logger.info(_('start clean chat log')) now = timezone.now() diff --git a/apps/common/job/clean_debug_file_job.py b/apps/common/job/clean_debug_file_job.py index c003923bd..41493decc 100644 --- a/apps/common/job/clean_debug_file_job.py +++ b/apps/common/job/clean_debug_file_job.py @@ -1,6 +1,5 @@ # coding=utf-8 -import logging from datetime import timedelta from apscheduler.schedulers.background import BackgroundScheduler @@ -8,30 +7,40 @@ from django.db.models import Q from django.utils import timezone from django_apscheduler.jobstores import DjangoJobStore -from common.utils.lock import un_lock, try_lock +from common.utils.lock import un_lock, try_lock, lock from common.utils.logger import maxkb_logger -from knowledge.models import File +from knowledge.models import File, FileSourceType scheduler = BackgroundScheduler() scheduler.add_jobstore(DjangoJobStore(), "default") def clean_debug_file(): + clean_debug_file_lock() + + +@lock(lock_key='clean_debug_file', timeout=30) +def clean_debug_file_lock(): from django.utils.translation import gettext_lazy as _ maxkb_logger.info(_('start clean debug file')) + minutes_30_ago = timezone.now() - timedelta(minutes=30) two_hours_ago = timezone.now() - timedelta(hours=2) + one_days_ago = timezone.now() - timedelta(hours=24) # 删除对应的文件 - File.objects.filter(Q(create_time__lt=two_hours_ago) & Q(meta__debug=True)).delete() + File.objects.filter( + Q(create_time__lt=one_days_ago, source_type=FileSourceType.TEMPORARY_1_DAY.value) | + Q(create_time__lt=two_hours_ago, source_type=FileSourceType.TEMPORARY_120_MINUTE.value) | + Q(create_time__lt=minutes_30_ago, source_type=FileSourceType.TEMPORARY_30_MINUTE.value)).delete() maxkb_logger.info(_('end clean debug file')) def run(): - if try_lock('clean_debug_file', 30 * 30): + if try_lock('clean_debug_file', 30): try: scheduler.start() clean_debug_file_job = scheduler.get_job(job_id='clean_debug_file') if clean_debug_file_job is not None: clean_debug_file_job.remove() - scheduler.add_job(clean_debug_file, 'cron', hour='2', minute='0', second='0', id='clean_debug_file') + scheduler.add_job(clean_debug_file, 'cron', hour='*', minute='*/30', second='0', id='clean_debug_file') finally: un_lock('clean_debug_file') diff --git a/apps/common/utils/lock.py b/apps/common/utils/lock.py index 010b69580..2866738ba 100644 --- a/apps/common/utils/lock.py +++ b/apps/common/utils/lock.py @@ -21,7 +21,7 @@ def try_lock(key: str, timeout=None): :return: 是否获取到锁 """ if timeout is None: - timeout = 3600 # 默认超时时间为3600秒 + timeout = 3600 # 默认超时时间为3600秒 return memory_cache.add(key, 'lock', timeout=timeout) @@ -34,18 +34,20 @@ def un_lock(key: str): return memory_cache.delete(key) -def lock(lock_key): +def lock(lock_key, timeout=None): """ 给一个函数上锁 - :param lock_key: 上锁key 字符串|函数 函数返回值为字符串 + @param lock_key: 上锁key 字符串|函数 函数返回值为字符串 + @param timeout: 超时时间 :return: 装饰器函数 当前装饰器主要限制一个key只能一个线程去调用 相同key只能阻塞等待上一个任务执行完毕 不同key不需要等待 + """ def inner(func): def run(*args, **kwargs): key = lock_key(*args, **kwargs) if callable(lock_key) else lock_key try: - if try_lock(key=key): + if try_lock(key=key, timeout=timeout): return func(*args, **kwargs) finally: un_lock(key=key)