性能优化: 任务列表瘦身+详情分页+auto状态独立存储+磁盘统计缓存+db异步同步; 统计数值未读取时显示-

- /api/tasks 响应 1.57MB -> 12KB (latest_run 只带摘要, 一次读 runs.json)
- /api/stats 磁盘占用加 30s 缓存, 去除重复全量读
- 详情接口分页返回 results (默认100条/页), 前端表格分页+页码跳转
- auto 任务 pending/visited 队列迁移到 data/auto_state/ 独立文件 (tasks.json 1.6MB -> 14KB)
- MySQL 同步改后台线程异步执行 (db.sync_run_async/upsert_task_async), 不再阻塞爬虫和 API
- 启动时自动迁移存量 auto 状态
- 统计/回收站/运行中数值在未读到真实数据前显示 '-'

store.py

@@ -10,6 +10,7 @@ HERE = os.path.dirname(os.path.abspath(__file__))
 DATA_DIR = os.path.join(HERE, "data")
 TASKS_FILE = os.path.join(DATA_DIR, "tasks.json")
 RUNS_FILE = os.path.join(DATA_DIR, "runs.json")   # {task_id: [run, ...]}
+AUTO_STATE_DIR = os.path.join(DATA_DIR, "auto_state")  # auto 任务待爬/已爬队列独立存储
 
 _lock = threading.RLock()
 
@@ -140,6 +141,46 @@ def purge_trash():
         return purged
 
 
+# ---------------- auto 任务状态 (待爬队列/已爬集合) ----------------
+# 独立于 tasks.json 存储: 避免上万条 URL 撑大任务文件导致全量读写变慢
+
+def auto_state_path(task_id):
+    return os.path.join(AUTO_STATE_DIR, f"{task_id}.json")
+
+
+def load_auto_state(task_id, fallback_task=None):
+    """读取 auto 任务的 pending/visited 状态; 无独立文件时从旧任务对象迁移"""
+    with _lock:
+        path = auto_state_path(task_id)
+        if os.path.exists(path):
+            st = _load(path, {}) or {}
+            return {"pending": st.get("pending", []), "visited": st.get("visited", [])}
+        if fallback_task:
+            auto = fallback_task.get("auto") or {}
+            st = {"pending": auto.get("pending", []), "visited": auto.get("visited", [])}
+            if st["pending"] or st["visited"]:
+                _save(path, st)   # 自动迁移旧数据
+                return st
+    return {"pending": [], "visited": []}
+
+
+def save_auto_state(task_id, state):
+    with _lock:
+        _save(auto_state_path(task_id), {
+            "pending": state.get("pending", []),
+            "visited": state.get("visited", []),
+        })
+
+
+def clear_auto_state(task_id):
+    """删除 auto 状态文件"""
+    with _lock:
+        try:
+            os.remove(auto_state_path(task_id))
+        except OSError:
+            pass
+
+
 # ---------------- 运行记录 ----------------
 
 def load_runs_map():