diff --git a/app.py b/app.py index 93c78ad..e52549b 100644 --- a/app.py +++ b/app.py @@ -104,13 +104,16 @@ def persist_cb(task_id, run): print(f"[db] 同步失败: {e}", flush=True) -def start_run(task): - """为任务启动一次爬取, 返回 (run, error)""" +def start_run(task, skip_seed=False): + """为任务启动一次爬取, 返回 (run, error) + skip_seed: auto 任务继续爬取模式, 跳过起始网址直接从缓存队列消费 + """ with JOBS_LOCK: job = JOBS.get(task["id"]) if job and job.is_running(): return None, "该任务已有正在运行的爬取" run = make_run(task) + run["skip_seed"] = bool(skip_seed) store.add_run(task["id"], run) job = CrawlJob(task, run, persist_cb) JOBS[task["id"]] = job @@ -424,6 +427,23 @@ def api_trash_clear(): # ---------------- API: 运行控制 ---------------- +@app.route("/api/tasks//continue", methods=["POST"]) +def api_continue(tid): + """继续爬取: auto 任务从待爬缓存队列接着爬 (跳过起始网址, 保留已爬集合)""" + task = store.get_task(tid) + if not task: + return jsonify({"error": "任务不存在"}), 404 + if task.get("mode") != "auto": + return jsonify({"error": "仅自动爬取任务支持继续爬取"}), 400 + st = store.load_auto_state(tid, task) + if not st.get("pending"): + return jsonify({"error": "没有待爬缓存链接，无需继续"}), 400 + run, err = start_run(task, skip_seed=True) + if err: + return jsonify({"error": err}), 409 + return jsonify(run) + + @app.route("/api/tasks//start", methods=["POST"]) def api_start(tid): task = store.get_task(tid) diff --git a/data/auto_state/t_0edd3eaa940f.json b/data/auto_state/t_0edd3eaa940f.json index d9b3e8f..559d1f3 100644 --- a/data/auto_state/t_0edd3eaa940f.json +++ b/data/auto_state/t_0edd3eaa940f.json @@ -1,5 +1,10 @@ { "pending": [ + { + "url": "https://www.cnblogs.com/", + "depth": 0, + "source": "" + }, { "url": "https://www.cnblogs.com/legacy/cate/go/", "depth": 2, @@ -46382,505 +46387,505 @@ } ], "visited": [ - "https://www.cnblogs.com/cate/rtc", - "https://www.cnblogs.com/sun-10387834/p/22395612", - "https://www.cnblogs.com/cmt/p/3464632.html", - "https://www.cnblogs.com/cmt/p/archive/2013/10", - "https://www.cnblogs.com/cmt/p/archive/2011/10", - "https://www.cnblogs.com/cmt/p/archive/2012/12", - "https://www.cnblogs.com/cmt/p/archive/2009/06", - "https://www.cnblogs.com/cate/sk", - "https://www.cnblogs.com/cmt/p/archive/2014/03", - "https://www.cnblogs.com/legacy/aggsite/mycommented", - "https://www.cnblogs.com/cmt/p/3465874.html", - "https://www.cnblogs.com/cmt/p/archive/2022/01", - "https://www.cnblogs.com/cmt/p/archive/2009/03", - "https://www.cnblogs.com/cmt/OtherPosts.html", - "https://www.cnblogs.com/cate/agentic", - "https://www.cnblogs.com/dbaxmg", - "https://www.cnblogs.com/cate/java", - "https://www.cnblogs.com/kewei/p/20887495", - "https://www.cnblogs.com/cmt/p/19320738", - "https://www.cnblogs.com/cmt/p/archive/2023/07", - "https://www.cnblogs.com/cate/pm", - "https://www.cnblogs.com/cmt/p/archive/2024/05", - "https://www.cnblogs.com/cate/erlang", - "https://www.cnblogs.com/cmt/p/archive/2021/07", - "https://www.cnblogs.com/legacy/cate/control", - "https://www.cnblogs.com/cate/articles", - "https://www.cnblogs.com/cate/cpp", - "https://www.cnblogs.com/cmt/p/archive/2016/07", - "https://www.cnblogs.com/cmt/archive/2010/10/08/1845636.html", - "https://www.cnblogs.com/cmt/p/archive/2012/02", - "https://www.cnblogs.com/cate/gis", - "https://www.cnblogs.com/cmt/p/archive/2013/11", - "https://www.cnblogs.com/Alkaid2077", - "https://www.cnblogs.com/cmt/p/archive/2017/06", - "https://www.cnblogs.com/cmt/p/archive/2013/08", - "https://www.cnblogs.com/legacy/subscription", - "https://www.cnblogs.com/xiexj/p/22393041", - "https://www.cnblogs.com/jinjiangongzuoshi/p/22385011", - "https://www.cnblogs.com/docker-mo/p/22393790", - "https://www.cnblogs.com/legacy/cate/108698", - "https://www.cnblogs.com/cmt/p/archive/2021/01", - "https://www.cnblogs.com/uniqueDong", - "https://www.cnblogs.com/legacy/cate/108724", - "https://www.cnblogs.com/cmt/p/archive/2022/08", - "https://www.cnblogs.com/cmt/p/archive/2017/03", - "https://www.cnblogs.com/helesheng", - "https://www.cnblogs.com/cate/life", - "https://www.cnblogs.com/cmt/p/archive/2009/11", - "https://www.cnblogs.com/cate/software", - "https://www.cnblogs.com/test-shirley/p/22391600", - "https://www.cnblogs.com/cmt/p/archive/2017/11", - "https://www.cnblogs.com/cmt/p/archive/2008/07", - "https://www.cnblogs.com/shanyou/p/22156095", - "https://www.cnblogs.com/legacy/cate/wpf", - "https://www.cnblogs.com/cmt/p/archive/2015/04", - "https://www.cnblogs.com/cmt/p/archive/2014/06", - "https://www.cnblogs.com/cmt/category/194500.html", - "https://www.cnblogs.com/cmt/p/archive/2018/04", - "https://www.cnblogs.com/my", - "https://www.cnblogs.com/cmt", - "https://www.cnblogs.com/aggsite/popular", - "https://www.cnblogs.com/cmt/p/archive/2012/05", - "https://www.cnblogs.com/cmt/p/archive/2012/01", - "https://www.cnblogs.com/cate/networksecurity", - "https://www.cnblogs.com/cmt/category/142965.html", - "https://www.cnblogs.com/cmt/comments", - "https://www.cnblogs.com/candidate", - "https://www.cnblogs.com/cmt/p/archive/2008/09", - "https://www.cnblogs.com/cate/scala", - "https://www.cnblogs.com/cmt/p/17520031.html", - "https://www.cnblogs.com/cmt/p/archive/2022/04", - "https://www.cnblogs.com/cmt/p/archive/2015/06", - "https://www.cnblogs.com/test-shirley", - "https://www.cnblogs.com/yuxl01/p/21437691", - "https://www.cnblogs.com/cmt/p/archive/2024/12", - "https://www.cnblogs.com/cmt/p/14595262.html", - "https://www.cnblogs.com/cate/Engineering", - "https://www.cnblogs.com/cmt/p/archive/2018/09", - "https://www.cnblogs.com/cmt/p/archive/2014/12", - "https://www.cnblogs.com/cmt/tag/%E7%BD%91%E7%AB%99%E5%85%AC%E5%91%8A", - "https://www.cnblogs.com/cmt/p/archive/2015/12", - "https://www.cnblogs.com/cmt/p/archive/2010/04", - "https://www.cnblogs.com/cmt/p/archive/2009/01", - "https://www.cnblogs.com/cate/agile", - "https://www.cnblogs.com/cmt/p/archive/2024/02", - "https://www.cnblogs.com/cmt/p/archive/2009/04", - "https://www.cnblogs.com/uniqueDong/p/22392198", - "https://www.cnblogs.com/aggsite/mycommented", - "https://www.cnblogs.com/cmt/p/archive/2020/09", - "https://www.cnblogs.com/legacy/cate/uwp", - "https://www.cnblogs.com/cmt/p/archive/2010/08", - "https://www.cnblogs.com/token-ai/p/22372387", - "https://www.cnblogs.com/cmt/tag/%E8%A1%8C%E8%A1%8CAI", - "https://www.cnblogs.com/cate/108712", - "https://www.cnblogs.com/cmt/tag/%E5%91%A8%E8%BE%B9", - "https://www.cnblogs.com/legacy/cate/dotnetcore", - "https://www.cnblogs.com/legacy/cate/mvc", - "https://www.cnblogs.com/legacy/sitehome/p/8", - "https://www.cnblogs.com/cate/llm", - "https://www.cnblogs.com/legacy/sitehome/p/5", - "https://www.cnblogs.com/shiyafan/p/20576627", - "https://www.cnblogs.com/cmt/p/archive/2018/03", - "https://www.cnblogs.com/cate/delphi", - "https://www.cnblogs.com/aspnetx", - "https://www.cnblogs.com/cate/quoted", - "https://www.cnblogs.com/cmt/p/17320765.html", - "https://www.cnblogs.com/cmt/p/archive/2009/02", - "https://www.cnblogs.com/cmt/p/14109447.html", - "https://www.cnblogs.com/cmt/p/archive/2018/07", - "https://www.cnblogs.com/erikzhouxin", - "https://www.cnblogs.com/cmt/p/archive/2025/02", - "https://www.cnblogs.com/cmt/p/archive/2019/04", - "https://www.cnblogs.com/cate/108705", - "https://www.cnblogs.com/cmt/p/archive/2019/06", - "https://www.cnblogs.com/cate/opensource", - "https://www.cnblogs.com/yuxl01", - "https://www.cnblogs.com/legacy/cate/2", - "https://www.cnblogs.com/chaogex", - "https://www.cnblogs.com/cate/codelife", - "https://www.cnblogs.com/cmt/p/archive/2012/06", - "https://www.cnblogs.com/legacy/sitehome/p/10", - "https://www.cnblogs.com/cmt/p/archive/2023/04", - "https://www.cnblogs.com/cmt/p/11316324.html", - "https://www.cnblogs.com/legacy/cate/108709", - "https://www.cnblogs.com/cate/infosec", - "https://www.cnblogs.com/cmt/p/archive/2024/03", - "https://www.cnblogs.com/xiaobaiysf/p/22117003", - "https://www.cnblogs.com/cate/108703", - "https://www.cnblogs.com/cate/react", - "https://www.cnblogs.com/cate/OracleERP", - "https://www.cnblogs.com/cate/ai", - "https://www.cnblogs.com/cate/r", - "https://www.cnblogs.com/cmt/p/archive/2013/04", - "https://www.cnblogs.com/cmt/p/archive/2011/12", - "https://www.cnblogs.com/cate/cg", - "https://www.cnblogs.com/skins.aspx", - "https://www.cnblogs.com/legacy/cate/4", - "https://www.cnblogs.com/cmt/p/archive/2009/12", - "https://www.cnblogs.com/wang_yb", - "https://www.cnblogs.com/cmt/p/archive/2016/08", - "https://www.cnblogs.com/cmt/p/archive/2010/11", - "https://www.cnblogs.com/cmt/p/archive/2014/09", - "https://www.cnblogs.com/cmt/p/archive/2010/09", - "https://www.cnblogs.com/cmt/p/archive/2012/07", - "https://www.cnblogs.com/cmt/p/archive/2017/01", - "https://www.cnblogs.com/cate/angular", - "https://www.cnblogs.com/cmt/tag/%E6%AF%8F%E5%91%A8%E7%83%AD%E7%82%B9", - "https://www.cnblogs.com/cmt/category/218599.html", - "https://www.cnblogs.com/cmt/p/archive/2014/01", - "https://www.cnblogs.com/cmt/p/archive/2025/06", - "https://www.cnblogs.com/legacy/cate/ai", - "https://www.cnblogs.com/legacy/sitehome/p/6", - "https://www.cnblogs.com/cmt/p/archive/2021/05", - "https://www.cnblogs.com/cmt/p/archive/2017/09", - "https://www.cnblogs.com/cmt/p/archive/2020/02", - "https://www.cnblogs.com/legacy/sitehome/p/7", - "https://www.cnblogs.com/cmt/p/archive/2024/11", - "https://www.cnblogs.com/cmt/p/archive/2024/07", - "https://www.cnblogs.com/cmt/p/14580194.html", - "https://www.cnblogs.com/cmt/p", - "https://www.cnblogs.com/cmt/p/archive/2016/06", - "https://www.cnblogs.com/rossiXYZ", - "https://www.cnblogs.com/cmt/p/archive/2023/10", - "https://www.cnblogs.com/cmt/p/archive/2015/02", - "https://www.cnblogs.com/cmt/p/archive/2018/06", - "https://www.cnblogs.com/cmt/p/archive/2011/02", - "https://www.cnblogs.com/cmt/p/archive/2009/10", - "https://www.cnblogs.com/legacy/cate/aspnet", - "https://www.cnblogs.com/cate/html5", - "https://www.cnblogs.com/cmt/p/archive/2022/07", - "https://www.cnblogs.com/cmt/p/archive/2016/04", - "https://www.cnblogs.com/cmt/p/archive/2020/05", - "https://www.cnblogs.com/cate/nosql", - "https://www.cnblogs.com/cmt/p/archive/2020/08", - "https://www.cnblogs.com/legacy/cate/csharp", - "https://www.cnblogs.com/nocobase", - "https://www.cnblogs.com/sitehome/p/3", - "https://www.cnblogs.com/legacy/comment", - "https://www.cnblogs.com/cmt/p/archive/2010/10", - "https://www.cnblogs.com/legacy/cate/108703", - "https://www.cnblogs.com/cmt/p/archive/2010/01", - "https://www.cnblogs.com/cmt/p/archive/2011/09", - "https://www.cnblogs.com/legacy/sitehome/p/11", - "https://www.cnblogs.com/cmt/p/archive/2021/09", - "https://www.cnblogs.com/cmt/p/archive/2020/03", - "https://www.cnblogs.com/cmt/p/14553189.html", - "https://www.cnblogs.com/cmt/p/archive/2026/04", - "https://www.cnblogs.com/ShoreKiten/p/22396532", - "https://www.cnblogs.com/legacy/cate/winform", - "https://www.cnblogs.com/cmt/MyComments.html", - "https://www.cnblogs.com/legacy/cate/ef", - "https://www.cnblogs.com/cmt/p/archive/2025/12", - "https://www.cnblogs.com/cmt/p/18302049", - "https://www.cnblogs.com/cmt/p/archive/2025/07", - "https://www.cnblogs.com/cmt/p/archive/2019/08", - "https://www.cnblogs.com/netcasewqs/p/22404834", - "https://www.cnblogs.com/pick", - "https://www.cnblogs.com/cmt/p/archive/2014/11", - "https://www.cnblogs.com/cate/c", - "https://www.cnblogs.com/cmt/p/archive/2020/12", - "https://www.cnblogs.com/cmt/p/archive/2013/06", - "https://www.cnblogs.com/cmt/p/17280054.html", - "https://www.cnblogs.com/cate/cloud", - "https://www.cnblogs.com/cmt/p/archive/2021/12", - "https://www.cnblogs.com/cmt/p/archive/2019/10", - "https://www.cnblogs.com/cmt/p/archive/2021/10", - "https://www.cnblogs.com/cmt/p/13406291.html", - "https://www.cnblogs.com/cmt/p/archive/2024/09", - "https://www.cnblogs.com/cmt/p/archive/2015/09", - "https://www.cnblogs.com/cmt/p/archive/2022/10", - "https://www.cnblogs.com/eventhorizon/p/21973620", - "https://www.cnblogs.com/cmt/p/archive/2013/05", - "https://www.cnblogs.com/cmt/post-categories", - "https://www.cnblogs.com/ShoreKiten", - "https://www.cnblogs.com/cate/4", - "https://www.cnblogs.com/xueweihan/p/21973699", - "https://www.cnblogs.com/legacy/news", - "https://www.cnblogs.com/cmt/p/archive/2022/05", - "https://www.cnblogs.com/cmt/p/archive/2016/09", - "https://www.cnblogs.com/cmt/p/archive/2017/04", - "https://www.cnblogs.com/cmt/p/archive/2013/01", - "https://www.cnblogs.com/cate/testing", - "https://www.cnblogs.com/cmt/p/archive/2019/09", - "https://www.cnblogs.com/cmt/p/archive/2025/11", - "https://www.cnblogs.com/cmt/p/archive/2015/03", - "https://www.cnblogs.com/sun-10387834", - "https://www.cnblogs.com/cmt/p/archive/2024/01", - "https://www.cnblogs.com/cate/oo", - "https://www.cnblogs.com/cmt/most-viewed", - "https://www.cnblogs.com/cmt/p/archive/2009/07", - "https://www.cnblogs.com/cate/sqlserver", - "https://www.cnblogs.com/cmt/p/18595846", - "https://www.cnblogs.com/cmt/p/archive/2022/09", - "https://www.cnblogs.com/cmt/p/archive/2025/03", - "https://www.cnblogs.com/cate/mcp", - "https://www.cnblogs.com/cmt/p/archive/2025/04", - "https://www.cnblogs.com/cmt/p/archive/2013/07", - "https://www.cnblogs.com/cmt/p/archive/2023/05", - "https://www.cnblogs.com/cmt/p/archive/2010/05", - "https://www.cnblogs.com/cmt/p/archive/2020/06", - "https://www.cnblogs.com/cate/rust", - "https://www.cnblogs.com/cmt/p/archive/2018/01", - "https://www.cnblogs.com/cmt/p/archive/2022/03", - "https://www.cnblogs.com/cmt/category/142376.html", - "https://www.cnblogs.com/cate/book", - "https://www.cnblogs.com/cate/sharepoint", - "https://www.cnblogs.com/lincats/p/21978729", - "https://www.cnblogs.com/cmt/p/archive/2011/08", - "https://www.cnblogs.com/cmt/p/archive/2021/03", - "https://www.cnblogs.com/cmt/p/archive/2015/07", - "https://www.cnblogs.com/legacy/cate/beginner", - "https://www.cnblogs.com/legacy/sitehome/p/3", - "https://www.cnblogs.com/legacy/candidate", - "https://www.cnblogs.com/cate/sap", - "https://www.cnblogs.com/hez2010/p/22380086/runtime-async-in-dotnet-11", - "https://www.cnblogs.com/legacy/cate/clr", - "https://www.cnblogs.com/cmt/p/archive/2010/06", - "https://www.cnblogs.com/cate/javascript", - "https://www.cnblogs.com/cmt/p/archive/2012/08", - "https://www.cnblogs.com/cate/3", - "https://www.cnblogs.com/aggsite/headline", - "https://www.cnblogs.com/cmt/p/archive/2023/06", - "https://www.cnblogs.com/legacy/sitehome/p/9", - "https://www.cnblogs.com/cate/mysql", - "https://www.cnblogs.com/InCerry/p/-/dotnet_week_26_7_3", - "https://www.cnblogs.com/cmt/p/archive/2017/05", - "https://www.cnblogs.com/cmt/p/archive/2025/01", - "https://www.cnblogs.com/cate/winserver", - "https://www.cnblogs.com/cate/verilog", - "https://www.cnblogs.com/iini/p/22366884", - "https://www.cnblogs.com/cate/swift", - "https://www.cnblogs.com/aggsite/myhistory", - "https://www.cnblogs.com/legacy/following", - "https://www.cnblogs.com/cate/oracle", - "https://www.cnblogs.com/cate/jquery", - "https://www.cnblogs.com/cmt/most-liked", - "https://www.cnblogs.com/jinjiangongzuoshi/p/22233575", - "https://www.cnblogs.com/following", - "https://www.cnblogs.com/cmt/p/archive/2012/11", - "https://www.cnblogs.com/cate/php", - "https://www.cnblogs.com/cmt/p/archive/2020/04", - "https://www.cnblogs.com/cmt/p/archive/2010/02", - "https://www.cnblogs.com/comment", - "https://www.cnblogs.com/cmt/p/archive/2024/10", - "https://www.cnblogs.com/cmt/p/archive/2024/08", - "https://www.cnblogs.com/cate/blockchain", - "https://www.cnblogs.com/cmt/p/14003277.html", - "https://www.cnblogs.com/cate/dart", - "https://www.cnblogs.com/cmt/p/archive/2008/08", - "https://www.cnblogs.com/cate/108724", - "https://www.cnblogs.com/legacy/cate/winrt_metro", - "https://www.cnblogs.com/cmt/p/archive/2017/12", - "https://www.cnblogs.com/cmt/p/archive/2018/10", - "https://www.cnblogs.com/cmt/p/archive/2017/02", - "https://www.cnblogs.com/cmt/p/18500368", - "https://www.cnblogs.com/cmt/p/archive/2019/05", - "https://www.cnblogs.com/aggsite/topviews", - "https://www.cnblogs.com/cmt/p/11357365.html", - "https://www.cnblogs.com/cate/108704", - "https://www.cnblogs.com/cmt/p/archive/2020/07", - "https://www.cnblogs.com/legacy/cate/108705", - "https://www.cnblogs.com/cmt/p/archive/2016/03", - "https://www.cnblogs.com/cmt/p/archive/2019/03", - "https://www.cnblogs.com/cate/database", - "https://www.cnblogs.com/cmt/p/archive/2021/02", - "https://www.cnblogs.com/cate/web", - "https://www.cnblogs.com/cmt/p/12009029.html", - "https://www.cnblogs.com/cmt/p/archive/2008/11", - "https://www.cnblogs.com/eventhorizon/p/22405144", - "https://www.cnblogs.com/legacy/cate/108712", - "https://www.cnblogs.com/cmt/p/archive/2012/10", - "https://www.cnblogs.com/cmt/p/archive/2023/12", - "https://www.cnblogs.com/cmt/p/archive/2016/05", - "https://www.cnblogs.com/zhayujie/p/21524003/cli-for-agents", - "https://www.cnblogs.com/cate/bigdata", - "https://www.cnblogs.com/cate/job", - "https://www.cnblogs.com/cmt/p/archive/2021/06", - "https://www.cnblogs.com/cmt/p/archive/2019/07", - "https://www.cnblogs.com/cmt/p/archive/2014/07", - "https://www.cnblogs.com/cmt/p/archive/2018/08", - "https://www.cnblogs.com/cmt/p/archive/2020/01", - "https://www.cnblogs.com/wang_yb/p/22393087", - "https://www.cnblogs.com/cmt/p/archive/2016/10", - "https://www.cnblogs.com/cate/osx", - "https://www.cnblogs.com/cmt/p/archive/2023/01", - "https://www.cnblogs.com/erikzhouxin/p/22403312", - "https://www.cnblogs.com/cate/android", - "https://www.cnblogs.com/cmt/p/archive/2022/06", - "https://www.cnblogs.com/cmt/p/archive/2015/01", - "https://www.cnblogs.com/cate/design", - "https://www.cnblogs.com/cmt/p/archive/2023/11", - "https://www.cnblogs.com/cmt/category/203680.html", - "https://www.cnblogs.com/cmt/tag/.NET%E8%B7%A8%E5%B9%B3%E5%8F%B0", - "https://www.cnblogs.com/legacy/pick", - "https://www.cnblogs.com/cate/108709", - "https://www.cnblogs.com/cmt/p/archive/2008/10", - "https://www.cnblogs.com/cate/python", - "https://www.cnblogs.com/cmt/p/archive/2019/12", - "https://www.cnblogs.com/docker-mo", - "https://www.cnblogs.com/cate/go", - "https://www.cnblogs.com/cmt/tag/%E7%9B%B4%E6%92%AD", - "https://www.cnblogs.com/cate/vue", - "https://www.cnblogs.com/cmt/p/archive/2024/04", - "https://www.cnblogs.com/cmt/p/archive/2021/04", - "https://www.cnblogs.com/cmt/category/179092.html", - "https://www.cnblogs.com/cmt/p/archive/2019/02", - "https://www.cnblogs.com/cmt/p/19350596", - "https://www.cnblogs.com/aggsite/topdiggs", - "https://www.cnblogs.com/cmt/p/archive/2009/05", - "https://www.cnblogs.com/cmt/tag/docker", - "https://www.cnblogs.com/cmt/p/archive/2016/01", - "https://www.cnblogs.com/cate/bpm", - "https://www.cnblogs.com/cmt/p/archive/2013/03", - "https://www.cnblogs.com/aspnetx/p/22392937", - "https://www.cnblogs.com/cate/harmonyos", - "https://www.cnblogs.com/cmt/tag/%E9%98%BF%E9%87%8C%E4%BA%91", - "https://www.cnblogs.com/cate/ddd", - "https://www.cnblogs.com/cmt/p/archive/2026/06", "https://www.cnblogs.com/subscription", - "https://www.cnblogs.com/cate/ml", - "https://www.cnblogs.com/cmt/category/220662.html", - "https://www.cnblogs.com/cmt/p/18341478", - "https://www.cnblogs.com/legacy/cate/108701", - "https://www.cnblogs.com/cmt/p/archive/2026/01", - "https://www.cnblogs.com/cmt/p/archive/2026/03", - "https://www.cnblogs.com/cmt/p/archive/2023/02", - "https://www.cnblogs.com/cate/all", - "https://www.cnblogs.com/cmt/p/archive/2020/10", - "https://www.cnblogs.com/cmt/p/archive/2009/08", - "https://www.cnblogs.com/cmt/category/142130.html", - "https://www.cnblogs.com/eventhorizon", - "https://www.cnblogs.com/cmt/p/archive/2023/03", - "https://www.cnblogs.com/cmt/p/archive/2022/02", - "https://www.cnblogs.com/cmt/p/archive/2008/06", - "https://www.cnblogs.com/cmt/p/archive/2010/07", - "https://www.cnblogs.com/legacy/cate/java", - "https://www.cnblogs.com/cate/ios", - "https://www.cnblogs.com/cate/flutter", - "https://www.cnblogs.com/cate/misc", - "https://www.cnblogs.com/cmt/p/archive/2014/04", - "https://www.cnblogs.com/cmt/p/archive/2013/12", - "https://www.cnblogs.com/legacy/sitehome/p/2", - "https://www.cnblogs.com/cmt/p/archive/2011/11", - "https://www.cnblogs.com/cmt/p/archive/2023/09", - "https://www.cnblogs.com/legacy/cate/108704", - "https://www.cnblogs.com/cmt/p/archive/2026/02", - "https://www.cnblogs.com/cmt/p/archive/2025/10", - "https://www.cnblogs.com/cmt/p/archive/2014/05", - "https://www.cnblogs.com/cate/108701", - "https://www.cnblogs.com/cmt/p/archive/2012/09", - "https://www.cnblogs.com/InCerry", - "https://www.cnblogs.com/cmt/p/14585828.html", - "https://www.cnblogs.com/legacy/cate/nhibernate", - "https://www.cnblogs.com/cmt/p/archive/2015/10", - "https://www.cnblogs.com/tangshiye/p/22363402", - "https://www.cnblogs.com/cmt/p/archive/2010/12", - "https://www.cnblogs.com/cate/108698", - "https://www.cnblogs.com/cmt/p/archive/2019/01", - "https://www.cnblogs.com/cmt/p/archive/2025/05", - "https://www.cnblogs.com/cate/dp", - "https://www.cnblogs.com/cmt/p/archive/2016/12", - "https://www.cnblogs.com/cmt/category/142131.html", - "https://www.cnblogs.com/cmt/p/archive/2016/11", - "https://www.cnblogs.com/cmt/p/archive/2021/08", - "https://www.cnblogs.com/cmt/p/archive/2008/12", - "https://www.cnblogs.com/legacy/sitehome/p/4", - "https://www.cnblogs.com/cmt/p/archive/2011/05", - "https://www.cnblogs.com/legacy", - "https://www.cnblogs.com/legacy/sitehome/p/200", - "https://www.cnblogs.com/cmt/p/archive/2011/03", - "https://www.cnblogs.com/cmt/p/archive/2026/05", - "https://www.cnblogs.com/cmt/p/archive/2012/04", - "https://www.cnblogs.com/cmt/p/archive/2022/12", - "https://www.cnblogs.com/cate/otherlang", - "https://www.cnblogs.com/cate/algorithm", - "https://www.cnblogs.com/cmt/p/archive/2012/03", - "https://www.cnblogs.com/cmt/p/archive/2018/02", - "https://www.cnblogs.com/cmt/p/archive/2026/07", - "https://www.cnblogs.com/cmt/p/archive/2024/06", - "https://www.cnblogs.com/cmt/most-commented", - "https://www.cnblogs.com/cmt/p/archive/2014/02", - "https://www.cnblogs.com/sitehome/p/100", - "https://www.cnblogs.com/corp/posts", - "https://www.cnblogs.com/cate/linux", - "https://www.cnblogs.com/cmt/p/archive/2011/07", - "https://www.cnblogs.com/netcasewqs", - "https://www.cnblogs.com/cate/postgresql", - "https://www.cnblogs.com/cate/gamedev", - "https://www.cnblogs.com/cmt/p/archive/2013/02", - "https://www.cnblogs.com/", - "https://www.cnblogs.com/cmt/articles/17677736.html", - "https://www.cnblogs.com/Alkaid2077/p/22405412", - "https://www.cnblogs.com/shenchuanchao/p/22236345/inno-setup-dotnet-packaging-guide", - "https://www.cnblogs.com/helesheng/p/22361625", - "https://www.cnblogs.com/cmt/p/archive/2019/11", - "https://www.cnblogs.com/cmt/p/archive/2010/03", - "https://www.cnblogs.com/cmt/tag/%E6%95%85%E9%9A%9C%E5%85%AC%E5%91%8A", - "https://www.cnblogs.com/cate/ruby", - "https://www.cnblogs.com/cmt/p/archive/2011/01", - "https://www.cnblogs.com/legacy/cate/vs2010", - "https://www.cnblogs.com/cmt/p/archive/2018/11", - "https://www.cnblogs.com/cmt/p/archive/2018/05", - "https://www.cnblogs.com/cate/2", - "https://www.cnblogs.com/cate/k8s", - "https://www.cnblogs.com/cmt/p/archive/2023/08", - "https://www.cnblogs.com/cate/ai-misc", - "https://www.cnblogs.com/legacy/cate/python", - "https://www.cnblogs.com/legacy/cate/all", - "https://www.cnblogs.com/cmt/p/archive/2014/08", - "https://www.cnblogs.com/cmt/p/archive/2017/08", - "https://www.cnblogs.com/guangzan/p/20774394", - "https://www.cnblogs.com/aggsite/topdigged24h", - "https://www.cnblogs.com/cate/dynamics", - "https://www.cnblogs.com/cmt/p/archive/2013/09", - "https://www.cnblogs.com/cmt/p/archive/2014/10", - "https://www.cnblogs.com/cate/mobile", - "https://www.cnblogs.com/xiexj/p/22344233", - "https://www.cnblogs.com/cmt/p/archive/2016/02", - "https://www.cnblogs.com/cmt/p/archive/2015/11", - "https://www.cnblogs.com/shanyou", - "https://www.cnblogs.com/cate/win7", - "https://www.cnblogs.com/rossiXYZ/p/22350905", - "https://www.cnblogs.com/cmt/p/archive/2022/11", - "https://www.cnblogs.com/cmt/tag", - "https://www.cnblogs.com/cmt/archive/2009/09/26/1574616.html", - "https://www.cnblogs.com/legacy/cate/silverlight", - "https://www.cnblogs.com/cmt/p/archive/2017/07", - "https://www.cnblogs.com/cmt/p/archive/2011/04", - "https://www.cnblogs.com/shanyou/p/22355348", - "https://www.cnblogs.com/legacy/aggsite/mydigged", - "https://www.cnblogs.com/cmt/p/archive/2025/08", - "https://www.cnblogs.com/cmt/p/archive/2018/12", - "https://www.cnblogs.com/cmt/p/archive/2015/05", - "https://www.cnblogs.com/zhangs1986/p/22340941", - "https://www.cnblogs.com/sitehome/p/2", - "https://www.cnblogs.com/cmt/tag/%E4%BA%91%E8%AE%A1%E7%AE%97", - "https://www.cnblogs.com/cmt/p/archive/2021/11", - "https://www.cnblogs.com/sheng_chao/p/22293492", - "https://www.cnblogs.com/cmt/p/19316348", - "https://www.cnblogs.com/cmt/p/archive/2025/09", - "https://www.cnblogs.com/nocobase/p/22405239", - "https://www.cnblogs.com/cmt/p/archive/2017/10", - "https://www.cnblogs.com/cate/translate", - "https://www.cnblogs.com/cmt/p/11335456.html", - "https://www.cnblogs.com/cmt/category/205680.html", - "https://www.cnblogs.com/legacy/cate/wcf", - "https://www.cnblogs.com/cmt/category/142128.html", - "https://www.cnblogs.com/cate/eos", - "https://www.cnblogs.com/aggsite/mydigged", - "https://www.cnblogs.com/chaogex/p/22404906", - "https://www.cnblogs.com/cmt/category/142129.html", - "https://www.cnblogs.com/cmt/p/archive/2020/11", - "https://www.cnblogs.com/hez2010/p/20979630/build-a-very-large-array-in-dotnet", - "https://www.cnblogs.com/dbaxmg/p/22398722", - "https://www.cnblogs.com/cmt/p/archive/2015/08", + "https://www.cnblogs.com/cmt/p/archive/2015/12", + "https://www.cnblogs.com/nocobase", + "https://www.cnblogs.com/cmt/p/archive/2020/09", + "https://www.cnblogs.com/cmt/p/archive/2024/12", "https://www.cnblogs.com/cmt/p/archive/2011/06", - "https://www.cnblogs.com/legacy/cate/xna", + "https://www.cnblogs.com/cmt/p/archive/2022/11", + "https://www.cnblogs.com/candidate", + "https://www.cnblogs.com/xiaobaiysf/p/22117003", + "https://www.cnblogs.com/cmt/most-commented", + "https://www.cnblogs.com/cate/oracle", + "https://www.cnblogs.com/shenchuanchao/p/22236345/inno-setup-dotnet-packaging-guide", + "https://www.cnblogs.com/cmt/category/142965.html", + "https://www.cnblogs.com/cate/testing", + "https://www.cnblogs.com/cmt/p/archive/2015/01", + "https://www.cnblogs.com/cate/life", + "https://www.cnblogs.com/cmt/p/archive/2011/01", + "https://www.cnblogs.com/cate/dart", + "https://www.cnblogs.com/cate/mobile", + "https://www.cnblogs.com/sun-10387834", + "https://www.cnblogs.com/cmt/p/archive/2022/04", + "https://www.cnblogs.com/cmt/p/archive/2016/07", + "https://www.cnblogs.com/cmt/p/archive/2026/04", + "https://www.cnblogs.com/cmt/p/archive/2018/05", + "https://www.cnblogs.com/cmt/p/archive/2026/07", + "https://www.cnblogs.com/cate/108709", + "https://www.cnblogs.com/Alkaid2077", + "https://www.cnblogs.com/cmt/p/archive/2012/03", + "https://www.cnblogs.com/cmt/p/archive/2025/07", + "https://www.cnblogs.com/cmt/p/3465874.html", + "https://www.cnblogs.com/cmt/comments", + "https://www.cnblogs.com/cmt/p/archive/2014/09", + "https://www.cnblogs.com/cmt/p/archive/2017/01", + "https://www.cnblogs.com/legacy/cate/control", + "https://www.cnblogs.com/cmt/p/18595846", + "https://www.cnblogs.com/sitehome/p/100", + "https://www.cnblogs.com/cmt/p/archive/2012/10", + "https://www.cnblogs.com/cmt/p/archive/2021/05", + "https://www.cnblogs.com/cmt/p/archive/2014/05", + "https://www.cnblogs.com/legacy/sitehome/p/8", + "https://www.cnblogs.com/cate/bigdata", + "https://www.cnblogs.com/xiexj", + "https://www.cnblogs.com/cmt/p/archive/2010/10", + "https://www.cnblogs.com/cate/php", + "https://www.cnblogs.com/cmt/tag/%E7%9B%B4%E6%92%AD", + "https://www.cnblogs.com/legacy/sitehome/p/200", + "https://www.cnblogs.com/legacy/following", + "https://www.cnblogs.com/cmt/p/archive/2016/09", + "https://www.cnblogs.com/cmt/p/archive/2010/05", + "https://www.cnblogs.com/legacy/sitehome/p/2", + "https://www.cnblogs.com/cmt/p/archive/2014/12", + "https://www.cnblogs.com/cmt/p/13406291.html", + "https://www.cnblogs.com/cmt/p/archive/2021/07", + "https://www.cnblogs.com/cmt/category/179092.html", + "https://www.cnblogs.com/legacy/comment", + "https://www.cnblogs.com/cmt/p/archive/2022/03", + "https://www.cnblogs.com/cmt/p/11335456.html", + "https://www.cnblogs.com/cate/dynamics", + "https://www.cnblogs.com/legacy/aggsite/mycommented", + "https://www.cnblogs.com/cate/108704", + "https://www.cnblogs.com/xiexj/p/22393041", + "https://www.cnblogs.com/cmt/p/archive/2023/04", + "https://www.cnblogs.com/cate/108705", + "https://www.cnblogs.com/cmt/p/archive/2022/02", + "https://www.cnblogs.com/shiyafan/p/20576627", + "https://www.cnblogs.com/cmt/p/archive/2013/03", + "https://www.cnblogs.com/cmt/p/archive/2012/01", + "https://www.cnblogs.com/cmt/p/archive/2023/06", + "https://www.cnblogs.com/cmt/most-liked", + "https://www.cnblogs.com/cmt/p/18500368", + "https://www.cnblogs.com/cmt/p/archive/2018/08", + "https://www.cnblogs.com/cate/rtc", + "https://www.cnblogs.com/cmt/p/archive/2018/12", + "https://www.cnblogs.com/uniqueDong", + "https://www.cnblogs.com/legacy/cate/ef", + "https://www.cnblogs.com/cmt/p/archive/2010/01", + "https://www.cnblogs.com/cmt/p/archive/2017/05", + "https://www.cnblogs.com/cmt/p/archive/2015/07", + "https://www.cnblogs.com/legacy", + "https://www.cnblogs.com/legacy/sitehome/p/3", + "https://www.cnblogs.com/cmt/p/archive/2009/05", + "https://www.cnblogs.com/cmt/p/archive/2024/03", + "https://www.cnblogs.com/cmt/p/archive/2016/12", + "https://www.cnblogs.com/aggsite/topdigged24h", + "https://www.cnblogs.com/nocobase/p/22405239", + "https://www.cnblogs.com/cmt/p/archive/2008/07", + "https://www.cnblogs.com/ShoreKiten", + "https://www.cnblogs.com/cmt/p/archive/2020/02", + "https://www.cnblogs.com/cmt/p/archive/2016/11", + "https://www.cnblogs.com/cmt/p/archive/2023/07", + "https://www.cnblogs.com/cmt/p/archive/2011/04", + "https://www.cnblogs.com/cmt/p/archive/2025/03", + "https://www.cnblogs.com/yuxl01", + "https://www.cnblogs.com/cmt/p/archive/2013/04", + "https://www.cnblogs.com/cmt/p/archive/2009/01", + "https://www.cnblogs.com/cmt/p/archive/2018/09", + "https://www.cnblogs.com/eventhorizon", + "https://www.cnblogs.com/cate/mcp", + "https://www.cnblogs.com/cmt/tag/%E4%BA%91%E8%AE%A1%E7%AE%97", + "https://www.cnblogs.com/cate/ai-misc", + "https://www.cnblogs.com/cmt/p/archive/2024/06", + "https://www.cnblogs.com/cate/gamedev", + "https://www.cnblogs.com/legacy/cate/ai", + "https://www.cnblogs.com/cmt/p/19320738", + "https://www.cnblogs.com/cate/108703", + "https://www.cnblogs.com/cmt/p/archive/2026/05", + "https://www.cnblogs.com/cmt/p/archive/2012/05", + "https://www.cnblogs.com/cmt/p/archive/2016/06", + "https://www.cnblogs.com/shanyou/p/22156095", + "https://www.cnblogs.com/legacy/cate/108712", + "https://www.cnblogs.com/legacy/aggsite/mydigged", + "https://www.cnblogs.com/cmt/p/archive/2012/04", + "https://www.cnblogs.com/sitehome/p/2", + "https://www.cnblogs.com/cate/rust", + "https://www.cnblogs.com/cmt/category/203680.html", + "https://www.cnblogs.com/cmt/p/archive/2026/03", + "https://www.cnblogs.com/cmt/p/archive/2020/06", + "https://www.cnblogs.com/cmt/p/archive/2025/08", + "https://www.cnblogs.com/cate/swift", + "https://www.cnblogs.com/cmt/p/archive/2019/01", + "https://www.cnblogs.com/cmt/p/archive/2024/10", + "https://www.cnblogs.com/cate/3", + "https://www.cnblogs.com/xueweihan/p/21973699", + "https://www.cnblogs.com/cmt/p/archive/2013/09", + "https://www.cnblogs.com/cmt/p/archive/2024/04", + "https://www.cnblogs.com/cmt/p/archive/2013/07", + "https://www.cnblogs.com/cmt", + "https://www.cnblogs.com/cmt/p/archive/2020/07", + "https://www.cnblogs.com/jinjiangongzuoshi/p/22233575", + "https://www.cnblogs.com/cate/108712", + "https://www.cnblogs.com/cate/r", + "https://www.cnblogs.com/cmt/p/archive/2016/08", + "https://www.cnblogs.com/cmt/p/archive/2010/07", + "https://www.cnblogs.com/cate/k8s", + "https://www.cnblogs.com/cate/harmonyos", + "https://www.cnblogs.com/cmt/p/archive/2020/03", + "https://www.cnblogs.com/cate/scala", + "https://www.cnblogs.com/legacy/cate/vs2010", + "https://www.cnblogs.com/cate/infosec", + "https://www.cnblogs.com/cmt/p/archive/2026/02", + "https://www.cnblogs.com/cmt/p/archive/2021/10", + "https://www.cnblogs.com/cmt/p/archive/2018/04", + "https://www.cnblogs.com/cmt/p/14595262.html", + "https://www.cnblogs.com/legacy/cate/beginner", + "https://www.cnblogs.com/legacy/cate/108701", + "https://www.cnblogs.com/cmt/category/218599.html", + "https://www.cnblogs.com/cate/algorithm", + "https://www.cnblogs.com/cmt/p/archive/2024/07", + "https://www.cnblogs.com/", + "https://www.cnblogs.com/cate/sap", + "https://www.cnblogs.com/cmt/p/archive/2014/02", + "https://www.cnblogs.com/cmt/p/archive/2014/10", + "https://www.cnblogs.com/cmt/p/archive/2012/09", + "https://www.cnblogs.com/cmt/p/archive/2015/11", "https://www.cnblogs.com/cmt/p/archive/2009/09", + "https://www.cnblogs.com/cmt/p/archive/2017/10", + "https://www.cnblogs.com/cmt/category/194500.html", + "https://www.cnblogs.com/cate/vue", + "https://www.cnblogs.com/aggsite/myhistory", + "https://www.cnblogs.com/sheng_chao/p/22293492", + "https://www.cnblogs.com/cmt/tag/%E6%95%85%E9%9A%9C%E5%85%AC%E5%91%8A", + "https://www.cnblogs.com/aspnetx/p/22392937", + "https://www.cnblogs.com/cmt/tag/%E7%BD%91%E7%AB%99%E5%85%AC%E5%91%8A", + "https://www.cnblogs.com/cmt/p/archive/2022/07", + "https://www.cnblogs.com/cate/go", + "https://www.cnblogs.com/aggsite/mydigged", + "https://www.cnblogs.com/cmt/p/archive/2010/12", + "https://www.cnblogs.com/legacy/cate/winform", + "https://www.cnblogs.com/cmt/p/archive/2010/03", + "https://www.cnblogs.com/cmt/tag/.NET%E8%B7%A8%E5%B9%B3%E5%8F%B0", + "https://www.cnblogs.com/legacy/cate/silverlight", + "https://www.cnblogs.com/cate/sk", + "https://www.cnblogs.com/aggsite/popular", + "https://www.cnblogs.com/cmt/p/archive/2023/10", + "https://www.cnblogs.com/cmt/p/archive/2019/10", + "https://www.cnblogs.com/cmt/p/18341478", + "https://www.cnblogs.com/cmt/p/14109447.html", + "https://www.cnblogs.com/cmt/most-viewed", + "https://www.cnblogs.com/cate/mysql", + "https://www.cnblogs.com/cmt/p/17280054.html", + "https://www.cnblogs.com/cmt/tag", + "https://www.cnblogs.com/shanyou", + "https://www.cnblogs.com/cmt/p/archive/2014/01", + "https://www.cnblogs.com/legacy/cate/108705", + "https://www.cnblogs.com/cate/web", + "https://www.cnblogs.com/cmt/p/14585828.html", + "https://www.cnblogs.com/cmt/p/archive/2021/08", + "https://www.cnblogs.com/cmt/p/archive/2016/05", + "https://www.cnblogs.com/cmt/p/archive/2025/02", + "https://www.cnblogs.com/legacy/sitehome/p/11", + "https://www.cnblogs.com/cmt/p/archive/2018/01", + "https://www.cnblogs.com/cmt/p/archive/2010/09", + "https://www.cnblogs.com/cate/python", + "https://www.cnblogs.com/cmt/p/archive/2016/10", + "https://www.cnblogs.com/cate/translate", + "https://www.cnblogs.com/cmt/p/archive/2018/02", + "https://www.cnblogs.com/ShoreKiten/p/22396532", + "https://www.cnblogs.com/cmt/p/archive/2018/11", + "https://www.cnblogs.com/cmt/OtherPosts.html", + "https://www.cnblogs.com/cmt/tag/docker", + "https://www.cnblogs.com/cmt/p/archive/2023/09", + "https://www.cnblogs.com/chaogex", + "https://www.cnblogs.com/cmt/p/archive/2015/05", + "https://www.cnblogs.com/cmt/tag/%E8%A1%8C%E8%A1%8CAI", + "https://www.cnblogs.com/cmt/p/archive/2009/10", + "https://www.cnblogs.com/cmt/p/archive/2025/11", + "https://www.cnblogs.com/cmt/p/archive/2025/06", + "https://www.cnblogs.com/legacy/cate/108704", + "https://www.cnblogs.com/cate/agentic", + "https://www.cnblogs.com/cmt/p/archive/2017/09", + "https://www.cnblogs.com/chaogex/p/22404906", + "https://www.cnblogs.com/cate/articles", + "https://www.cnblogs.com/cmt/p/archive/2017/11", + "https://www.cnblogs.com/cmt/p/archive/2014/08", + "https://www.cnblogs.com/cate/cloud", + "https://www.cnblogs.com/cmt/p/archive/2019/07", + "https://www.cnblogs.com/cate/ai", + "https://www.cnblogs.com/cmt/p/archive/2014/03", + "https://www.cnblogs.com/cmt/category/142131.html", + "https://www.cnblogs.com/legacy/cate/108703", + "https://www.cnblogs.com/dbaxmg", + "https://www.cnblogs.com/cmt/p/archive/2011/02", + "https://www.cnblogs.com/cmt/p/archive/2018/03", + "https://www.cnblogs.com/cmt/p/archive/2015/10", + "https://www.cnblogs.com/test-shirley", + "https://www.cnblogs.com/cmt/p/archive/2023/05", + "https://www.cnblogs.com/cmt/p/18302049", + "https://www.cnblogs.com/iini/p/22366884", + "https://www.cnblogs.com/cmt/p/archive/2011/07", + "https://www.cnblogs.com/cmt/p/archive/2025/09", + "https://www.cnblogs.com/cmt/p/archive/2020/04", + "https://www.cnblogs.com/cmt/p/archive/2010/11", + "https://www.cnblogs.com/cmt/p/archive/2022/08", + "https://www.cnblogs.com/cmt/p/11316324.html", + "https://www.cnblogs.com/cmt/p/archive/2019/02", + "https://www.cnblogs.com/cmt/p/archive/2021/03", + "https://www.cnblogs.com/tangshiye/p/22363402", + "https://www.cnblogs.com/hez2010/p/22380086/runtime-async-in-dotnet-11", + "https://www.cnblogs.com/legacy/cate/wcf", + "https://www.cnblogs.com/cmt/p/17320765.html", + "https://www.cnblogs.com/sun-10387834/p/22395612", + "https://www.cnblogs.com/aspnetx", + "https://www.cnblogs.com/cate/postgresql", + "https://www.cnblogs.com/legacy/cate/2", + "https://www.cnblogs.com/cmt/p/archive/2019/11", + "https://www.cnblogs.com/aggsite/topviews", + "https://www.cnblogs.com/cmt/p/archive/2023/11", + "https://www.cnblogs.com/cmt/p/archive/2025/04", + "https://www.cnblogs.com/cmt/p/archive/2015/03", + "https://www.cnblogs.com/cmt/p/archive/2010/08", + "https://www.cnblogs.com/cmt/p/archive/2016/01", + "https://www.cnblogs.com/cmt/p/archive/2013/06", + "https://www.cnblogs.com/cmt/p/archive/2018/07", + "https://www.cnblogs.com/cmt/p/archive/2016/03", + "https://www.cnblogs.com/cate/javascript", + "https://www.cnblogs.com/cate/win7", + "https://www.cnblogs.com/kewei/p/20887495", + "https://www.cnblogs.com/legacy/cate/4", + "https://www.cnblogs.com/cmt/p/archive/2020/12", + "https://www.cnblogs.com/cmt/p/archive/2024/05", + "https://www.cnblogs.com/cmt/p/archive/2024/02", + "https://www.cnblogs.com/legacy/cate/uwp", + "https://www.cnblogs.com/cmt/p/archive/2017/06", + "https://www.cnblogs.com/cate/erlang", + "https://www.cnblogs.com/cmt/p/archive/2011/10", + "https://www.cnblogs.com/cmt/p/archive/2022/10", + "https://www.cnblogs.com/legacy/cate/csharp", + "https://www.cnblogs.com/cate/linux", + "https://www.cnblogs.com/cate/pm", + "https://www.cnblogs.com/cmt/tag/%E9%98%BF%E9%87%8C%E4%BA%91", + "https://www.cnblogs.com/cate/dp", + "https://www.cnblogs.com/cmt/p/archive/2010/06", + "https://www.cnblogs.com/cmt/p/archive/2008/12", + "https://www.cnblogs.com/cmt/p/archive/2015/04", + "https://www.cnblogs.com/cate/html5", + "https://www.cnblogs.com/cate/design", + "https://www.cnblogs.com/cmt/p/archive/2009/07", + "https://www.cnblogs.com/legacy/cate/winrt_metro", + "https://www.cnblogs.com/cate/blockchain", + "https://www.cnblogs.com/cmt/category/142129.html", + "https://www.cnblogs.com/cmt/p/archive/2021/01", + "https://www.cnblogs.com/cmt/p/archive/2021/09", + "https://www.cnblogs.com/following", + "https://www.cnblogs.com/cmt/p/archive/2026/01", + "https://www.cnblogs.com/cmt/p/archive/2010/02", + "https://www.cnblogs.com/cmt/p/12009029.html", + "https://www.cnblogs.com/cate/java", + "https://www.cnblogs.com/cmt/p/archive/2013/01", + "https://www.cnblogs.com/cmt/p/archive/2016/04", + "https://www.cnblogs.com/cate/c", + "https://www.cnblogs.com/cmt/p/archive/2017/07", + "https://www.cnblogs.com/cate/quoted", + "https://www.cnblogs.com/cate/software", + "https://www.cnblogs.com/cate/angular", + "https://www.cnblogs.com/cate/misc", + "https://www.cnblogs.com/cmt/p/archive/2019/08", + "https://www.cnblogs.com/cmt/p/archive/2024/01", + "https://www.cnblogs.com/cmt/p/archive/2023/12", + "https://www.cnblogs.com/cate/Engineering", + "https://www.cnblogs.com/cmt/category/205680.html", + "https://www.cnblogs.com/hez2010/p/20979630/build-a-very-large-array-in-dotnet", + "https://www.cnblogs.com/cmt/p/14580194.html", + "https://www.cnblogs.com/lincats/p/21978729", + "https://www.cnblogs.com/wang_yb", + "https://www.cnblogs.com/cmt/p/archive/2026/06", + "https://www.cnblogs.com/cmt/p/archive/2024/11", + "https://www.cnblogs.com/cate/networksecurity", + "https://www.cnblogs.com/cmt/p/archive/2011/05", + "https://www.cnblogs.com/cmt/p/archive/2024/09", + "https://www.cnblogs.com/cmt/p/archive/2023/08", + "https://www.cnblogs.com/cmt/p/archive/2022/12", + "https://www.cnblogs.com/cate/gis", + "https://www.cnblogs.com/cmt/archive/2009/09/26/1574616.html", + "https://www.cnblogs.com/cmt/p/archive/2021/06", + "https://www.cnblogs.com/cmt/post-categories", + "https://www.cnblogs.com/cmt/p", + "https://www.cnblogs.com/cmt/p/archive/2015/02", + "https://www.cnblogs.com/cmt/p/archive/2009/02", + "https://www.cnblogs.com/erikzhouxin/p/22403312", + "https://www.cnblogs.com/cmt/p/archive/2012/06", + "https://www.cnblogs.com/corp/posts", + "https://www.cnblogs.com/cate/ml", + "https://www.cnblogs.com/netcasewqs", + "https://www.cnblogs.com/legacy/sitehome/p/5", + "https://www.cnblogs.com/my", + "https://www.cnblogs.com/cate/job", + "https://www.cnblogs.com/legacy/cate/java", + "https://www.cnblogs.com/cmt/p/archive/2008/09", + "https://www.cnblogs.com/cmt/p/archive/2014/06", + "https://www.cnblogs.com/cmt/p/archive/2025/10", + "https://www.cnblogs.com/cmt/category/142376.html", + "https://www.cnblogs.com/wang_yb/p/22393087", + "https://www.cnblogs.com/cmt/p/archive/2022/01", + "https://www.cnblogs.com/cmt/p/archive/2017/12", + "https://www.cnblogs.com/legacy/subscription", + "https://www.cnblogs.com/cmt/category/142130.html", + "https://www.cnblogs.com/cmt/p/archive/2019/03", + "https://www.cnblogs.com/cmt/p/archive/2011/03", + "https://www.cnblogs.com/cmt/p/archive/2012/11", + "https://www.cnblogs.com/cate/cg", + "https://www.cnblogs.com/cmt/p/19350596", + "https://www.cnblogs.com/cmt/category/142128.html", + "https://www.cnblogs.com/InCerry/p/-/dotnet_week_26_7_3", + "https://www.cnblogs.com/pick", + "https://www.cnblogs.com/cmt/p/14553189.html", + "https://www.cnblogs.com/uniqueDong/p/22392198", + "https://www.cnblogs.com/cmt/p/archive/2014/04", + "https://www.cnblogs.com/cmt/p/archive/2015/09", + "https://www.cnblogs.com/cmt/p/archive/2021/12", + "https://www.cnblogs.com/rossiXYZ/p/22350905", + "https://www.cnblogs.com/cate/ios", + "https://www.cnblogs.com/cmt/p/archive/2020/11", + "https://www.cnblogs.com/cmt/p/archive/2019/09", + "https://www.cnblogs.com/sitehome/p/3", + "https://www.cnblogs.com/legacy/cate/clr", + "https://www.cnblogs.com/cate/winserver", + "https://www.cnblogs.com/cmt/p/archive/2013/05", + "https://www.cnblogs.com/cmt/p/archive/2013/10", + "https://www.cnblogs.com/cate/database", + "https://www.cnblogs.com/skins.aspx", + "https://www.cnblogs.com/cmt/p/3464632.html", + "https://www.cnblogs.com/cmt/p/archive/2017/08", + "https://www.cnblogs.com/cmt/p/14003277.html", + "https://www.cnblogs.com/legacy/cate/nhibernate", + "https://www.cnblogs.com/cate/book", + "https://www.cnblogs.com/cate/ddd", + "https://www.cnblogs.com/cmt/p/archive/2023/03", + "https://www.cnblogs.com/cmt/p/archive/2011/08", + "https://www.cnblogs.com/cmt/p/archive/2010/04", + "https://www.cnblogs.com/cmt/p/archive/2022/09", "https://www.cnblogs.com/wackysoft/p/22144512", - "https://www.cnblogs.com/xiexj" + "https://www.cnblogs.com/cate/cpp", + "https://www.cnblogs.com/cate/llm", + "https://www.cnblogs.com/cmt/p/archive/2008/10", + "https://www.cnblogs.com/cate/bpm", + "https://www.cnblogs.com/cate/android", + "https://www.cnblogs.com/cate/all", + "https://www.cnblogs.com/docker-mo/p/22393790", + "https://www.cnblogs.com/cmt/p/archive/2009/12", + "https://www.cnblogs.com/cmt/tag/%E6%AF%8F%E5%91%A8%E7%83%AD%E7%82%B9", + "https://www.cnblogs.com/cmt/p/archive/2020/08", + "https://www.cnblogs.com/cate/ruby", + "https://www.cnblogs.com/zhangs1986/p/22340941", + "https://www.cnblogs.com/legacy/sitehome/p/7", + "https://www.cnblogs.com/cate/OracleERP", + "https://www.cnblogs.com/eventhorizon/p/22405144", + "https://www.cnblogs.com/cmt/p/archive/2019/04", + "https://www.cnblogs.com/cmt/p/archive/2012/02", + "https://www.cnblogs.com/cmt/p/archive/2018/10", + "https://www.cnblogs.com/legacy/cate/mvc", + "https://www.cnblogs.com/cmt/p/archive/2013/12", + "https://www.cnblogs.com/cmt/p/archive/2025/01", + "https://www.cnblogs.com/shanyou/p/22355348", + "https://www.cnblogs.com/legacy/cate/xna", + "https://www.cnblogs.com/cate/2", + "https://www.cnblogs.com/cmt/p/archive/2015/06", + "https://www.cnblogs.com/guangzan/p/20774394", + "https://www.cnblogs.com/cmt/p/archive/2009/03", + "https://www.cnblogs.com/legacy/cate/aspnet", + "https://www.cnblogs.com/cmt/category/220662.html", + "https://www.cnblogs.com/cmt/p/archive/2009/11", + "https://www.cnblogs.com/legacy/cate/wpf", + "https://www.cnblogs.com/legacy/candidate", + "https://www.cnblogs.com/cmt/p/archive/2008/08", + "https://www.cnblogs.com/yuxl01/p/21437691", + "https://www.cnblogs.com/cmt/p/archive/2013/11", + "https://www.cnblogs.com/aggsite/headline", + "https://www.cnblogs.com/cate/4", + "https://www.cnblogs.com/dbaxmg/p/22398722", + "https://www.cnblogs.com/cmt/p/archive/2008/11", + "https://www.cnblogs.com/cate/nosql", + "https://www.cnblogs.com/cmt/p/archive/2017/03", + "https://www.cnblogs.com/cmt/p/17520031.html", + "https://www.cnblogs.com/cmt/tag/%E5%91%A8%E8%BE%B9", + "https://www.cnblogs.com/cate/agile", + "https://www.cnblogs.com/legacy/sitehome/p/9", + "https://www.cnblogs.com/cmt/p/archive/2023/01", + "https://www.cnblogs.com/cate/opensource", + "https://www.cnblogs.com/cmt/p/archive/2023/02", + "https://www.cnblogs.com/cate/108724", + "https://www.cnblogs.com/cmt/p/archive/2008/06", + "https://www.cnblogs.com/cmt/p/archive/2020/01", + "https://www.cnblogs.com/cmt/p/archive/2011/11", + "https://www.cnblogs.com/cmt/p/archive/2014/11", + "https://www.cnblogs.com/cmt/p/archive/2009/04", + "https://www.cnblogs.com/cmt/p/archive/2021/04", + "https://www.cnblogs.com/legacy/sitehome/p/10", + "https://www.cnblogs.com/cmt/p/archive/2009/06", + "https://www.cnblogs.com/jinjiangongzuoshi/p/22385011", + "https://www.cnblogs.com/cmt/p/archive/2019/06", + "https://www.cnblogs.com/cate/react", + "https://www.cnblogs.com/cmt/MyComments.html", + "https://www.cnblogs.com/cmt/p/archive/2015/08", + "https://www.cnblogs.com/cate/108701", + "https://www.cnblogs.com/erikzhouxin", + "https://www.cnblogs.com/cate/eos", + "https://www.cnblogs.com/cmt/p/archive/2024/08", + "https://www.cnblogs.com/cmt/archive/2010/10/08/1845636.html", + "https://www.cnblogs.com/cmt/p/archive/2020/05", + "https://www.cnblogs.com/legacy/cate/python", + "https://www.cnblogs.com/cmt/p/archive/2013/02", + "https://www.cnblogs.com/cmt/articles/17677736.html", + "https://www.cnblogs.com/cate/otherlang", + "https://www.cnblogs.com/aggsite/topdiggs", + "https://www.cnblogs.com/cate/jquery", + "https://www.cnblogs.com/cmt/p/archive/2019/12", + "https://www.cnblogs.com/rossiXYZ", + "https://www.cnblogs.com/cmt/p/11357365.html", + "https://www.cnblogs.com/cate/verilog", + "https://www.cnblogs.com/cmt/p/archive/2012/07", + "https://www.cnblogs.com/cmt/p/archive/2009/08", + "https://www.cnblogs.com/cmt/p/archive/2022/05", + "https://www.cnblogs.com/cate/osx", + "https://www.cnblogs.com/cmt/p/archive/2011/09", + "https://www.cnblogs.com/cmt/p/archive/2017/04", + "https://www.cnblogs.com/cmt/p/archive/2012/08", + "https://www.cnblogs.com/legacy/cate/108698", + "https://www.cnblogs.com/legacy/cate/108724", + "https://www.cnblogs.com/Alkaid2077/p/22405412", + "https://www.cnblogs.com/cmt/p/archive/2016/02", + "https://www.cnblogs.com/legacy/news", + "https://www.cnblogs.com/cate/oo", + "https://www.cnblogs.com/cmt/p/19316348", + "https://www.cnblogs.com/aggsite/mycommented", + "https://www.cnblogs.com/cmt/p/archive/2017/02", + "https://www.cnblogs.com/cate/codelife", + "https://www.cnblogs.com/cate/flutter", + "https://www.cnblogs.com/docker-mo", + "https://www.cnblogs.com/InCerry", + "https://www.cnblogs.com/cmt/p/archive/2022/06", + "https://www.cnblogs.com/xiexj/p/22344233", + "https://www.cnblogs.com/eventhorizon/p/21973620", + "https://www.cnblogs.com/legacy/sitehome/p/6", + "https://www.cnblogs.com/cate/sqlserver", + "https://www.cnblogs.com/legacy/pick", + "https://www.cnblogs.com/legacy/cate/108709", + "https://www.cnblogs.com/netcasewqs/p/22404834", + "https://www.cnblogs.com/cmt/p/archive/2025/05", + "https://www.cnblogs.com/legacy/sitehome/p/4", + "https://www.cnblogs.com/cmt/p/archive/2025/12", + "https://www.cnblogs.com/cmt/p/archive/2021/02", + "https://www.cnblogs.com/cate/108698", + "https://www.cnblogs.com/zhayujie/p/21524003/cli-for-agents", + "https://www.cnblogs.com/legacy/cate/all", + "https://www.cnblogs.com/cmt/p/archive/2020/10", + "https://www.cnblogs.com/test-shirley/p/22391600", + "https://www.cnblogs.com/cmt/p/archive/2011/12", + "https://www.cnblogs.com/cate/sharepoint", + "https://www.cnblogs.com/legacy/cate/dotnetcore", + "https://www.cnblogs.com/cmt/p/archive/2021/11", + "https://www.cnblogs.com/cmt/p/archive/2013/08", + "https://www.cnblogs.com/helesheng", + "https://www.cnblogs.com/cmt/p/archive/2019/05", + "https://www.cnblogs.com/helesheng/p/22361625", + "https://www.cnblogs.com/cmt/p/archive/2014/07", + "https://www.cnblogs.com/cmt/p/archive/2012/12", + "https://www.cnblogs.com/cate/delphi", + "https://www.cnblogs.com/token-ai/p/22372387", + "https://www.cnblogs.com/comment", + "https://www.cnblogs.com/cmt/p/archive/2018/06" ] } \ No newline at end of file diff --git a/data/auto_state/t_99e73a95c81f.json b/data/auto_state/t_99e73a95c81f.json new file mode 100644 index 0000000..7129629 --- /dev/null +++ b/data/auto_state/t_99e73a95c81f.json @@ -0,0 +1,11 @@ +{ + "pending": [], + "visited": [ + "http://127.0.0.1:38081/p2", + "http://127.0.0.1:38081/p5", + "http://127.0.0.1:38081/p3", + "http://127.0.0.1:38081/", + "http://127.0.0.1:38081/p1", + "http://127.0.0.1:38081/p4" + ] +} \ No newline at end of file diff --git a/engine.py b/engine.py index 5ce85f9..7c64bc4 100644 --- a/engine.py +++ b/engine.py @@ -598,9 +598,10 @@ class CrawlJob: def _crawl_auto(self): """自动爬取: 持续递归 爬取->发现链接->爬取... 直到无新链接可爬或达到上限 - max_depth: 0=无限制, N=只爬 N 层 - - max_pages: 0=无限制, N=安全上限 + - max_pages: 0=无限制, N=安全上限 (继续爬取模式按本次新增页数重新计算) - 提取但未爬取的链接持久化到 auto.pending, 已爬集合持久化到 auto.visited - 停止后再次运行从缓存队列继续爬; 起始网址每次运行都重新爬(不去重) + - skip_seed(继续爬取): 跳过起始网址直接消费缓存队列, 页数上限按本次新增重新计算 """ run = self.run auto = self.task.get("auto", {}) @@ -615,8 +616,13 @@ class CrawlJob: state = store.load_auto_state(self.task["id"], self.task) visited = set(state.get("visited", []) or []) pending = state.get("pending", []) or [] + visited_base = len(visited) # 起始网址每次运行都爬(不做去重), 缓存队列继续消费 - queue = [(seed, 0, "")] + # 继续爬取模式(skip_seed): 有缓存队列时跳过起始网址, 直接从待爬队列接着爬 + skip_seed = bool(self.run.get("skip_seed")) + queue = [] + if not (skip_seed and pending): + queue.append((seed, 0, "")) if pending: queue.extend((p["url"], p.get("depth", 0), p.get("source", "")) for p in pending) queued = set(visited) @@ -630,9 +636,15 @@ class CrawlJob: try: while queue and not self._stop.is_set(): self._wait_if_paused() - if max_pages > 0 and len(visited) >= max_pages: - self._log("info", f"达到最大页数上限 {max_pages}, 停止") - break + if max_pages > 0: + if skip_seed: + # 继续爬取模式: 页数上限按本次新增页数重新计算 (累计 visited 不阻塞继续) + if len(visited) - visited_base >= max_pages: + self._log("info", f"本次继续爬取达到页数上限 {max_pages}, 停止") + break + elif len(visited) >= max_pages: + self._log("info", f"达到最大页数上限 {max_pages}, 停止") + break url, depth, src = queue.pop(0) key = normalize_url(url) if key in visited and url != seed: # 起始网址不去重, 其余已爬跳过 diff --git a/static/app.js b/static/app.js index 828e08d..f0ce1da 100644 --- a/static/app.js +++ b/static/app.js @@ -602,7 +602,10 @@ function renderDetail() { ${t.running ? ` ` - : ``} + : ` + + ${t.mode === "auto" && (t.auto_pending_count || 0) > 0 ? ` + ` : ""}`} ${t.mode === "auto" ? ` ` : ""} @@ -734,6 +737,15 @@ function stopLogPoll() { if (state.logTimer) { clearInterval(state.logTimer); state.logTimer = null; } } +async function contTask(tid) { + try { + const r = await api(`/api/tasks/${tid}/continue`, { method: "POST" }); + toast("已开始继续爬取缓存队列"); + loadTasks(); + if (state.detail.task && state.detail.task.id === tid) openDetail(tid); + } catch (e) { toast(e.message, true); } +} + /* ---------------- 打包导出 ---------------- */ async function exportZip(tid) { const btn = window.event && window.event.target;