From 69b6f8ccfb8d382f9e9ced4af5ed2aaec2ea0e15 Mon Sep 17 00:00:00 2001 From: hz4th_coder Date: Mon, 10 Aug 2026 12:01:05 +0800 Subject: [PATCH] =?UTF-8?q?v1.1.0=20=E6=96=B0=E5=A2=9E=E5=A4=A7=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E9=85=8D=E7=BD=AE=E7=AE=A1=E7=90=86=EF=BC=9A=E9=A1=B5?= =?UTF-8?q?=E9=9D=A2=E5=8F=AF=E9=85=8D=E7=BD=AE=E5=88=86=E6=9E=90=E7=BD=91?= =?UTF-8?q?=E9=A1=B5=E7=9A=84LLM=E6=8E=A5=E5=8F=A3(=E5=90=8D=E7=A7=B0/Base?= =?UTF-8?q?URL/Key/=E6=A8=A1=E5=9E=8B/=E6=B8=A9=E5=BA=A6/=E8=B6=85?= =?UTF-8?q?=E6=97=B6/=E9=BB=98=E8=AE=A4)=EF=BC=8C=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E8=A7=86=E8=A7=89=E6=A0=87=E8=AE=B0=E2=80=94=E2=80=94=E8=A7=86?= =?UTF-8?q?=E8=A7=89=E6=A8=A1=E5=9E=8B=E8=B5=B0=E6=88=AA=E5=9B=BE=E8=A7=86?= =?UTF-8?q?=E8=A7=89=E5=88=86=E6=9E=90=E8=B7=AF=E7=BA=BF=EF=BC=8C=E9=9D=9E?= =?UTF-8?q?=E8=A7=86=E8=A7=89=E6=A8=A1=E5=9E=8B=E8=B5=B0DOM=E5=BF=AB?= =?UTF-8?q?=E7=85=A7=E6=96=87=E6=9C=AC=E5=88=86=E6=9E=90=E8=B7=AF=E7=BA=BF?= =?UTF-8?q?=EF=BC=9B=E4=BB=BB=E5=8A=A1=E5=88=9B=E5=BB=BA=E5=8F=AF=E9=80=89?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E5=B9=B6=E8=AE=B0=E5=BD=95=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E6=83=85=E5=86=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent.py | 109 ++++++++++++++++++++++---- app.py | 153 ++++++++++++++++++++++++++++++++++-- config.py | 4 +- db.py | 136 +++++++++++++++++++++++++++++++- llm.py | 56 +++++++++---- static/css/style.css | 20 +++++ static/index.html | 60 +++++++++++++- static/js/app.js | 183 ++++++++++++++++++++++++++++++++++++++++++- 8 files changed, 678 insertions(+), 43 deletions(-) diff --git a/agent.py b/agent.py index b90bbd0..0c1260b 100644 --- a/agent.py +++ b/agent.py @@ -53,6 +53,55 @@ SYSTEM_PROMPT = """你是一个专业的网页自动化测试工程师，正在 6. 快照可能被截断，必要时用 wait 等待页面加载完成再操作。 """ +# 视觉模型专用提示词：通过截图观察页面视觉状态，而不是依赖 DOM 快照文本 +VISION_SYSTEM_PROMPT = """你是一个专业的网页自动化测试工程师，通过**页面截图**观察网页的真实视觉状态，并使用浏览器工具执行端到端测试任务。 +你的目标：根据用户给的测试目标，直接观察截图画面，实际操作网页完成测试，并通过断言判断测试是否通过。 + +## 你的感知方式（重点） +- 每次决策时，你都会收到一张当前页面的**截图**，请仔细观察截图中的：页面布局、可见文案、按钮、输入框、弹窗、错误提示、加载状态等视觉信息。 +- 截图里能看到什么，就以什么为准；看不到的内容不要臆测。 +- 若截图不完整或看不清楚，可以用 wait 等待加载，或用 scroll 滚动后截图再看。 + +## 可用的浏览器能力 +- click text:按钮文字 点击页面上可见的按钮/链接（用截图里看到的文字定位，推荐） +- click @e1 点击元素（@e1 是附带的元素引用列表里的编号） +- fill text:输入框 "值" 在输入框填入文字（会先清空） +- select text:选项 "值" 选择下拉框选项 +- press "Enter" 按键（Enter/Tab/Escape/Control+a 等） +- wait "2000" 等待毫秒；wait text:"xx" 等待文本出现；wait url:"/path" 等待 URL 变化 +- scroll "down 500" 滚动页面 +- screenshot 截图留证（重要步骤请调用） +- assert 断言验证（见下） +- done 测试完成（全部通过） +- fail 测试失败（无法继续或断言不通过） + +## 断言类型 assert +{"type":"text","expect":"欢迎","present":true} 截图中/页面上应出现"欢迎"文本 +{"type":"text","expect":"错误提示","present":false} 页面上不应出现"错误提示" +{"type":"url","expect":"/dashboard","present":true} URL 应包含 /dashboard +{"type":"title","expect":"首页","present":true} 页面标题应包含"首页" +{"type":"element","expect":"登录","present":true} 页面上应存在文本为"登录"的元素 +断言执行后会返回 PASS 或 FAIL，根据结果决定下一步。 + +## 输出格式（严格 JSON，不要输出其他内容） +{ + "reason": "简要中文说明你在截图中看到了什么、当前判断和下一步计划", + "action": "click|fill|select|press|wait|scroll|screenshot|assert|done|fail", + "target": "text:按钮文字 或 @e1", + "value": "填入的值/按键名/等待参数", + "assert": {断言对象，action=assert 时必填}, + "summary": "测试结论（action=done/fail 时必填，说明验证了什么）" +} + +## 工作原则 +1. 先看清截图再动手：观察布局、找目标元素、确认它的文字和位置，每一步只做一件事。 +2. 优先用截图里看到的文字做 text: 语义定位；附带的元素引用列表可帮助确定编号。 +3. 操作后必须用 assert 验证结果，不要盲目继续。 +4. 测试失败(断言 FAIL 且无补救)时用 fail 结束，并说明原因。 +5. 全部验证通过后用 done 结束，summary 里总结测试覆盖的内容。 +6. 页面可能未加载完（白屏/转圈），先用 wait 等待再操作。 +""" + def _fmt_snapshot(snap, max_chars=DEFAULT_MAX_SNAPSHOT_CHARS): """把快照数据转成紧凑文本给 LLM""" @@ -89,13 +138,15 @@ def _fmt_history(steps): class TaskRunner(threading.Thread): - def __init__(self, task_id, url, goal, max_steps, timeout): + def __init__(self, task_id, url, goal, max_steps, timeout, llm_config=None): super().__init__(daemon=True, name=f'task-{task_id}') self.task_id = task_id self.url = url self.goal = goal self.max_steps = max_steps self.timeout = timeout + self.llm_cfg = llm_config or {} # 模型配置快照（base_url/api_key/model/temperature/timeout/vision） + self.vision = bool(self.llm_cfg.get('vision')) # True=截图视觉分析, False=DOM快照文本分析 self.stop_flag = threading.Event() self.task_dir = os.path.join(TASKS_DIR, task_id) os.makedirs(self.task_dir, exist_ok=True) @@ -174,9 +225,10 @@ class TaskRunner(threading.Thread): cur_url = browser.url() cur_title = browser.title() - # 2. LLM 决策 + # 2. LLM 决策（视觉模型带截图，非视觉模型带 DOM 快照文本） self._set_current(phase='AI 决策中', step=step_n, - detail='正在分析页面并决定下一步动作...') + detail=('正在视觉分析页面截图...' if self.vision + else '正在分析页面并决定下一步动作...')) user_msg = ( f'## 测试目标\n{self.goal}\n\n' f'## 当前页面\nURL: {cur_url}\n标题: {cur_title}\n\n' @@ -184,11 +236,13 @@ class TaskRunner(threading.Thread): f'## 已执行步骤\n' + (_fmt_history(steps) if steps else '(尚无)') + f'\n\n请输出下一步动作的 JSON。' ) + user_content = self._build_user_content(user_msg, browser, step_n) try: decision = chat_json( - [{'role': 'system', 'content': SYSTEM_PROMPT}, - {'role': 'user', 'content': user_msg}], - temperature=0.2, max_tokens=500) + [{'role': 'system', + 'content': VISION_SYSTEM_PROMPT if self.vision else SYSTEM_PROMPT}, + {'role': 'user', 'content': user_content}], + cfg=self.llm_cfg, temperature=0.2, max_tokens=500) except LLMError as e: self._log(f'LLM 错误: {e}') self._finish(browser, 'error', f'LLM 决策失败: {e}', steps) @@ -237,6 +291,26 @@ class TaskRunner(threading.Thread): self._log(f'未知异常: {traceback.format_exc()}') self._finish(browser, 'error', f'异常: {e}', steps) + def _build_user_content(self, user_msg, browser, step_n): + """构建用户消息：视觉模型=文本+截图；非视觉模型=纯文本。 + 截图失败时自动降级为纯文本分析。""" + if not self.vision: + return user_msg + import base64 + shot_path = os.path.join(self.task_dir, f'decide{step_n:02d}.png') + try: + browser.screenshot(shot_path, timeout=30) + with open(shot_path, 'rb') as f: + b64 = base64.b64encode(f.read()).decode('ascii') + return [ + {'type': 'text', 'text': user_msg}, + {'type': 'image_url', + 'image_url': {'url': f'data:image/png;base64,{b64}'}}, + ] + except Exception as e: + self._log(f'视觉截图失败，降级为文本分析: {e}') + return user_msg + def _record(self, n, decision, result, detail='', extra=None): rec = { 'n': n, @@ -316,17 +390,20 @@ class TaskRunner(threading.Thread): snap_text = _fmt_snapshot(snap, max_chars=5000) except Exception: snap_text = '(快照失败)' + retry_msg = ( + f'## 测试目标\n{self.goal}\n\n' + f'## 刚才执行失败\n动作: {action} 目标: {target} 值: {value}\n' + f'错误: {last_err}\n\n' + f'## 当前页面元素\n{snap_text}\n\n' + f'请换一种方式完成相同意图，输出下一步动作 JSON。' + f'如果确认无法完成，输出 {{"action":"fail","reason":"...","summary":"..."}}' + ) retry_dec = chat_json( - [{'role': 'system', 'content': SYSTEM_PROMPT}, - {'role': 'user', 'content': ( - f'## 测试目标\n{self.goal}\n\n' - f'## 刚才执行失败\n动作: {action} 目标: {target} 值: {value}\n' - f'错误: {last_err}\n\n' - f'## 当前页面元素\n{snap_text}\n\n' - f'请换一种方式完成相同意图，输出下一步动作 JSON。' - f'如果确认无法完成，输出 {{"action":"fail","reason":"...","summary":"..."}}' - )}], - temperature=0.2, max_tokens=400) + [{'role': 'system', + 'content': VISION_SYSTEM_PROMPT if self.vision else SYSTEM_PROMPT}, + {'role': 'user', + 'content': self._build_user_content(retry_msg, browser, self.current.get('step', 0) + 99)}], + cfg=self.llm_cfg, temperature=0.2, max_tokens=400) new_action = retry_dec.get('action', '') if new_action == 'fail': return 'error', f'自愈放弃: {retry_dec.get("summary", last_err)}', {} diff --git a/app.py b/app.py index 606e568..aef8822 100644 --- a/app.py +++ b/app.py @@ -7,7 +7,10 @@ from flask import Flask, request, jsonify, send_from_directory, abort from flask_cors import CORS import config -from db import init_db, create_task, get_task, list_tasks, load_step_logs, update_task +from db import (init_db, create_task, get_task, list_tasks, load_step_logs, + update_task, list_llm_configs, get_llm_config, + get_default_llm_config, create_llm_config, update_llm_config, + delete_llm_config, set_default_llm_config, llm_config_to_dict) from agent import TaskRunner app = Flask(__name__, static_folder='static', static_url_path='/static') @@ -27,11 +30,134 @@ def index(): @app.route('/health') def health(): + default_cfg = get_default_llm_config() return jsonify({'status': 'ok', 'version': '1.0.0', - 'llm_model': config.LLM_MODEL, + 'llm_model': default_cfg.get('model') if default_cfg else None, + 'llm_name': default_cfg.get('name') if default_cfg else None, + 'llm_vision': bool(default_cfg.get('vision')) if default_cfg else None, 'concurrent': len(_runners)}) +# ========== 大模型配置管理 ========== + + +@app.route('/api/llm-configs', methods=['GET']) +def api_list_llm_configs(): + return jsonify({'configs': list_llm_configs()}) + + +def _parse_llm_fields(data, partial=False): + """校验并提取模型配置字段，返回 (fields, error)""" + fields = {} + for key, label in (('name', '名称'), ('base_url', 'Base URL'), + ('api_key', 'API Key'), ('model', '模型名')): + val = (data.get(key) or '').strip() + if partial and key not in data: + continue + if not val: + return None, f'缺少{label}' + fields[key] = val + if 'vision' in data: + fields['vision'] = 1 if data.get('vision') else 0 + if 'temperature' in data: + try: + fields['temperature'] = float(data.get('temperature')) + except (TypeError, ValueError): + return None, '温度必须是数字' + if 'timeout' in data: + try: + fields['timeout'] = max(10, min(int(data.get('timeout')), 600)) + except (TypeError, ValueError): + return None, '超时必须是数字' + if 'is_default' in data: + fields['is_default'] = 1 if data.get('is_default') else 0 + return fields, None + + +@app.route('/api/llm-configs', methods=['POST']) +def api_create_llm_config(): + data = request.get_json(silent=True) or {} + fields, err = _parse_llm_fields(data) + if err: + return jsonify({'error': err}), 400 + cid = create_llm_config( + name=fields['name'], base_url=fields['base_url'], + api_key=fields['api_key'], model=fields['model'], + vision=fields.get('vision', 0), + temperature=fields.get('temperature', 0.2), + timeout=fields.get('timeout', 120), + is_default=fields.get('is_default', False)) + return jsonify({'ok': True, 'id': cid}), 201 + + +@app.route('/api/llm-configs/', methods=['PUT']) +def api_update_llm_config(cid): + if not get_llm_config(cid): + abort(404) + data = request.get_json(silent=True) or {} + fields, err = _parse_llm_fields(data, partial=True) + if err: + return jsonify({'error': err}), 400 + update_llm_config(cid, **fields) + return jsonify({'ok': True}) + + +@app.route('/api/llm-configs/', methods=['DELETE']) +def api_delete_llm_config(cid): + row = get_llm_config(cid) + if not row: + abort(404) + if row.get('is_default'): + return jsonify({'error': '默认配置不能删除，请先设置其他配置为默认'}), 400 + delete_llm_config(cid) + return jsonify({'ok': True}) + + +@app.route('/api/llm-configs//default', methods=['POST']) +def api_set_default_llm_config(cid): + if not get_llm_config(cid): + abort(404) + set_default_llm_config(cid) + return jsonify({'ok': True}) + + +@app.route('/api/llm-configs//test', methods=['POST']) +def api_test_llm_config(cid): + row = get_llm_config(cid) + if not row: + abort(404) + return _test_llm(llm_config_to_dict(row)) + + +@app.route('/api/llm-configs/test-form', methods=['POST']) +def api_test_llm_config_form(): + """测试未保存的表单配置（前端弹窗里点「测试连接」用）""" + data = request.get_json(silent=True) or {} + base_url = (data.get('base_url') or '').strip() + api_key = (data.get('api_key') or '').strip() + model = (data.get('model') or '').strip() + if not base_url or not api_key or not model: + return jsonify({'error': '缺少 base_url / api_key / model'}), 400 + return _test_llm({ + 'base_url': base_url, + 'api_key': api_key, + 'model': model, + 'temperature': data.get('temperature', 0.2), + 'timeout': data.get('timeout', 30), + }) + + +def _test_llm(cfg): + from llm import chat, LLMError + try: + reply = chat( + [{'role': 'user', 'content': '请只回复两个字：正常'}], + cfg=cfg, max_tokens=20, timeout=30) + return jsonify({'ok': True, 'reply': (reply or '')[:200]}) + except LLMError as e: + return jsonify({'ok': False, 'error': str(e)}), 400 + + @app.route('/api/tasks', methods=['POST']) def api_create_task(): data = request.get_json(silent=True) or {} @@ -48,13 +174,28 @@ def api_create_task(): max_steps = max(1, min(max_steps, 100)) timeout = max(30, min(timeout, 3600)) - tid = create_task(url, goal, max_steps, timeout) + # 选择分析模型：指定 id → 默认配置 + cfg_row = None + if data.get('llm_config_id'): + cfg_row = get_llm_config(int(data.get('llm_config_id'))) + if not cfg_row: + return jsonify({'error': '指定的模型配置不存在'}), 400 + else: + cfg_row = get_default_llm_config() + if not cfg_row: + return jsonify({'error': '尚未配置任何大模型，请先在「大模型配置」中添加'}), 400 + llm_cfg = llm_config_to_dict(cfg_row) + + tid = create_task(url, goal, max_steps, timeout, + llm_config_id=cfg_row['id'], + llm_name=cfg_row['name'], + vision=cfg_row['vision']) def _launch(): with _semaphore: if get_task(tid) and get_task(tid).get('status') == 'stopped': return - runner = TaskRunner(tid, url, goal, max_steps, timeout) + runner = TaskRunner(tid, url, goal, max_steps, timeout, llm_cfg) with _runner_lock: _runners[tid] = runner runner.start() @@ -64,7 +205,9 @@ def api_create_task(): threading.Thread(target=_launch, daemon=True).start() return jsonify({'task_id': tid, 'status': 'queued', - 'url': url, 'goal': goal}), 202 + 'url': url, 'goal': goal, + 'llm_name': cfg_row['name'], + 'vision': bool(cfg_row['vision'])}), 202 @app.route('/api/tasks', methods=['GET']) diff --git a/config.py b/config.py index 7a35b80..a538ff9 100644 --- a/config.py +++ b/config.py @@ -17,12 +17,14 @@ NODE_BIN_DIR = os.path.dirname(AGENT_BROWSER) # agent-browser 需要可写的 socket 目录，固定用 /tmp 下的（系统 XDG_RUNTIME_DIR 可能属于其他用户） XDG_RUNTIME_DIR = '/tmp/xdg-rt' -# LLM 配置（OpenAI 兼容接口） +# LLM 配置（OpenAI 兼容接口）——仅作为首次初始化默认模型配置的种子值， +# 运行后以页面「大模型配置」中保存的配置为准（存于 SQLite llm_configs 表） LLM_BASE_URL = os.environ.get('LLM_BASE_URL', 'https://ark.cn-beijing.volces.com/api/plan/v3') LLM_API_KEY = os.environ.get('LLM_API_KEY', 'ark-2b06dc9d-8878-4c6e-b201-f422376e79cb-246d9') LLM_MODEL = os.environ.get('LLM_MODEL', 'doubao-seed-evolving') LLM_TEMPERATURE = 0.2 LLM_TIMEOUT = 120 +DEFAULT_LLM_NAME = '火山引擎豆包（默认）' # 首次启动时写入 llm_configs 的显示名 # Agent 默认参数 DEFAULT_MAX_STEPS = 30 # 最大动作步数 diff --git a/db.py b/db.py index 3829ab5..0181e97 100644 --- a/db.py +++ b/db.py @@ -34,17 +34,51 @@ def init_db(): report_path TEXT ) ''') + # 大模型配置表 + c.execute(''' + CREATE TABLE IF NOT EXISTS llm_configs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + base_url TEXT NOT NULL, + api_key TEXT NOT NULL, + model TEXT NOT NULL, + vision INTEGER DEFAULT 0, + temperature REAL DEFAULT 0.2, + timeout INTEGER DEFAULT 120, + is_default INTEGER DEFAULT 0, + created_at REAL + ) + ''') + # tasks 表迁移：新增模型相关列（老库升级） + cols = [r[1] for r in c.execute('PRAGMA table_info(tasks)')] + if 'llm_config_id' not in cols: + c.execute('ALTER TABLE tasks ADD COLUMN llm_config_id INTEGER') + if 'llm_name' not in cols: + c.execute('ALTER TABLE tasks ADD COLUMN llm_name TEXT') + if 'vision' not in cols: + c.execute('ALTER TABLE tasks ADD COLUMN vision INTEGER DEFAULT 0') + # 首次启动：写入默认模型配置 + n = c.execute('SELECT COUNT(*) FROM llm_configs').fetchone()[0] + if n == 0: + from config import (LLM_BASE_URL, LLM_API_KEY, LLM_MODEL, + LLM_TEMPERATURE, LLM_TIMEOUT, DEFAULT_LLM_NAME) + c.execute( + 'INSERT INTO llm_configs (name, base_url, api_key, model, vision, ' + 'temperature, timeout, is_default, created_at) VALUES (?,?,?,?,?,?,?,?,?)', + (DEFAULT_LLM_NAME, LLM_BASE_URL, LLM_API_KEY, LLM_MODEL, 0, + LLM_TEMPERATURE, LLM_TIMEOUT, 1, time.time())) c.commit() c.close() -def create_task(url, goal, max_steps, timeout): +def create_task(url, goal, max_steps, timeout, llm_config_id=None, llm_name=None, vision=0): tid = uuid.uuid4().hex[:12] c = _conn() c.execute( - 'INSERT INTO tasks (id, url, goal, status, max_steps, timeout, created_at) ' - 'VALUES (?,?,?,?,?,?,?)', - (tid, url, goal, 'queued', max_steps, timeout, time.time())) + 'INSERT INTO tasks (id, url, goal, status, max_steps, timeout, created_at, ' + 'llm_config_id, llm_name, vision) VALUES (?,?,?,?,?,?,?,?,?,?)', + (tid, url, goal, 'queued', max_steps, timeout, time.time(), + llm_config_id, llm_name, 1 if vision else 0)) c.commit() c.close() return tid @@ -104,3 +138,97 @@ def load_step_logs(tid): except json.JSONDecodeError: pass return out + + +# ========== 大模型配置 ========== + + +def list_llm_configs(): + c = _conn() + rows = c.execute( + 'SELECT * FROM llm_configs ORDER BY is_default DESC, id ASC' + ).fetchall() + c.close() + return [dict(r) for r in rows] + + +def get_llm_config(cid): + c = _conn() + row = c.execute('SELECT * FROM llm_configs WHERE id=?', (cid,)).fetchone() + c.close() + return dict(row) if row else None + + +def get_default_llm_config(): + c = _conn() + row = c.execute('SELECT * FROM llm_configs WHERE is_default=1').fetchone() + if not row: + row = c.execute('SELECT * FROM llm_configs ORDER BY id ASC').fetchone() + c.close() + return dict(row) if row else None + + +def create_llm_config(name, base_url, api_key, model, vision=0, + temperature=0.2, timeout=120, is_default=False): + c = _conn() + if is_default: + c.execute('UPDATE llm_configs SET is_default=0') + c.execute( + 'INSERT INTO llm_configs (name, base_url, api_key, model, vision, ' + 'temperature, timeout, is_default, created_at) VALUES (?,?,?,?,?,?,?,?,?)', + (name, base_url, api_key, model, 1 if vision else 0, + temperature, timeout, 1 if is_default else 0, time.time())) + c.commit() + cid = c.execute('SELECT last_insert_rowid()').fetchone()[0] + c.close() + return cid + + +def update_llm_config(cid, **fields): + allowed = {'name', 'base_url', 'api_key', 'model', 'vision', + 'temperature', 'timeout', 'is_default'} + c = _conn() + if fields.get('is_default'): + c.execute('UPDATE llm_configs SET is_default=0') + sets = [] + vals = [] + for k, v in fields.items(): + if k not in allowed: + continue + if k == 'vision': + v = 1 if v else 0 + if k == 'is_default': + v = 1 if v else 0 + sets.append(f'{k}=?') + vals.append(v) + if sets: + c.execute(f'UPDATE llm_configs SET {", ".join(sets)} WHERE id=?', (*vals, cid)) + c.commit() + c.close() + + +def delete_llm_config(cid): + c = _conn() + c.execute('DELETE FROM llm_configs WHERE id=?', (cid,)) + c.commit() + c.close() + + +def set_default_llm_config(cid): + c = _conn() + c.execute('UPDATE llm_configs SET is_default=0') + c.execute('UPDATE llm_configs SET is_default=1 WHERE id=?', (cid,)) + c.commit() + c.close() + + +def llm_config_to_dict(row): + """把配置行转成 llm.py 可用的 dict（不含数据库元信息）""" + return { + 'base_url': row['base_url'], + 'api_key': row['api_key'], + 'model': row['model'], + 'temperature': row['temperature'], + 'timeout': row['timeout'], + 'vision': bool(row['vision']), + } diff --git a/llm.py b/llm.py index 4fb064d..d1bf82d 100644 --- a/llm.py +++ b/llm.py @@ -1,5 +1,9 @@ #!/usr/bin/env python3 -"""LLM 客户端（OpenAI 兼容接口）""" +"""LLM 客户端（OpenAI 兼容接口） + +支持按模型配置调用（base_url / api_key / model / temperature / timeout）， +支持多模态消息（content 为 [{type:text},{type:image_url}] 列表，用于视觉模型分析截图）。 +""" import json import re import urllib.request @@ -12,6 +16,21 @@ class LLMError(Exception): pass +def _cfg(cfg): + """补齐默认值，返回完整配置 dict""" + base = { + 'base_url': LLM_BASE_URL, + 'api_key': LLM_API_KEY, + 'model': LLM_MODEL, + 'temperature': LLM_TEMPERATURE, + 'timeout': LLM_TIMEOUT, + 'vision': False, + } + if cfg: + base.update({k: v for k, v in cfg.items() if v is not None}) + return base + + def _extract_json(text): """从 LLM 输出中提取 JSON（容忍 markdown 代码块等包裹）""" if not text: @@ -35,13 +54,19 @@ def _extract_json(text): return None -def chat(messages, temperature=None, max_tokens=None, timeout=None): - """调用 OpenAI 兼容 chat/completions，返回 content 字符串""" - url = f'{LLM_BASE_URL}/chat/completions' +def chat(messages, cfg=None, temperature=None, max_tokens=None, timeout=None): + """调用 OpenAI 兼容 chat/completions，返回 content 字符串 + + messages 中的 content 可以是字符串，也可以是多模态列表： + [{"type":"text","text":"..."}, + {"type":"image_url","image_url":{"url":"data:image/png;base64,..."}}] + """ + c = _cfg(cfg) + url = f"{c['base_url'].rstrip('/')}/chat/completions" body = { - 'model': LLM_MODEL, + 'model': c['model'], 'messages': messages, - 'temperature': temperature if temperature is not None else LLM_TEMPERATURE, + 'temperature': temperature if temperature is not None else c['temperature'], } if max_tokens: body['max_tokens'] = max_tokens @@ -50,12 +75,12 @@ def chat(messages, temperature=None, max_tokens=None, timeout=None): data=json.dumps(body).encode('utf-8'), headers={ 'Content-Type': 'application/json', - 'Authorization': f'Bearer {LLM_API_KEY}', + 'Authorization': f'Bearer {c["api_key"]}', }, method='POST', ) try: - with urllib.request.urlopen(req, timeout=timeout or LLM_TIMEOUT) as resp: + with urllib.request.urlopen(req, timeout=timeout or c['timeout']) as resp: data = json.loads(resp.read().decode('utf-8')) except urllib.error.HTTPError as e: detail = e.read().decode('utf-8', 'ignore')[:300] @@ -69,21 +94,24 @@ def chat(messages, temperature=None, max_tokens=None, timeout=None): raise LLMError(f'LLM 响应异常: {str(data)[:300]}') -def chat_json(messages, temperature=None, max_tokens=None, retries=2): +def chat_json(messages, cfg=None, temperature=None, max_tokens=None, retries=2): """调用 LLM 并强制解析 JSON，失败重试""" last_err = None + last_content = '' for i in range(retries + 1): try: - content = chat(messages, temperature=temperature, max_tokens=max_tokens) - obj = _extract_json(content) + last_content = chat(messages, cfg=cfg, temperature=temperature, + max_tokens=max_tokens) + obj = _extract_json(last_content) if obj is not None: return obj - last_err = f'无法从输出解析 JSON: {content[:200]}' + last_err = f'无法从输出解析 JSON: {last_content[:200]}' except LLMError as e: last_err = str(e) if i < retries: messages = messages + [ - {'role': 'assistant', 'content': content if 'content' in dir() else ''}, - {'role': 'user', 'content': f'刚才的输出不是合法 JSON，请只输出严格的 JSON 对象。错误: {last_err}'}, + {'role': 'assistant', 'content': last_content or ''}, + {'role': 'user', + 'content': f'刚才的输出不是合法 JSON，请只输出严格的 JSON 对象。错误: {last_err}'}, ] raise LLMError(f'LLM JSON 解析失败: {last_err}') diff --git a/static/css/style.css b/static/css/style.css index f202451..7a43c3b 100644 --- a/static/css/style.css +++ b/static/css/style.css @@ -78,3 +78,23 @@ a.report-link:hover { text-decoration: underline; } .step-shot img { max-width: 320px; border-radius: 8px; border: 1px solid #e5e7eb; display: block; margin-top: 6px; cursor: zoom-in; } .step-detail { font-size: 12px; color: #374151; } .step-time { font-size: 11px; color: #9ca3af; margin-left: 8px; } + +/* 大模型配置 */ +select { padding: 10px 12px; border: 1px solid #d1d5db; border-radius: 8px; font-size: 14px; background: #fff; } +select:focus { outline: none; border-color: #2563eb; box-shadow: 0 0 0 3px rgba(37,99,235,.1); } +.mode-badge { display: inline-block; padding: 2px 8px; border-radius: 999px; font-size: 11px; font-weight: 600; white-space: nowrap; } +.mode-badge.vision { background: #ede9fe; color: #6d28d9; } +.mode-badge.text { background: #f3f4f6; color: #4b5563; } +.tag { font-size: 11px; font-weight: 400; margin-left: 6px; padding: 1px 8px; border-radius: 999px; } +.tag-vision { background: #ede9fe; color: #6d28d9; } +.tag-text { background: #f3f4f6; color: #4b5563; } +.url-cell { max-width: 240px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.model-cell { font-weight: 600; } +.ops { white-space: nowrap; } +.llm-modal-body { width: 560px; height: auto; max-height: 92vh; } +.llm-form { padding: 18px 20px; display: grid; grid-template-columns: 1fr 1fr; gap: 14px; overflow-y: auto; } +.llm-opts { display: flex; flex-direction: row; gap: 20px; align-items: center; } +.check { display: flex; align-items: center; gap: 6px; font-size: 13px; color: #374151; cursor: pointer; } +.check input { width: auto; } +.llm-actions { grid-column: span 2; display: flex; align-items: center; gap: 12px; } +.llm-test-result { font-size: 12px; color: #6b7280; word-break: break-all; } diff --git a/static/index.html b/static/index.html index e2742c2..dc87347 100644 --- a/static/index.html +++ b/static/index.html @@ -28,13 +28,27 @@ +
+ + +
-
测试由 AI 自动驱动浏览器执行：打开页面 → 逐步操作 → 断言验证 → 生成报告（含截图）
+
测试由 AI 自动驱动浏览器执行：打开页面 → 逐步操作 → 断言验证 → 生成报告（含截图）。 + 🖼️ 视觉模型：通过截图直接观察页面视觉状态分析；📄 文本模型：通过 DOM 元素快照分析。
+ + +
+

🔌 大模型配置

+ + + +
名称模型Base URL分析方式温度超时默认操作
加载中...
+
配置 OpenAI 兼容接口的大模型。支持视觉的模型（如 GPT-4o、豆包视觉版）用截图分析网页，不支持视觉的模型用 DOM 快照分析。
@@ -80,6 +94,50 @@ + + diff --git a/static/js/app.js b/static/js/app.js index 5fda57a..38aa9ea 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -6,7 +6,8 @@ async function refreshHealth() { try { const r = await fetch(API + '/health'); const d = await r.json(); - $('#health').textContent = `服务正常 | 模型: ${d.llm_model} | 运行中任务: ${d.concurrent}`; + const mode = d.llm_vision ? '🖼️视觉' : '📄文本'; + $('#health').textContent = `服务正常 | 默认模型: ${d.llm_name || d.llm_model || '-'}（${mode}） | 运行中任务: ${d.concurrent}`; $('#health').classList.add('ok'); } catch (e) { $('#health').textContent = '服务异常'; @@ -21,6 +22,180 @@ const RESULT_MAP = { pending: '待定', pass: '✅ 通过', fail: '❌ 失败', error: '⚠️ 错误', stopped: '⏹️ 停止' }; +/* ========== 大模型配置 ========== */ +let llmConfigs = []; +let editingLlmId = null; + +function llmBadge(cfg) { + return cfg.vision ? '🖼️ 视觉' : '📄 文本'; +} + +async function loadLlmConfigs() { + try { + const r = await fetch(API + '/api/llm-configs'); + const d = await r.json(); + llmConfigs = d.configs || []; + renderLlmTable(); + renderLlmSelect(); + } catch (e) { /* ignore */ } +} + +function renderLlmTable() { + const tb = $('#llm-table tbody'); + if (!llmConfigs.length) { + tb.innerHTML = '暂无配置，点右上角「新增配置」添加'; + return; + } + tb.innerHTML = llmConfigs.map(c => ` + + ${esc(c.name)} + ${esc(c.model)} + ${esc(c.base_url)} + ${llmBadge(c)} + ${c.temperature} + ${c.timeout}s + ${c.is_default ? '⭐' : ''} + + + + ${c.is_default ? '' : ``} + ${c.is_default ? '' : ``} + + `).join(''); +} + +function renderLlmSelect() { + const sel = $('#llm_config_id'); + if (!sel) return; + if (!llmConfigs.length) { + sel.innerHTML = ''; + $('#model-tag').textContent = ''; + return; + } + sel.innerHTML = llmConfigs.map(c => + `` + ).join(''); + updateModelTag(); +} + +function updateModelTag() { + const id = parseInt($('#llm_config_id').value); + const cfg = llmConfigs.find(c => c.id === id); + $('#model-tag').textContent = cfg ? llmBadge(cfg) : ''; + $('#model-tag').className = 'tag ' + (cfg && cfg.vision ? 'tag-vision' : 'tag-text'); +} + +function openLlmModal(id) { + editingLlmId = id || null; + const c = id ? llmConfigs.find(x => x.id === id) : null; + $('#llm-modal-title').textContent = c ? `编辑配置 - ${c.name}` : '新增模型配置'; + $('#llm-name').value = c ? c.name : ''; + $('#llm-base-url').value = c ? c.base_url : ''; + $('#llm-api-key').value = c ? c.api_key : ''; + $('#llm-model').value = c ? c.model : ''; + $('#llm-vision').checked = c ? !!c.vision : false; + $('#llm-default').checked = c ? !!c.is_default : false; + $('#llm-temperature').value = c ? c.temperature : 0.2; + $('#llm-timeout').value = c ? c.timeout : 120; + $('#llm-test-result').textContent = ''; + $('#llm-modal').classList.remove('hidden'); +} + +function closeLlmModal() { + $('#llm-modal').classList.add('hidden'); + editingLlmId = null; +} + +async function saveLlm() { + const body = { + name: $('#llm-name').value.trim(), + base_url: $('#llm-base-url').value.trim(), + api_key: $('#llm-api-key').value.trim(), + model: $('#llm-model').value.trim(), + vision: $('#llm-vision').checked, + is_default: $('#llm-default').checked, + temperature: parseFloat($('#llm-temperature').value) || 0.2, + timeout: parseInt($('#llm-timeout').value) || 120 + }; + if (!body.name || !body.base_url || !body.api_key || !body.model) { + alert('请填写名称、Base URL、API Key、模型名'); + return; + } + const url = editingLlmId ? `${API}/api/llm-configs/${editingLlmId}` : `${API}/api/llm-configs`; + const method = editingLlmId ? 'PUT' : 'POST'; + try { + const r = await fetch(url, { + method, headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) + }); + const d = await r.json(); + if (d.error) { alert('保存失败: ' + d.error); return; } + closeLlmModal(); + loadLlmConfigs(); + } catch (e) { alert('保存失败: ' + e); } +} + +async function deleteLlm(id) { + const c = llmConfigs.find(x => x.id === id); + if (!confirm(`确定删除配置「${c.name}」？`)) return; + const r = await fetch(API + `/api/llm-configs/${id}`, { method: 'DELETE' }); + const d = await r.json(); + if (d.error) { alert('删除失败: ' + d.error); return; } + loadLlmConfigs(); +} + +async function setDefaultLlm(id) { + await fetch(API + `/api/llm-configs/${id}/default`, { method: 'POST' }); + loadLlmConfigs(); +} + +async function testLlm(id) { + const c = llmConfigs.find(x => x.id === id); + const el = $('#llm-test-result'); + if (el) el.textContent = ''; + try { + const r = await fetch(API + `/api/llm-configs/${id}/test`, { method: 'POST' }); + const d = await r.json(); + if (d.ok) { + alert(`✅ 连接成功！模型回复: ${d.reply}`); + } else { + alert(`❌ 连接失败: ${d.error}`); + } + } catch (e) { alert('测试请求失败: ' + e); } +} + +$('#add-llm').onclick = () => openLlmModal(null); +$('#llm-modal-close').onclick = closeLlmModal; +$('#llm-cancel').onclick = closeLlmModal; +$('#llm-save').onclick = saveLlm; +$('#llm-modal').onclick = e => { if (e.target === $('#llm-modal')) closeLlmModal(); }; +$('#llm-test').onclick = async () => { + // 用表单当前内容测试（未保存也能测） + const body = { + name: $('#llm-name').value.trim() || '测试', + base_url: $('#llm-base-url').value.trim(), + api_key: $('#llm-api-key').value.trim(), + model: $('#llm-model').value.trim(), + vision: $('#llm-vision').checked, + temperature: parseFloat($('#llm-temperature').value) || 0.2, + timeout: parseInt($('#llm-timeout').value) || 120 + }; + if (!body.base_url || !body.api_key || !body.model) { + alert('请先填写 Base URL、API Key、模型名再测试'); + return; + } + const el = $('#llm-test-result'); + el.textContent = '测试中...'; + try { + const r = await fetch(API + '/api/llm-configs/test-form', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) + }); + const d = await r.json(); + el.textContent = d.ok ? `✅ ${d.reply}` : `❌ ${d.error}`; + } catch (e) { el.textContent = '❌ ' + e; } +}; +$('#llm_config_id').onchange = updateModelTag; + + function esc(s) { return String(s ?? '').replace(/[&<>"']/g, c => ( {'&':'&','<':'<','>':'>','"':'"',"'":'''}[c])); @@ -46,6 +221,7 @@ async function loadTasks() { ${esc(t.id)} ${esc(t.url)}
${esc(t.goal)}
+ ${t.llm_name ? `${esc(t.llm_name)} ${t.vision ? '🖼️' : '📄'}` : '-'} ${STATUS_MAP[t.status] || esc(t.status)} ${RESULT_MAP[t.result] || esc(t.result)} ${t.steps || 0} @@ -105,6 +281,7 @@ async function refreshDetail() { $('#detail-meta').innerHTML = `
目标网址${esc(t.url)}
测试目标${esc(t.goal)}
+
分析模型${t.llm_name ? esc(t.llm_name) + ' ' + (t.vision ? '🖼️' : '📄') : '-'}
状态${STATUS_MAP[t.status] || esc(t.status)} / ${RESULT_MAP[t.result] || esc(t.result)}
步骤${t.steps || 0} / ${t.max_steps}
创建时间${esc(t.created)}
@@ -185,7 +362,8 @@ $('#submit').onclick = async () => { body: JSON.stringify({ url, goal, max_steps: parseInt($('#max_steps').value) || 30, - timeout: parseInt($('#timeout').value) || 600 + timeout: parseInt($('#timeout').value) || 600, + llm_config_id: parseInt($('#llm_config_id').value) || null }) }); const d = await r.json(); @@ -217,5 +395,6 @@ $('#refresh').onclick = loadTasks; refreshHealth(); loadTasks(); +loadLlmConfigs(); setInterval(refreshHealth, 30000); setInterval(loadTasks, 5000);