diff --git a/app.py b/app.py index c7e74d3..91ba3d1 100644 --- a/app.py +++ b/app.py @@ -263,6 +263,35 @@ def worker_test(wid): return jsonify({'ok': False, 'error': str(e)}) +@app.route('/api/workers//vision_test', methods=['POST']) +@require_auth +def worker_vision_test(wid): + """视觉智能体能力测试：传入图片（URL 或 base64）与问题，验证多模态理解""" + w = db.q('SELECT * FROM workers WHERE id=?', (wid,), one=True) + if not w: + return jsonify({'ok': False, 'error': '不存在'}), 404 + d = request.get_json(force=True) + image_url = (d.get('image_url') or '').strip() + image_b64 = (d.get('image_base64') or '').strip() + question = (d.get('question') or '请描述这张图片的内容').strip() + if not image_url and not image_b64: + return jsonify({'ok': False, 'error': '请提供图片 URL 或 base64 数据'}), 400 + try: + if image_b64: + r = llm_gateway.chat_vision( + w['provider'], w['model'], question, image_url=f'data:image/png;base64,{image_b64}', + temperature=0.3, max_tokens=w['max_tokens'] or 2000, + base_url=w['base_url'] or None, api_key=w['api_key'] or None) + else: + r = llm_gateway.chat_vision( + w['provider'], w['model'], question, image_url=image_url, + temperature=0.3, max_tokens=w['max_tokens'] or 2000, + base_url=w['base_url'] or None, api_key=w['api_key'] or None) + return jsonify({'ok': True, 'data': r}) + except Exception as e: + return jsonify({'ok': False, 'error': str(e)}) + + @app.route('/api/providers') @require_auth def providers(): @@ -273,7 +302,7 @@ def providers(): 'has_key': bool(v['api_key']), 'models': [m for m in config.MODEL_PRICING.keys() if m.startswith( {'doubao': 'doubao', 'deepseek': 'deepseek', 'openai': 'gpt', - 'qwen': 'qwen', 'vllm': ''}.get(k, '__none__'))], + 'qwen': 'qwen', 'vllm': '', 'autodl': 'qwen3'}.get(k, '__none__'))], }) return jsonify({'ok': True, 'data': data}) @@ -1119,8 +1148,8 @@ def template_apply(tid): pid, ids = tplmod.apply_project_template(t, variables, worker_id) return jsonify({'ok': True, 'kind': 'project', 'id': pid, 'task_ids': ids}) if t['type'] == 'team': - ids = tplmod.apply_team_template(t, variables, provider=d.get('provider', 'doubao'), - model=d.get('model', 'doubao-seed-evolving')) + ids = tplmod.apply_team_template(t, variables, provider=d.get('provider', 'deepseek'), + model=d.get('model', 'deepseek-v4-flash')) return jsonify({'ok': True, 'kind': 'team', 'ids': ids}) return jsonify({'ok': False, 'error': '未知模板类型'}), 400 except Exception as e: diff --git a/config.py b/config.py index c5f2a54..0baed97 100644 --- a/config.py +++ b/config.py @@ -33,8 +33,14 @@ PROVIDERS = { }, 'deepseek': { 'name': 'DeepSeek', - 'base_url': 'https://api.deepseek.com/v1', - 'api_key': os.environ.get('DEEPSEEK_API_KEY', ''), + 'base_url': 'https://api.deepseek.com', + 'api_key': os.environ.get('DEEPSEEK_API_KEY', 'sk-edb9df58ff574f8c98df1cd6a425e97c'), + 'timeout': 600, + }, + 'autodl': { + 'name': 'AutoDL 多模态（视觉）', + 'base_url': 'https://www.autodl.art/api/v1', + 'api_key': os.environ.get('AUTODL_API_KEY', 'F9MBfolzuapqTsD4KmUf9qen720rXvUZ3Sp3IrWiCTukqonx'), 'timeout': 600, }, 'openai': { @@ -66,10 +72,12 @@ MODEL_PRICING = { 'doubao-pro-32k': {'input': 2.0, 'output': 8.0}, 'deepseek-chat': {'input': 2.0, 'output': 8.0}, 'deepseek-reasoner': {'input': 4.0, 'output': 16.0}, + 'deepseek-v4-flash': {'input': 0.2, 'output': 0.6}, 'gpt-4o': {'input': 17.5, 'output': 70.0}, 'gpt-4o-mini': {'input': 1.1, 'output': 4.4}, 'qwen-plus': {'input': 0.8, 'output': 2.0}, 'qwen-max': {'input': 4.0, 'output': 12.0}, + 'qwen3.6-plus': {'input': 2.0, 'output': 8.0}, } DEFAULT_PRICE = {'input': 2.0, 'output': 8.0} diff --git a/engine.py b/engine.py index ce603ae..7268a09 100644 --- a/engine.py +++ b/engine.py @@ -123,7 +123,9 @@ def _budget_alert(project_id): def _build_messages(task, worker): - """构造提示词：任务指令 + RAG 知识库上下文""" + """构造提示词：任务指令 + RAG 知识库上下文 + 支持图片注入：任务描述中的 ![说明](图片URL) 或 图片:URL 会转成多模态消息（视觉 Worker）。""" + import re as _re messages = [] if worker['system_prompt']: messages.append({'role': 'system', 'content': worker['system_prompt']}) @@ -132,7 +134,20 @@ def _build_messages(task, worker): if ctx: user_text = f'{ctx}\n\n----\n\n任务指令：{user_text}' _log(task['id'], 'info', f'📚 RAG 知识库命中 {len(refs)} 个片段：' + '；'.join(refs[:5])) - messages.append({'role': 'user', 'content': user_text}) + # 提取图片 URL（![alt](url) 或 图片:url 或 image:url） + img_urls = [] + for m in _re.finditer(r'!\[[^\]]*\]\(([^)\s]+)\)', user_text): + img_urls.append(m.group(1)) + for m in _re.finditer(r'(?:图片|image)\s*[:：]\s*(https?://\S+)', user_text, _re.I): + img_urls.append(m.group(1)) + if img_urls: + content = [{'type': 'text', 'text': user_text}] + for u in img_urls: + content.append({'type': 'image_url', 'image_url': {'url': u}}) + messages.append({'role': 'user', 'content': content}) + _log(task['id'], 'info', f'🖼️ 检测到 {len(img_urls)} 张图片，已注入多模态消息') + else: + messages.append({'role': 'user', 'content': user_text}) return messages diff --git a/llm_gateway.py b/llm_gateway.py index 183c470..680112b 100644 --- a/llm_gateway.py +++ b/llm_gateway.py @@ -30,7 +30,12 @@ def calc_cost(model, prompt_tokens, completion_tokens): def chat(provider, model, messages, temperature=0.7, max_tokens=None, base_url=None, api_key=None, timeout=None, retries=None): - """调用 OpenAI 兼容 chat/completions，返回 {text, usage, cost, model}""" + """调用 OpenAI 兼容 chat/completions，返回 {text, usage, cost, model} + messages 支持两种格式： + - 纯文本：[{'role':'user','content':'...'}] + - 多模态：[{'role':'user','content':[{'type':'text','text':'...'}, + {'type':'image_url','image_url':{'url':'...'}}]}] + """ cfg = get_provider_cfg(provider) url = (base_url or cfg['base_url']).rstrip('/') + '/chat/completions' key = api_key or cfg['api_key'] @@ -56,7 +61,14 @@ def chat(provider, model, messages, temperature=0.7, max_tokens=None, resp = requests.post(url, json=payload, headers=headers, timeout=timeout) if resp.status_code == 200: data = resp.json() - text = data['choices'][0]['message']['content'] or '' + msg = data['choices'][0]['message'] + text = msg.get('content') or '' + if not text: + # 推理模型偶发 content 为空：用 reasoning_content 兜底 + text = msg.get('reasoning_content') or '' + if not text: + last_err = LLMError('模型返回空内容，重试中…') + continue usage = data.get('usage', {}) pt = usage.get('prompt_tokens', 0) ct = usage.get('completion_tokens', 0) @@ -84,6 +96,48 @@ def chat(provider, model, messages, temperature=0.7, max_tokens=None, raise last_err or LLMError('未知错误') +def chat_vision(provider, model, text, image_url=None, image_path=None, + temperature=0.4, max_tokens=2000, base_url=None, api_key=None, + retries=3): + """多模态视觉调用：文本 + 图片（URL 或本地路径/base64）。 + 返回与 chat() 相同结构。 + 容错：聚合 API 偶发路由到纯文本后端（不认识 image_url），自动重试。""" + import base64 as _b64 + import time as _time + content = [{'type': 'text', 'text': text}] + img_url = image_url + if image_path: + with open(image_path, 'rb') as f: + raw = f.read() + mime = 'image/png' + if image_path.lower().endswith(('.jpg', '.jpeg')): + mime = 'image/jpeg' + elif image_path.lower().endswith('.gif'): + mime = 'image/gif' + elif image_path.lower().endswith('.webp'): + mime = 'image/webp' + img_url = f'data:{mime};base64,{_b64.b64encode(raw).decode()}' + if img_url: + content.append({'type': 'image_url', 'image_url': {'url': img_url}}) + messages = [{'role': 'user', 'content': content}] + last_err = None + for attempt in range(max(1, retries)): + try: + return chat(provider, model, messages, + temperature=temperature, max_tokens=max_tokens, + base_url=base_url, api_key=api_key) + except LLMError as e: + last_err = e + msg = str(e) + # 仅对“多模态格式不被支持/图片无效”类错误重试（聚合后端路由问题） + if any(k in msg for k in ('image_url', 'InvalidParameter', 'invalid_parameter', + 'does not appear to be valid', 'image')): + _time.sleep(2 * (attempt + 1)) + continue + raise + raise last_err or LLMError('视觉调用失败') + + def test_connection(provider, model, base_url=None, api_key=None): """连通性测试：发一条最小请求""" t0 = time.time() diff --git a/static/app.js b/static/app.js index 008e3d8..5dd63fe 100644 --- a/static/app.js +++ b/static/app.js @@ -623,6 +623,7 @@ async function pageWorkers() { ${w.task_cost_limit ? '单任务 ¥' + w.task_cost_limit : '—'}
${w.monthly_cost_limit ? '月 ¥' + w.monthly_cost_limit : ''} ${w.status === 'enabled' ? '启用' : '停用'} + `).join('') || '还没有 Worker，注册第一个虚拟员工吧'} @@ -636,6 +637,27 @@ async function testWorker(wid) { catch (e) { toast('❌ ' + e.message, 'err'); } } +async function visionTest(wid) { + openModal(` +

🖼️ 视觉能力测试

+
多模态智能体：输入图片 URL 与问题，验证图像理解能力。示例：
+ https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/640px-Cat03.jpg
+ + + +
`); + $('#vt-go').addEventListener('click', async () => { + const url = $('#vt-url').value.trim(); + if (!url) return toast('请填写图片 URL', 'err'); + $('#vt-out').innerHTML = '
视觉分析中（多模态调用约 10-60 秒）…
'; + try { + const r = await api(`/api/workers/${wid}/vision_test`, {method:'POST', body: {image_url: url, question: $('#vt-q').value}}); + $('#vt-out').innerHTML = `
${esc(r.data.text)}
+
tokens ${r.data.total_tokens} · 成本 ${fmtMoney(r.data.cost)} · 延迟 ${r.data.latency_ms ?? '—'}ms
`; + } catch (e) { $('#vt-out').innerHTML = `
${esc(e.message)}
`; } + }); +} + async function delWorker(wid) { if (!confirm('确认删除该 Worker？（历史任务将解除指派）')) return; await api(`/api/workers/${wid}`, {method:'DELETE'}); diff --git a/templates.py b/templates.py index 013a98b..bddc663 100644 --- a/templates.py +++ b/templates.py @@ -180,7 +180,7 @@ def apply_project_template(tpl, variables, worker_id=None): return pid, created -def apply_team_template(tpl, variables, provider='doubao', model='doubao-seed-evolving'): +def apply_team_template(tpl, variables, provider='deepseek', model='deepseek-v4-flash'): """应用团队模板 → 批量注册 Worker""" content = render(json.loads(tpl['content']), variables) created = []