v2.0.1:修复推理型模型(Qwen3/DeepSeek reasoning_content)误报未收到输出+连接测试按连通判定+采样失败不中断+流式超时放宽60s/300s
This commit is contained in:
@@ -24,6 +24,8 @@
|
|||||||
|
|
||||||
### 📊 指标与结果
|
### 📊 指标与结果
|
||||||
- 实时指标卡:首字延迟、预填充速度、解码速度、上文/输出 tokens、总耗时
|
- 实时指标卡:首字延迟、预填充速度、解码速度、上文/输出 tokens、总耗时
|
||||||
|
- **推理型模型兼容**:支持 Qwen3 / DeepSeek 等思维链模型(`reasoning_content` / `thinking` / `thought`),思维过程计入输出,不会误报“未收到输出”
|
||||||
|
- **采样失败不中断**:单次采样失败会记录并继续,不会让整个测试半途终止;全部失败才标记 error
|
||||||
- 实时控制台日志:校准、预热、每次采样明细全程可追溯
|
- 实时控制台日志:校准、预热、每次采样明细全程可追溯
|
||||||
- **每次完整测试**支持:
|
- **每次完整测试**支持:
|
||||||
- **网页点击查看**:历史记录「查看」按钮弹出详情(整体平均 + 按上下文长度分组 + 每次采样明细 + 完整日志)
|
- **网页点击查看**:历史记录「查看」按钮弹出详情(整体平均 + 按上下文长度分组 + 每次采样明细 + 完整日志)
|
||||||
@@ -139,13 +141,14 @@ llm-speed-tester/
|
|||||||
|
|
||||||
## 常见问题
|
## 常见问题
|
||||||
|
|
||||||
|
- **连接测试提示“未收到任何输出内容”**:多为推理型模型(Qwen3/DeepSeek 思维链)或只返回 usage 的网关。已兼容 `reasoning_content` 等思维字段,连接成功即视为通过;若仍出现,请检查 API Key/Base URL/模型名。
|
||||||
- **无 openpyxl**:`pip install openpyxl`(已加入 requirements.txt)
|
- **无 openpyxl**:`pip install openpyxl`(已加入 requirements.txt)
|
||||||
- **老版本数据库**:程序启动时自动迁移,为 `test_runs` 表补充 `context_length` 列,无需手动处理
|
- **老版本数据库**:程序启动时自动迁移,为 `test_runs` 表补充 `context_length` 列,无需手动处理
|
||||||
- **慢模型超时**:连接超时 30s、两次数据包间隔 120s,足够覆盖大多数慢模型;超长文(131072)生成慢属正常,请耐心等待
|
- **慢模型/长上文超时**:连接超时 60s、两次数据包间隔 300s(`config.STREAM_READ_TIMEOUT` 可调);推理型模型思考阶段停顿不计超时,超长文(131072)生成慢属正常,请耐心等待
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Git
|
## Git
|
||||||
|
|
||||||
- **仓库:** `hz4th_coder/llm-speed-tester`
|
- **仓库:** `hz4th_coder/llm-speed-tester`
|
||||||
- **版本:** v2.0.0(新增多上下文长度测试 + 预热 + Excel 导出 + 界面优化)
|
- **版本:** v2.0.1(新增多上下文长度测试 + 预热 + Excel 导出 + 界面优化 + 推理型模型兼容 + 采样失败不中断)
|
||||||
@@ -74,8 +74,13 @@ def test_config():
|
|||||||
if not cfg.get("api_key"):
|
if not cfg.get("api_key"):
|
||||||
return jsonify({"ok": False, "error": "请填写 API Key"}), 400
|
return jsonify({"ok": False, "error": "请填写 API Key"}), 400
|
||||||
try:
|
try:
|
||||||
m = call_stream(cfg, "你好,请只回复:OK", {"max_tokens": 16, "avoid_cache": False})
|
# 连接测试:只要流式请求成功返回(哪怕正文为空/只有思维链)都算连通
|
||||||
return jsonify({"ok": True, "total_ms": m["total_ms"], "metrics": m})
|
m = call_stream(cfg, "你好,请简要回答:1+1=?",
|
||||||
|
{"max_tokens": 32, "avoid_cache": False})
|
||||||
|
note = ""
|
||||||
|
if not (m.get("output_tokens") or m.get("output_chars")):
|
||||||
|
note = "(连接正常,但本次未返回正文内容,可能为推理型模型)"
|
||||||
|
return jsonify({"ok": True, "total_ms": m["total_ms"], "metrics": m, "note": note})
|
||||||
except ProviderError as e:
|
except ProviderError as e:
|
||||||
return jsonify({"ok": False, "error": str(e)})
|
return jsonify({"ok": False, "error": str(e)})
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|||||||
@@ -10,6 +10,6 @@ DATA_DIR = os.path.join(BASE_DIR, "data")
|
|||||||
LOG_DIR = os.path.join(BASE_DIR, "logs")
|
LOG_DIR = os.path.join(BASE_DIR, "logs")
|
||||||
DB_PATH = os.path.join(DATA_DIR, "llm_speed_tester.db")
|
DB_PATH = os.path.join(DATA_DIR, "llm_speed_tester.db")
|
||||||
|
|
||||||
# 流式请求超时:连接 30s,两次数据包间隔最长 120s(慢模型也够用)
|
# 流式请求超时:连接 60s,两次数据包间隔最长 300s(推理型/长上文模型也够用)
|
||||||
CONNECT_TIMEOUT = 30
|
CONNECT_TIMEOUT = 60
|
||||||
STREAM_READ_TIMEOUT = 120
|
STREAM_READ_TIMEOUT = 300
|
||||||
+23
-11
@@ -56,6 +56,8 @@ def _parse_sse_line(line):
|
|||||||
|
|
||||||
def _metrics(start, first_token_at, end, prompt_tokens, output_tokens,
|
def _metrics(start, first_token_at, end, prompt_tokens, output_tokens,
|
||||||
cached_tokens, output_chars, prompt_chars):
|
cached_tokens, output_chars, prompt_chars):
|
||||||
|
if first_token_at is None:
|
||||||
|
first_token_at = end # 未收到正文但请求完成(如纯 usage 响应)
|
||||||
ttft_ms = (first_token_at - start) * 1000
|
ttft_ms = (first_token_at - start) * 1000
|
||||||
decode_ms = (end - first_token_at) * 1000
|
decode_ms = (end - first_token_at) * 1000
|
||||||
total_ms = (end - start) * 1000
|
total_ms = (end - start) * 1000
|
||||||
@@ -103,6 +105,7 @@ def stream_openai(cfg, prompt, gen, log, should_stop=None):
|
|||||||
first_token_at = None
|
first_token_at = None
|
||||||
output_chars = 0
|
output_chars = 0
|
||||||
prompt_tokens = output_tokens = cached_tokens = 0
|
prompt_tokens = output_tokens = cached_tokens = 0
|
||||||
|
event_count = 0
|
||||||
resp = None
|
resp = None
|
||||||
try:
|
try:
|
||||||
while True:
|
while True:
|
||||||
@@ -126,13 +129,15 @@ def stream_openai(cfg, prompt, gen, log, should_stop=None):
|
|||||||
for obj in _iter_json(resp):
|
for obj in _iter_json(resp):
|
||||||
if should_stop and should_stop():
|
if should_stop and should_stop():
|
||||||
raise StopRequested()
|
raise StopRequested()
|
||||||
|
event_count += 1
|
||||||
if obj.get("choices"):
|
if obj.get("choices"):
|
||||||
delta = obj["choices"][0].get("delta") or {}
|
delta = obj["choices"][0].get("delta") or {}
|
||||||
text = delta.get("content") or ""
|
# 兼容推理型模型:Qwen3/DeepSeek 思维链在 reasoning_content
|
||||||
if text:
|
piece = delta.get("content") or delta.get("reasoning_content") or ""
|
||||||
|
if piece:
|
||||||
if first_token_at is None:
|
if first_token_at is None:
|
||||||
first_token_at = time.time()
|
first_token_at = time.time()
|
||||||
output_chars += len(text)
|
output_chars += len(piece)
|
||||||
usage = obj.get("usage")
|
usage = obj.get("usage")
|
||||||
if usage:
|
if usage:
|
||||||
prompt_tokens = usage.get("prompt_tokens") or 0
|
prompt_tokens = usage.get("prompt_tokens") or 0
|
||||||
@@ -148,8 +153,8 @@ def stream_openai(cfg, prompt, gen, log, should_stop=None):
|
|||||||
if resp is not None:
|
if resp is not None:
|
||||||
resp.close()
|
resp.close()
|
||||||
|
|
||||||
if first_token_at is None:
|
if event_count == 0:
|
||||||
raise ProviderError("未收到任何输出内容")
|
raise ProviderError("未收到任何输出内容(HTTP 200 但响应流为空)")
|
||||||
end = time.time()
|
end = time.time()
|
||||||
return _metrics(start, first_token_at, end, prompt_tokens, output_tokens,
|
return _metrics(start, first_token_at, end, prompt_tokens, output_tokens,
|
||||||
cached_tokens, output_chars, len(prompt))
|
cached_tokens, output_chars, len(prompt))
|
||||||
@@ -190,6 +195,7 @@ def stream_anthropic(cfg, prompt, gen, log, should_stop=None):
|
|||||||
first_token_at = None
|
first_token_at = None
|
||||||
output_chars = 0
|
output_chars = 0
|
||||||
prompt_tokens = output_tokens = 0
|
prompt_tokens = output_tokens = 0
|
||||||
|
event_count = 0
|
||||||
resp = None
|
resp = None
|
||||||
try:
|
try:
|
||||||
if should_stop and should_stop():
|
if should_stop and should_stop():
|
||||||
@@ -204,12 +210,15 @@ def stream_anthropic(cfg, prompt, gen, log, should_stop=None):
|
|||||||
for obj in _iter_json(resp):
|
for obj in _iter_json(resp):
|
||||||
if should_stop and should_stop():
|
if should_stop and should_stop():
|
||||||
raise StopRequested()
|
raise StopRequested()
|
||||||
|
event_count += 1
|
||||||
etype = obj.get("type")
|
etype = obj.get("type")
|
||||||
if etype == "message_start":
|
if etype == "message_start":
|
||||||
usage = (obj.get("message") or {}).get("usage") or {}
|
usage = (obj.get("message") or {}).get("usage") or {}
|
||||||
prompt_tokens = usage.get("input_tokens") or 0
|
prompt_tokens = usage.get("input_tokens") or 0
|
||||||
elif etype == "content_block_delta":
|
elif etype == "content_block_delta":
|
||||||
text = (obj.get("delta") or {}).get("text") or ""
|
delta = obj.get("delta") or {}
|
||||||
|
# 兼容 extended thinking:thinking 文本也算输出
|
||||||
|
text = delta.get("text") or delta.get("thinking") or ""
|
||||||
if text:
|
if text:
|
||||||
if first_token_at is None:
|
if first_token_at is None:
|
||||||
first_token_at = time.time()
|
first_token_at = time.time()
|
||||||
@@ -225,8 +234,8 @@ def stream_anthropic(cfg, prompt, gen, log, should_stop=None):
|
|||||||
if resp is not None:
|
if resp is not None:
|
||||||
resp.close()
|
resp.close()
|
||||||
|
|
||||||
if first_token_at is None:
|
if event_count == 0:
|
||||||
raise ProviderError("未收到任何输出内容")
|
raise ProviderError("未收到任何输出内容(HTTP 200 但响应流为空)")
|
||||||
end = time.time()
|
end = time.time()
|
||||||
return _metrics(start, first_token_at, end, prompt_tokens, output_tokens,
|
return _metrics(start, first_token_at, end, prompt_tokens, output_tokens,
|
||||||
0, output_chars, len(prompt))
|
0, output_chars, len(prompt))
|
||||||
@@ -252,6 +261,7 @@ def stream_google(cfg, prompt, gen, log, should_stop=None):
|
|||||||
first_token_at = None
|
first_token_at = None
|
||||||
output_chars = 0
|
output_chars = 0
|
||||||
prompt_tokens = output_tokens = cached_tokens = 0
|
prompt_tokens = output_tokens = cached_tokens = 0
|
||||||
|
event_count = 0
|
||||||
resp = None
|
resp = None
|
||||||
try:
|
try:
|
||||||
if should_stop and should_stop():
|
if should_stop and should_stop():
|
||||||
@@ -266,11 +276,13 @@ def stream_google(cfg, prompt, gen, log, should_stop=None):
|
|||||||
for obj in _iter_json(resp):
|
for obj in _iter_json(resp):
|
||||||
if should_stop and should_stop():
|
if should_stop and should_stop():
|
||||||
raise StopRequested()
|
raise StopRequested()
|
||||||
|
event_count += 1
|
||||||
cands = obj.get("candidates") or []
|
cands = obj.get("candidates") or []
|
||||||
if cands:
|
if cands:
|
||||||
parts = (cands[0].get("content") or {}).get("parts") or []
|
parts = (cands[0].get("content") or {}).get("parts") or []
|
||||||
for part in parts:
|
for part in parts:
|
||||||
text = part.get("text") or ""
|
# 兼容 thinking 模型:thought 文本也算输出
|
||||||
|
text = part.get("text") or part.get("thought") or ""
|
||||||
if text:
|
if text:
|
||||||
if first_token_at is None:
|
if first_token_at is None:
|
||||||
first_token_at = time.time()
|
first_token_at = time.time()
|
||||||
@@ -288,8 +300,8 @@ def stream_google(cfg, prompt, gen, log, should_stop=None):
|
|||||||
if resp is not None:
|
if resp is not None:
|
||||||
resp.close()
|
resp.close()
|
||||||
|
|
||||||
if first_token_at is None:
|
if event_count == 0:
|
||||||
raise ProviderError("未收到任何输出内容")
|
raise ProviderError("未收到任何输出内容(HTTP 200 但响应流为空)")
|
||||||
end = time.time()
|
end = time.time()
|
||||||
return _metrics(start, first_token_at, end, prompt_tokens, output_tokens,
|
return _metrics(start, first_token_at, end, prompt_tokens, output_tokens,
|
||||||
cached_tokens, output_chars, len(prompt))
|
cached_tokens, output_chars, len(prompt))
|
||||||
|
|||||||
+1
-1
@@ -158,7 +158,7 @@ async function testConnection() {
|
|||||||
if (r.ok) {
|
if (r.ok) {
|
||||||
const m = r.metrics || {};
|
const m = r.metrics || {};
|
||||||
showConn(true,
|
showConn(true,
|
||||||
`✅ 连接成功(${r.total_ms}ms)| 首字 ${fmt(m.ttft_ms)}ms | 提示词 ${fmt(m.prompt_tokens)} tok | 输出 ${fmt(m.output_tokens)} tok`);
|
`✅ 连接成功(${r.total_ms}ms)| 首字 ${fmt(m.ttft_ms)}ms | 提示词 ${fmt(m.prompt_tokens)} tok | 输出 ${fmt(m.output_tokens)} tok${r.note ? " " + r.note : ""}`);
|
||||||
} else {
|
} else {
|
||||||
showConn(false, "❌ " + (r.error || "连接失败"));
|
showConn(false, "❌ " + (r.error || "连接失败"));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ class TestRunner(threading.Thread):
|
|||||||
self.start_wall = time.time()
|
self.start_wall = time.time()
|
||||||
self.ratio = None
|
self.ratio = None
|
||||||
self.samples = []
|
self.samples = []
|
||||||
|
self.last_error = None
|
||||||
|
|
||||||
def request_cancel(self):
|
def request_cancel(self):
|
||||||
self.cancel_flag = True
|
self.cancel_flag = True
|
||||||
@@ -97,14 +98,26 @@ class TestRunner(threading.Thread):
|
|||||||
except StopRequested:
|
except StopRequested:
|
||||||
raise
|
raise
|
||||||
except ProviderError as e:
|
except ProviderError as e:
|
||||||
|
# 单次采样失败:记录并继续后续采样,不让整个测试中断
|
||||||
|
self.last_error = str(e)
|
||||||
self.log("ERROR", "[%d tok] 采样 %d/%d 失败: %s" % (L, i, n, e))
|
self.log("ERROR", "[%d tok] 采样 %d/%d 失败: %s" % (L, i, n, e))
|
||||||
self.samples.append({"run_index": i, "context_length": L, "ok": False, "error": str(e)})
|
self.samples.append({"run_index": i, "context_length": L, "ok": False, "error": str(e)})
|
||||||
db.add_run(self.test_id, i, {}, str(e), context_length=L)
|
db.add_run(self.test_id, i, {}, str(e), context_length=L)
|
||||||
raise e
|
|
||||||
|
|
||||||
summary = self._make_summary()
|
summary = self._make_summary()
|
||||||
db.update_status(self.test_id, "done", summary=summary)
|
ok_count = summary.get("samples_ok") or 0
|
||||||
self.log("INFO", "═══ 测试完成 ═══")
|
fail_count = summary.get("samples_total", 0) - ok_count
|
||||||
|
if ok_count:
|
||||||
|
db.update_status(self.test_id, "done", summary=summary,
|
||||||
|
error=("%d 次采样失败:%s" % (fail_count, self.last_error)) if fail_count else "")
|
||||||
|
self.log("INFO", "═══ 测试完成 ═══")
|
||||||
|
if fail_count:
|
||||||
|
self.log("WARN", "共 %d 次采样失败(最后错误:%s)" % (fail_count, self.last_error))
|
||||||
|
else:
|
||||||
|
db.update_status(self.test_id, "error", summary=summary,
|
||||||
|
error=self.last_error or "所有采样均失败")
|
||||||
|
self.log("ERROR", "所有采样均失败,测试标记为 error(最后错误:%s)" % (self.last_error or "未知"))
|
||||||
|
return
|
||||||
self.log("INFO", "汇总: 平均首字 %.1f ms | 平均预填充 %.1f tok/s | 平均解码 %.1f tok/s"
|
self.log("INFO", "汇总: 平均首字 %.1f ms | 平均预填充 %.1f tok/s | 平均解码 %.1f tok/s"
|
||||||
% (summary.get("avg_ttft_ms") or 0,
|
% (summary.get("avg_ttft_ms") or 0,
|
||||||
summary.get("avg_prefill_speed") or 0,
|
summary.get("avg_prefill_speed") or 0,
|
||||||
|
|||||||
Reference in New Issue
Block a user