2 Commits

Author SHA1 Message Date
fc43e6e806 优化等待设置,支持长时间验证过程
改进:
- 页面加载等待时间上限提高到 60秒
- 添加等待 networkidle 状态逻辑
- 前端界面增加等待设置说明
- 提示用户针对验证网站设置 10-30秒等待

使用场景:
- Cloudflare 等验证网站
- 复杂加载过程网站
- 需要用户确认的页面
2026-07-05 01:32:52 +08:00
5783feb60d 集成 playwright-stealth 专业反检测插件
添加:
- playwright-stealth 反检测库
- 增强 navigator 属性模拟
- 真实 Chrome 浏览器支持(如果已安装)
- 更强的反自动化检测配置

注意:
- 某些高防护网站(如 Cloudflare)仍可能拦截
- 建议使用代理或真实 Chrome 浏览器
2026-07-05 01:28:24 +08:00
2 changed files with 124 additions and 36 deletions

110
app.py
View File

@@ -20,8 +20,17 @@ from datetime import datetime
try:
from playwright.async_api import async_playwright
PLAYWRIGHT_AVAILABLE = True
# 尝试导入 stealth 反检测
try:
from playwright_stealth import stealth_async
STEALTH_AVAILABLE = True
except ImportError:
STEALTH_AVAILABLE = False
print("⚠️ playwright-stealth 未安装,反爬虫能力降低")
except ImportError:
PLAYWRIGHT_AVAILABLE = False
STEALTH_AVAILABLE = False
print("⚠️ Playwright 未安装,请运行: pip install playwright && playwright install chromium")
# 检查 agent-browser 是否可用
def check_agent_browser():
@@ -204,14 +213,28 @@ async def capture_with_playwright(
try:
async with async_playwright() as p:
# 启动浏览器,添加反检测配置
# 尝试使用真实 Chrome如果安装了
try:
# 首先尝试使用真实 Chrome 浏览器
browser = await p.chromium.launch(
headless=True,
channel='chrome', # 使用真实 Chrome 而不是 Chromium
args=[
'--disable-blink-features=AutomationControlled',
'--disable-dev-shm-usage',
'--no-sandbox',
'--ignore-certificate-errors',
]
)
except:
# 如果没有 Chrome使用 Chromium
browser = await p.chromium.launch(
headless=True,
args=[
'--disable-blink-features=AutomationControlled', # 反自动化检测
'--disable-blink-features=AutomationControlled',
'--disable-dev-shm-usage',
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-web-security',
'--ignore-certificate-errors',
]
)
@@ -223,40 +246,97 @@ async def capture_with_playwright(
"width": viewport_settings.get("width", 1920),
"height": viewport_settings.get("height", 1080)
},
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.6099.109 Safari/537.36',
# 添加更多浏览器特征
locale='zh-CN',
timezone_id='Asia/Shanghai',
geolocation={'latitude': 31.2304, 'longitude': 121.4737}, # 上海坐标
permissions=['geolocation'],
locale='en-US',
timezone_id='America/New_York',
color_scheme='light',
has_touch=False,
is_mobile=False,
java_script_enabled=True,
accept_downloads=True,
)
# 注入反检测脚本
await context.add_init_script("""
// 移除 webdriver 标记
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined
});
// 添加插件
Object.defineProperty(navigator, 'plugins', {
get: () => [1, 2, 3, 4, 5]
get: () => {
return [
{name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format'},
{name: 'Chrome PDF Viewer', filename: 'mhjfbmdgncjokjjpdkfnmlglggk', description: ''},
{name: 'Native Client', filename: 'internal-nacl-plugin', description: ''}
];
}
});
// 语言列表
Object.defineProperty(navigator, 'languages', {
get: () => ['zh-CN', 'zh', 'en', 'en-US']
get: () => ['en-US', 'en', 'zh-CN']
});
// 平台
Object.defineProperty(navigator, 'platform', {
get: () => 'Win32'
});
// 硬件并发
Object.defineProperty(navigator, 'hardwareConcurrency', {
get: () => 8
});
// 设备内存
Object.defineProperty(navigator, 'deviceMemory', {
get: () => 8
});
// Chrome 对象
window.chrome = {
runtime: {}
runtime: {
PlatformOs: {MAC: 'mac', WIN: 'win', ANDROID: 'android', CROS: 'cros', LINUX: 'linux', OPENBSD: 'openbsd'},
PlatformArch: {ARM: 'arm', X86_32: 'x86-32', X86_64: 'x86-64'},
PlatformNaclArch: {ARM: 'arm', X86_32: 'x86-32', X86_64: 'x86-64'},
Request: function() {},
connect: function() {},
sendMessage: function() {}
},
loadTimes: function() {},
csi: function() {},
app: {}
};
""")
page = await context.new_page()
try:
await page.goto(url, wait_until="networkidle", timeout=30000)
except:
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
# 应用 stealth 反检测(如果可用)
if STEALTH_AVAILABLE:
await stealth_async(page)
try:
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
except Exception as e:
# 如果 domcontentloaded 超时,尝试 commit
try:
await page.goto(url, wait_until="commit", timeout=30000)
except:
pass
# 固定等待时间(用于验证、加载等过程)
if wait_time > 0:
await page.wait_for_timeout(wait_time)
# 等待页面基本稳定
try:
await page.wait_for_load_state("networkidle", timeout=5000)
except:
# networkidle 超时没关系,继续执行
pass
if scroll_times > 0:
for i in range(scroll_times):
await page.evaluate("window.scrollBy(0, 800)")

View File

@@ -295,20 +295,21 @@
</div>
<div class="form-group">
<label>滚动设置(用于加载动态内容</label>
<label>等待设置(重要!</label>
<div class="row">
<div>
<label for="scrollTimes">滚动次数</label>
<label for="waitTime">页面加载等待(ms) ⏱️</label>
<input type="number" id="waitTime" value="2000" min="0" max="60000" step="500">
<small style="color: #666; display: block; margin-top: 4px;">某些网站需要较长验证过程,可设置为 10000-30000</small>
</div>
<div>
<label for="scrollTimes">滚动次数(可选)</label>
<input type="number" id="scrollTimes" value="0" min="0" max="20">
</div>
<div>
<label for="scrollDelay">滚动间隔(ms)</label>
<input type="number" id="scrollDelay" value="1000" min="500" max="5000" step="100">
</div>
<div>
<label for="waitTime">页面加载等待(ms)</label>
<input type="number" id="waitTime" value="2000" min="0" max="10000" step="500">
</div>
</div>
</div>
@@ -350,16 +351,23 @@
</div>
<div class="card api-docs">
<h3>📚 API 文档</h3>
<p>POST /api/capture</p>
<pre><code>{
<h3>📚 使用说明</h3>
<p><strong>重要提示:</strong></p>
<ul style="margin: 10px 0; line-height: 1.8;">
<li>⏱️ <strong>页面加载等待</strong>:某些网站需要验证过程(如 Cloudflare请设置较长时间10000-30000ms</li>
<li>🔄 <strong>滚动次数</strong>:用于加载动态内容(如微博、推特等),建议 3-5 次</li>
<li>🎯 <strong>全页截图</strong>:滚动加载后建议开启此选项</li>
</ul>
<p style="margin-top: 15px;"><strong>API 调用:</strong></p>
<pre><code>POST /api/capture
{
"url": "https://example.com",
"action": "screenshot", // 或 "html"
"scroll_times": 3,
"scroll_delay": 1000,
"full_page": true,
"viewport": {"width": 1920, "height": 1080},
"wait_time": 2000
"wait_time": 15000, // 重要!验证网站需设置较长等待
"scroll_times": 3, // 可选:加载动态内容
"scroll_delay": 1000, // 可选:滚动间隔
"full_page": true, // 可选:全页截图
"backend": "playwright" // 可选playwright 或 agent-browser
}</code></pre>
</div>
</div>