2 Commits

Author SHA1 Message Date
fc43e6e806 优化等待设置,支持长时间验证过程
改进:
- 页面加载等待时间上限提高到 60秒
- 添加等待 networkidle 状态逻辑
- 前端界面增加等待设置说明
- 提示用户针对验证网站设置 10-30秒等待

使用场景:
- Cloudflare 等验证网站
- 复杂加载过程网站
- 需要用户确认的页面
2026-07-05 01:32:52 +08:00
5783feb60d 集成 playwright-stealth 专业反检测插件
添加:
- playwright-stealth 反检测库
- 增强 navigator 属性模拟
- 真实 Chrome 浏览器支持(如果已安装)
- 更强的反自动化检测配置

注意:
- 某些高防护网站(如 Cloudflare)仍可能拦截
- 建议使用代理或真实 Chrome 浏览器
2026-07-05 01:28:24 +08:00
2 changed files with 124 additions and 36 deletions

124
app.py
View File

@@ -20,8 +20,17 @@ from datetime import datetime
try: try:
from playwright.async_api import async_playwright from playwright.async_api import async_playwright
PLAYWRIGHT_AVAILABLE = True PLAYWRIGHT_AVAILABLE = True
# 尝试导入 stealth 反检测
try:
from playwright_stealth import stealth_async
STEALTH_AVAILABLE = True
except ImportError:
STEALTH_AVAILABLE = False
print("⚠️ playwright-stealth 未安装,反爬虫能力降低")
except ImportError: except ImportError:
PLAYWRIGHT_AVAILABLE = False PLAYWRIGHT_AVAILABLE = False
STEALTH_AVAILABLE = False
print("⚠️ Playwright 未安装,请运行: pip install playwright && playwright install chromium")
# 检查 agent-browser 是否可用 # 检查 agent-browser 是否可用
def check_agent_browser(): def check_agent_browser():
@@ -204,16 +213,30 @@ async def capture_with_playwright(
try: try:
async with async_playwright() as p: async with async_playwright() as p:
# 启动浏览器,添加反检测配置 # 启动浏览器,添加反检测配置
browser = await p.chromium.launch( # 尝试使用真实 Chrome如果安装了
headless=True, try:
args=[ # 首先尝试使用真实 Chrome 浏览器
'--disable-blink-features=AutomationControlled', # 反自动化检测 browser = await p.chromium.launch(
'--disable-dev-shm-usage', headless=True,
'--no-sandbox', channel='chrome', # 使用真实 Chrome 而不是 Chromium
'--disable-setuid-sandbox', args=[
'--disable-web-security', '--disable-blink-features=AutomationControlled',
] '--disable-dev-shm-usage',
) '--no-sandbox',
'--ignore-certificate-errors',
]
)
except:
# 如果没有 Chrome使用 Chromium
browser = await p.chromium.launch(
headless=True,
args=[
'--disable-blink-features=AutomationControlled',
'--disable-dev-shm-usage',
'--no-sandbox',
'--ignore-certificate-errors',
]
)
viewport_settings = viewport or {"width": 1920, "height": 1080} viewport_settings = viewport or {"width": 1920, "height": 1080}
@@ -223,40 +246,97 @@ async def capture_with_playwright(
"width": viewport_settings.get("width", 1920), "width": viewport_settings.get("width", 1920),
"height": viewport_settings.get("height", 1080) "height": viewport_settings.get("height", 1080)
}, },
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.6099.109 Safari/537.36',
# 添加更多浏览器特征 # 添加更多浏览器特征
locale='zh-CN', locale='en-US',
timezone_id='Asia/Shanghai', timezone_id='America/New_York',
geolocation={'latitude': 31.2304, 'longitude': 121.4737}, # 上海坐标 color_scheme='light',
permissions=['geolocation'], has_touch=False,
is_mobile=False,
java_script_enabled=True,
accept_downloads=True,
) )
# 注入反检测脚本 # 注入反检测脚本
await context.add_init_script(""" await context.add_init_script("""
// 移除 webdriver 标记
Object.defineProperty(navigator, 'webdriver', { Object.defineProperty(navigator, 'webdriver', {
get: () => undefined get: () => undefined
}); });
// 添加插件
Object.defineProperty(navigator, 'plugins', { Object.defineProperty(navigator, 'plugins', {
get: () => [1, 2, 3, 4, 5] get: () => {
return [
{name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format'},
{name: 'Chrome PDF Viewer', filename: 'mhjfbmdgncjokjjpdkfnmlglggk', description: ''},
{name: 'Native Client', filename: 'internal-nacl-plugin', description: ''}
];
}
}); });
// 语言列表
Object.defineProperty(navigator, 'languages', { Object.defineProperty(navigator, 'languages', {
get: () => ['zh-CN', 'zh', 'en', 'en-US'] get: () => ['en-US', 'en', 'zh-CN']
}); });
// 平台
Object.defineProperty(navigator, 'platform', {
get: () => 'Win32'
});
// 硬件并发
Object.defineProperty(navigator, 'hardwareConcurrency', {
get: () => 8
});
// 设备内存
Object.defineProperty(navigator, 'deviceMemory', {
get: () => 8
});
// Chrome 对象
window.chrome = { window.chrome = {
runtime: {} runtime: {
PlatformOs: {MAC: 'mac', WIN: 'win', ANDROID: 'android', CROS: 'cros', LINUX: 'linux', OPENBSD: 'openbsd'},
PlatformArch: {ARM: 'arm', X86_32: 'x86-32', X86_64: 'x86-64'},
PlatformNaclArch: {ARM: 'arm', X86_32: 'x86-32', X86_64: 'x86-64'},
Request: function() {},
connect: function() {},
sendMessage: function() {}
},
loadTimes: function() {},
csi: function() {},
app: {}
}; };
""") """)
page = await context.new_page() page = await context.new_page()
try: # 应用 stealth 反检测(如果可用)
await page.goto(url, wait_until="networkidle", timeout=30000) if STEALTH_AVAILABLE:
except: await stealth_async(page)
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
try:
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
except Exception as e:
# 如果 domcontentloaded 超时,尝试 commit
try:
await page.goto(url, wait_until="commit", timeout=30000)
except:
pass
# 固定等待时间(用于验证、加载等过程)
if wait_time > 0: if wait_time > 0:
await page.wait_for_timeout(wait_time) await page.wait_for_timeout(wait_time)
# 等待页面基本稳定
try:
await page.wait_for_load_state("networkidle", timeout=5000)
except:
# networkidle 超时没关系,继续执行
pass
if scroll_times > 0: if scroll_times > 0:
for i in range(scroll_times): for i in range(scroll_times):
await page.evaluate("window.scrollBy(0, 800)") await page.evaluate("window.scrollBy(0, 800)")

View File

@@ -295,20 +295,21 @@
</div> </div>
<div class="form-group"> <div class="form-group">
<label>滚动设置(用于加载动态内容</label> <label>等待设置(重要!</label>
<div class="row"> <div class="row">
<div> <div>
<label for="scrollTimes">滚动次数</label> <label for="waitTime">页面加载等待(ms) ⏱️</label>
<input type="number" id="waitTime" value="2000" min="0" max="60000" step="500">
<small style="color: #666; display: block; margin-top: 4px;">某些网站需要较长验证过程,可设置为 10000-30000</small>
</div>
<div>
<label for="scrollTimes">滚动次数(可选)</label>
<input type="number" id="scrollTimes" value="0" min="0" max="20"> <input type="number" id="scrollTimes" value="0" min="0" max="20">
</div> </div>
<div> <div>
<label for="scrollDelay">滚动间隔(ms)</label> <label for="scrollDelay">滚动间隔(ms)</label>
<input type="number" id="scrollDelay" value="1000" min="500" max="5000" step="100"> <input type="number" id="scrollDelay" value="1000" min="500" max="5000" step="100">
</div> </div>
<div>
<label for="waitTime">页面加载等待(ms)</label>
<input type="number" id="waitTime" value="2000" min="0" max="10000" step="500">
</div>
</div> </div>
</div> </div>
@@ -350,16 +351,23 @@
</div> </div>
<div class="card api-docs"> <div class="card api-docs">
<h3>📚 API 文档</h3> <h3>📚 使用说明</h3>
<p>POST /api/capture</p> <p><strong>重要提示:</strong></p>
<pre><code>{ <ul style="margin: 10px 0; line-height: 1.8;">
<li>⏱️ <strong>页面加载等待</strong>:某些网站需要验证过程(如 Cloudflare请设置较长时间10000-30000ms</li>
<li>🔄 <strong>滚动次数</strong>:用于加载动态内容(如微博、推特等),建议 3-5 次</li>
<li>🎯 <strong>全页截图</strong>:滚动加载后建议开启此选项</li>
</ul>
<p style="margin-top: 15px;"><strong>API 调用:</strong></p>
<pre><code>POST /api/capture
{
"url": "https://example.com", "url": "https://example.com",
"action": "screenshot", // 或 "html" "action": "screenshot", // 或 "html"
"scroll_times": 3, "wait_time": 15000, // 重要!验证网站需设置较长等待
"scroll_delay": 1000, "scroll_times": 3, // 可选:加载动态内容
"full_page": true, "scroll_delay": 1000, // 可选:滚动间隔
"viewport": {"width": 1920, "height": 1080}, "full_page": true, // 可选:全页截图
"wait_time": 2000 "backend": "playwright" // 可选playwright 或 agent-browser
}</code></pre> }</code></pre>
</div> </div>
</div> </div>