Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| fc43e6e806 | |||
| 5783feb60d |
124
app.py
124
app.py
@@ -20,8 +20,17 @@ from datetime import datetime
|
|||||||
try:
|
try:
|
||||||
from playwright.async_api import async_playwright
|
from playwright.async_api import async_playwright
|
||||||
PLAYWRIGHT_AVAILABLE = True
|
PLAYWRIGHT_AVAILABLE = True
|
||||||
|
# 尝试导入 stealth 反检测
|
||||||
|
try:
|
||||||
|
from playwright_stealth import stealth_async
|
||||||
|
STEALTH_AVAILABLE = True
|
||||||
|
except ImportError:
|
||||||
|
STEALTH_AVAILABLE = False
|
||||||
|
print("⚠️ playwright-stealth 未安装,反爬虫能力降低")
|
||||||
except ImportError:
|
except ImportError:
|
||||||
PLAYWRIGHT_AVAILABLE = False
|
PLAYWRIGHT_AVAILABLE = False
|
||||||
|
STEALTH_AVAILABLE = False
|
||||||
|
print("⚠️ Playwright 未安装,请运行: pip install playwright && playwright install chromium")
|
||||||
|
|
||||||
# 检查 agent-browser 是否可用
|
# 检查 agent-browser 是否可用
|
||||||
def check_agent_browser():
|
def check_agent_browser():
|
||||||
@@ -204,16 +213,30 @@ async def capture_with_playwright(
|
|||||||
try:
|
try:
|
||||||
async with async_playwright() as p:
|
async with async_playwright() as p:
|
||||||
# 启动浏览器,添加反检测配置
|
# 启动浏览器,添加反检测配置
|
||||||
browser = await p.chromium.launch(
|
# 尝试使用真实 Chrome(如果安装了)
|
||||||
headless=True,
|
try:
|
||||||
args=[
|
# 首先尝试使用真实 Chrome 浏览器
|
||||||
'--disable-blink-features=AutomationControlled', # 反自动化检测
|
browser = await p.chromium.launch(
|
||||||
'--disable-dev-shm-usage',
|
headless=True,
|
||||||
'--no-sandbox',
|
channel='chrome', # 使用真实 Chrome 而不是 Chromium
|
||||||
'--disable-setuid-sandbox',
|
args=[
|
||||||
'--disable-web-security',
|
'--disable-blink-features=AutomationControlled',
|
||||||
]
|
'--disable-dev-shm-usage',
|
||||||
)
|
'--no-sandbox',
|
||||||
|
'--ignore-certificate-errors',
|
||||||
|
]
|
||||||
|
)
|
||||||
|
except:
|
||||||
|
# 如果没有 Chrome,使用 Chromium
|
||||||
|
browser = await p.chromium.launch(
|
||||||
|
headless=True,
|
||||||
|
args=[
|
||||||
|
'--disable-blink-features=AutomationControlled',
|
||||||
|
'--disable-dev-shm-usage',
|
||||||
|
'--no-sandbox',
|
||||||
|
'--ignore-certificate-errors',
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
viewport_settings = viewport or {"width": 1920, "height": 1080}
|
viewport_settings = viewport or {"width": 1920, "height": 1080}
|
||||||
|
|
||||||
@@ -223,40 +246,97 @@ async def capture_with_playwright(
|
|||||||
"width": viewport_settings.get("width", 1920),
|
"width": viewport_settings.get("width", 1920),
|
||||||
"height": viewport_settings.get("height", 1080)
|
"height": viewport_settings.get("height", 1080)
|
||||||
},
|
},
|
||||||
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.6099.109 Safari/537.36',
|
||||||
# 添加更多浏览器特征
|
# 添加更多浏览器特征
|
||||||
locale='zh-CN',
|
locale='en-US',
|
||||||
timezone_id='Asia/Shanghai',
|
timezone_id='America/New_York',
|
||||||
geolocation={'latitude': 31.2304, 'longitude': 121.4737}, # 上海坐标
|
color_scheme='light',
|
||||||
permissions=['geolocation'],
|
has_touch=False,
|
||||||
|
is_mobile=False,
|
||||||
|
java_script_enabled=True,
|
||||||
|
accept_downloads=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
# 注入反检测脚本
|
# 注入反检测脚本
|
||||||
await context.add_init_script("""
|
await context.add_init_script("""
|
||||||
|
// 移除 webdriver 标记
|
||||||
Object.defineProperty(navigator, 'webdriver', {
|
Object.defineProperty(navigator, 'webdriver', {
|
||||||
get: () => undefined
|
get: () => undefined
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// 添加插件
|
||||||
Object.defineProperty(navigator, 'plugins', {
|
Object.defineProperty(navigator, 'plugins', {
|
||||||
get: () => [1, 2, 3, 4, 5]
|
get: () => {
|
||||||
|
return [
|
||||||
|
{name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format'},
|
||||||
|
{name: 'Chrome PDF Viewer', filename: 'mhjfbmdgncjokjjpdkfnmlglggk', description: ''},
|
||||||
|
{name: 'Native Client', filename: 'internal-nacl-plugin', description: ''}
|
||||||
|
];
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// 语言列表
|
||||||
Object.defineProperty(navigator, 'languages', {
|
Object.defineProperty(navigator, 'languages', {
|
||||||
get: () => ['zh-CN', 'zh', 'en', 'en-US']
|
get: () => ['en-US', 'en', 'zh-CN']
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// 平台
|
||||||
|
Object.defineProperty(navigator, 'platform', {
|
||||||
|
get: () => 'Win32'
|
||||||
|
});
|
||||||
|
|
||||||
|
// 硬件并发
|
||||||
|
Object.defineProperty(navigator, 'hardwareConcurrency', {
|
||||||
|
get: () => 8
|
||||||
|
});
|
||||||
|
|
||||||
|
// 设备内存
|
||||||
|
Object.defineProperty(navigator, 'deviceMemory', {
|
||||||
|
get: () => 8
|
||||||
|
});
|
||||||
|
|
||||||
|
// Chrome 对象
|
||||||
window.chrome = {
|
window.chrome = {
|
||||||
runtime: {}
|
runtime: {
|
||||||
|
PlatformOs: {MAC: 'mac', WIN: 'win', ANDROID: 'android', CROS: 'cros', LINUX: 'linux', OPENBSD: 'openbsd'},
|
||||||
|
PlatformArch: {ARM: 'arm', X86_32: 'x86-32', X86_64: 'x86-64'},
|
||||||
|
PlatformNaclArch: {ARM: 'arm', X86_32: 'x86-32', X86_64: 'x86-64'},
|
||||||
|
Request: function() {},
|
||||||
|
connect: function() {},
|
||||||
|
sendMessage: function() {}
|
||||||
|
},
|
||||||
|
loadTimes: function() {},
|
||||||
|
csi: function() {},
|
||||||
|
app: {}
|
||||||
};
|
};
|
||||||
""")
|
""")
|
||||||
|
|
||||||
page = await context.new_page()
|
page = await context.new_page()
|
||||||
|
|
||||||
try:
|
# 应用 stealth 反检测(如果可用)
|
||||||
await page.goto(url, wait_until="networkidle", timeout=30000)
|
if STEALTH_AVAILABLE:
|
||||||
except:
|
await stealth_async(page)
|
||||||
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
||||||
|
except Exception as e:
|
||||||
|
# 如果 domcontentloaded 超时,尝试 commit
|
||||||
|
try:
|
||||||
|
await page.goto(url, wait_until="commit", timeout=30000)
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# 固定等待时间(用于验证、加载等过程)
|
||||||
if wait_time > 0:
|
if wait_time > 0:
|
||||||
await page.wait_for_timeout(wait_time)
|
await page.wait_for_timeout(wait_time)
|
||||||
|
|
||||||
|
# 等待页面基本稳定
|
||||||
|
try:
|
||||||
|
await page.wait_for_load_state("networkidle", timeout=5000)
|
||||||
|
except:
|
||||||
|
# networkidle 超时没关系,继续执行
|
||||||
|
pass
|
||||||
|
|
||||||
if scroll_times > 0:
|
if scroll_times > 0:
|
||||||
for i in range(scroll_times):
|
for i in range(scroll_times):
|
||||||
await page.evaluate("window.scrollBy(0, 800)")
|
await page.evaluate("window.scrollBy(0, 800)")
|
||||||
|
|||||||
@@ -295,20 +295,21 @@
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="form-group">
|
<div class="form-group">
|
||||||
<label>滚动设置(用于加载动态内容)</label>
|
<label>等待设置(重要!)</label>
|
||||||
<div class="row">
|
<div class="row">
|
||||||
<div>
|
<div>
|
||||||
<label for="scrollTimes">滚动次数</label>
|
<label for="waitTime">页面加载等待(ms) ⏱️</label>
|
||||||
|
<input type="number" id="waitTime" value="2000" min="0" max="60000" step="500">
|
||||||
|
<small style="color: #666; display: block; margin-top: 4px;">某些网站需要较长验证过程,可设置为 10000-30000</small>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label for="scrollTimes">滚动次数(可选)</label>
|
||||||
<input type="number" id="scrollTimes" value="0" min="0" max="20">
|
<input type="number" id="scrollTimes" value="0" min="0" max="20">
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<label for="scrollDelay">滚动间隔(ms)</label>
|
<label for="scrollDelay">滚动间隔(ms)</label>
|
||||||
<input type="number" id="scrollDelay" value="1000" min="500" max="5000" step="100">
|
<input type="number" id="scrollDelay" value="1000" min="500" max="5000" step="100">
|
||||||
</div>
|
</div>
|
||||||
<div>
|
|
||||||
<label for="waitTime">页面加载等待(ms)</label>
|
|
||||||
<input type="number" id="waitTime" value="2000" min="0" max="10000" step="500">
|
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -350,16 +351,23 @@
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="card api-docs">
|
<div class="card api-docs">
|
||||||
<h3>📚 API 文档</h3>
|
<h3>📚 使用说明</h3>
|
||||||
<p>POST /api/capture</p>
|
<p><strong>重要提示:</strong></p>
|
||||||
<pre><code>{
|
<ul style="margin: 10px 0; line-height: 1.8;">
|
||||||
|
<li>⏱️ <strong>页面加载等待</strong>:某些网站需要验证过程(如 Cloudflare),请设置较长时间(10000-30000ms)</li>
|
||||||
|
<li>🔄 <strong>滚动次数</strong>:用于加载动态内容(如微博、推特等),建议 3-5 次</li>
|
||||||
|
<li>🎯 <strong>全页截图</strong>:滚动加载后建议开启此选项</li>
|
||||||
|
</ul>
|
||||||
|
<p style="margin-top: 15px;"><strong>API 调用:</strong></p>
|
||||||
|
<pre><code>POST /api/capture
|
||||||
|
{
|
||||||
"url": "https://example.com",
|
"url": "https://example.com",
|
||||||
"action": "screenshot", // 或 "html"
|
"action": "screenshot", // 或 "html"
|
||||||
"scroll_times": 3,
|
"wait_time": 15000, // 重要!验证网站需设置较长等待
|
||||||
"scroll_delay": 1000,
|
"scroll_times": 3, // 可选:加载动态内容
|
||||||
"full_page": true,
|
"scroll_delay": 1000, // 可选:滚动间隔
|
||||||
"viewport": {"width": 1920, "height": 1080},
|
"full_page": true, // 可选:全页截图
|
||||||
"wait_time": 2000
|
"backend": "playwright" // 可选:playwright 或 agent-browser
|
||||||
}</code></pre>
|
}</code></pre>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
Reference in New Issue
Block a user