feat: add external site parser worker
This commit is contained in:
Your Name
@@ -0,0 +1,192 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import secrets
|
||||
from datetime import datetime, timezone
|
||||
from time import struct_time
|
||||
from typing import Annotated, Any
|
||||
|
||||
import feedparser
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
from fastapi import FastAPI, Header, HTTPException
|
||||
from playwright.async_api import async_playwright
|
||||
from playwright_captcha import CaptchaType, FrameworkType, TwoCaptchaSolver
|
||||
from pydantic import BaseModel, Field, HttpUrl, SecretStr
|
||||
from twocaptcha import AsyncTwoCaptcha
|
||||
|
||||
app = FastAPI(title="Site Parser Worker", version="0.1.0")
|
||||
browser_lock = asyncio.Lock()
|
||||
|
||||
|
||||
class ParseRequest(BaseModel):
|
||||
url: HttpUrl
|
||||
config: dict[str, Any]
|
||||
rucaptcha_token: SecretStr | None = None
|
||||
browser_state: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class ParsedItem(BaseModel):
|
||||
external_id: str
|
||||
url: str
|
||||
title: str
|
||||
text: str
|
||||
html: str
|
||||
published_at: str | None
|
||||
author: str | None
|
||||
media: list[dict[str, str]] = Field(default_factory=list)
|
||||
|
||||
|
||||
def require_token(worker_token: str | None) -> None:
|
||||
expected = os.getenv("WORKER_TOKEN", "")
|
||||
if not expected:
|
||||
raise HTTPException(status_code=503, detail="WORKER_TOKEN is not configured")
|
||||
if not worker_token or not secrets.compare_digest(worker_token, expected):
|
||||
raise HTTPException(status_code=401, detail="Invalid worker token")
|
||||
|
||||
|
||||
def is_cloudflare_challenge(status: int, body: str) -> bool:
|
||||
sample = body[:20_000].lower()
|
||||
return status in {403, 429, 503} and (
|
||||
"just a moment" in sample or "cf-chl" in sample or "challenge-platform" in sample
|
||||
)
|
||||
|
||||
|
||||
def iso_date(value: struct_time | None) -> str | None:
|
||||
if not value:
|
||||
return None
|
||||
return datetime(*value[:6], tzinfo=timezone.utc).isoformat()
|
||||
|
||||
|
||||
def clean_html(value: Any) -> tuple[str, str, list[dict[str, str]]]:
|
||||
html = str(value or "").strip()
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
text = soup.get_text("\n", strip=True)
|
||||
media = [{"type": "photo", "url": str(image["src"])} for image in soup.find_all("img", src=True)]
|
||||
return html, text, media
|
||||
|
||||
|
||||
def parse_rss(xml: str, max_items: int) -> list[ParsedItem]:
|
||||
feed = feedparser.parse(xml)
|
||||
if feed.bozo and not feed.entries:
|
||||
raise ValueError(f"Invalid RSS: {feed.bozo_exception}")
|
||||
items = []
|
||||
for entry in feed.entries[:max_items]:
|
||||
title = BeautifulSoup(str(entry.get("title") or ""), "html.parser").get_text(" ", strip=True)
|
||||
html, text, media = clean_html(entry.get("description") or entry.get("summary") or "")
|
||||
for enclosure in entry.get("enclosures") or []:
|
||||
url = str(enclosure.get("href") or enclosure.get("url") or "").strip()
|
||||
media_type = str(enclosure.get("type") or "")
|
||||
if url and (not media_type or media_type.startswith("image/")):
|
||||
media.append({"type": "photo", "url": url})
|
||||
url = str(entry.get("link") or "").strip()
|
||||
items.append(
|
||||
ParsedItem(
|
||||
external_id=str(entry.get("id") or entry.get("guid") or url).strip(),
|
||||
url=url,
|
||||
title=title,
|
||||
text=text,
|
||||
html=html,
|
||||
published_at=iso_date(entry.get("published_parsed") or entry.get("updated_parsed")),
|
||||
author=str(entry.get("author") or "").strip() or None,
|
||||
media=list({item["url"]: item for item in media}.values()),
|
||||
)
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
async def fetch_in_browser(
|
||||
url: str,
|
||||
rucaptcha_token: str,
|
||||
browser_state: dict[str, Any] | None,
|
||||
) -> tuple[str, dict[str, Any]]:
|
||||
# ponytail: one browser at a time; use a queue only when parallel source parsing is needed.
|
||||
async with browser_lock:
|
||||
async with async_playwright() as playwright:
|
||||
browser = await playwright.chromium.launch(channel="chrome", headless=False)
|
||||
context = await browser.new_context(storage_state=browser_state) if browser_state else await browser.new_context()
|
||||
page = await context.new_page()
|
||||
async with TwoCaptchaSolver(
|
||||
framework=FrameworkType.PLAYWRIGHT,
|
||||
page=page,
|
||||
async_two_captcha_client=AsyncTwoCaptcha(rucaptcha_token),
|
||||
max_attempts=1,
|
||||
) as solver:
|
||||
response = await page.goto(url, wait_until="domcontentloaded", timeout=60_000)
|
||||
if response and response.status in {403, 429, 503} and "Just a moment" in await page.title():
|
||||
await solver.solve_captcha(
|
||||
captcha_container=page,
|
||||
captcha_type=CaptchaType.CLOUDFLARE_INTERSTITIAL,
|
||||
)
|
||||
try:
|
||||
await page.locator("pre").wait_for(state="visible", timeout=60_000)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"RSS did not load after Cloudflare challenge: {await page.title()}") from exc
|
||||
xml = await page.locator("pre").inner_text()
|
||||
state = await context.storage_state()
|
||||
await browser.close()
|
||||
return xml, state
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
async def health() -> dict[str, bool]:
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@app.post("/v1/parse")
|
||||
async def parse_source(
|
||||
request: ParseRequest,
|
||||
worker_token: Annotated[str | None, Header(alias="X-Worker-Token")] = None,
|
||||
) -> dict[str, Any]:
|
||||
require_token(worker_token)
|
||||
config = request.config
|
||||
if not config:
|
||||
raise HTTPException(status_code=422, detail="config is required and cannot be empty")
|
||||
if config.get("format") != "rss":
|
||||
raise HTTPException(status_code=422, detail="Only config.format=rss is supported")
|
||||
access = str(config.get("access") or "auto")
|
||||
if access not in {"auto", "http", "cloudflare"}:
|
||||
raise HTTPException(status_code=422, detail="config.access must be auto, http or cloudflare")
|
||||
try:
|
||||
max_items = int(config.get("max_items", 20))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise HTTPException(status_code=422, detail="config.max_items must be an integer") from exc
|
||||
if not 1 <= max_items <= 100:
|
||||
raise HTTPException(status_code=422, detail="config.max_items must be between 1 and 100")
|
||||
url = str(request.url)
|
||||
xml = ""
|
||||
state = request.browser_state
|
||||
fetched_via = "http"
|
||||
if access != "cloudflare":
|
||||
try:
|
||||
async with httpx.AsyncClient(follow_redirects=True, timeout=30) as client:
|
||||
response = await client.get(url, headers={"User-Agent": "Mozilla/5.0 SiteParser/0.1"})
|
||||
except httpx.HTTPError as exc:
|
||||
raise HTTPException(status_code=502, detail=f"Source request failed: {exc}") from exc
|
||||
if not is_cloudflare_challenge(response.status_code, response.text):
|
||||
try:
|
||||
response.raise_for_status()
|
||||
except httpx.HTTPStatusError as exc:
|
||||
raise HTTPException(status_code=502, detail=f"Source returned HTTP {response.status_code}") from exc
|
||||
xml = response.text
|
||||
elif access == "http":
|
||||
raise HTTPException(status_code=502, detail="Cloudflare challenge received in http-only mode")
|
||||
if not xml:
|
||||
if not request.rucaptcha_token:
|
||||
raise HTTPException(status_code=422, detail="rucaptcha_token is required for Cloudflare")
|
||||
fetched_via = "cloudflare"
|
||||
try:
|
||||
xml, state = await fetch_in_browser(url, request.rucaptcha_token.get_secret_value(), state)
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=502, detail=str(exc)) from exc
|
||||
try:
|
||||
items = parse_rss(xml, max_items)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=502, detail=str(exc)) from exc
|
||||
return {
|
||||
"source_url": url,
|
||||
"fetched_via": fetched_via,
|
||||
"items": [item.model_dump() for item in items],
|
||||
"browser_state": state,
|
||||
}
|
||||
Reference in New Issue
Block a user