feat: add external site parser worker

site_parser_worker/Dockerfile

@@ -0,0 +1,11 @@
+FROM mcr.microsoft.com/playwright/python:v1.62.0-noble
+
+WORKDIR /app
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+RUN playwright install chrome
+COPY app.py .
+
+ENV PYTHONUNBUFFERED=1
+EXPOSE 8080
+CMD ["bash", "-lc", "Xvfb :99 -screen 0 1280x1024x24 -nolisten tcp & export DISPLAY=:99; sleep 1; exec uvicorn app:app --host 0.0.0.0 --port 8080"]

site_parser_worker/app.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+import asyncio
+import os
+import secrets
+from datetime import datetime, timezone
+from time import struct_time
+from typing import Annotated, Any
+
+import feedparser
+import httpx
+from bs4 import BeautifulSoup
+from fastapi import FastAPI, Header, HTTPException
+from playwright.async_api import async_playwright
+from playwright_captcha import CaptchaType, FrameworkType, TwoCaptchaSolver
+from pydantic import BaseModel, Field, HttpUrl, SecretStr
+from twocaptcha import AsyncTwoCaptcha
+
+app = FastAPI(title="Site Parser Worker", version="0.1.0")
+browser_lock = asyncio.Lock()
+
+
+class ParseRequest(BaseModel):
+    url: HttpUrl
+    config: dict[str, Any]
+    rucaptcha_token: SecretStr | None = None
+    browser_state: dict[str, Any] | None = None
+
+
+class ParsedItem(BaseModel):
+    external_id: str
+    url: str
+    title: str
+    text: str
+    html: str
+    published_at: str | None
+    author: str | None
+    media: list[dict[str, str]] = Field(default_factory=list)
+
+
+def require_token(worker_token: str | None) -> None:
+    expected = os.getenv("WORKER_TOKEN", "")
+    if not expected:
+        raise HTTPException(status_code=503, detail="WORKER_TOKEN is not configured")
+    if not worker_token or not secrets.compare_digest(worker_token, expected):
+        raise HTTPException(status_code=401, detail="Invalid worker token")
+
+
+def is_cloudflare_challenge(status: int, body: str) -> bool:
+    sample = body[:20_000].lower()
+    return status in {403, 429, 503} and (
+        "just a moment" in sample or "cf-chl" in sample or "challenge-platform" in sample
+    )
+
+
+def iso_date(value: struct_time | None) -> str | None:
+    if not value:
+        return None
+    return datetime(*value[:6], tzinfo=timezone.utc).isoformat()
+
+
+def clean_html(value: Any) -> tuple[str, str, list[dict[str, str]]]:
+    html = str(value or "").strip()
+    soup = BeautifulSoup(html, "html.parser")
+    text = soup.get_text("\n", strip=True)
+    media = [{"type": "photo", "url": str(image["src"])} for image in soup.find_all("img", src=True)]
+    return html, text, media
+
+
+def parse_rss(xml: str, max_items: int) -> list[ParsedItem]:
+    feed = feedparser.parse(xml)
+    if feed.bozo and not feed.entries:
+        raise ValueError(f"Invalid RSS: {feed.bozo_exception}")
+    items = []
+    for entry in feed.entries[:max_items]:
+        title = BeautifulSoup(str(entry.get("title") or ""), "html.parser").get_text(" ", strip=True)
+        html, text, media = clean_html(entry.get("description") or entry.get("summary") or "")
+        for enclosure in entry.get("enclosures") or []:
+            url = str(enclosure.get("href") or enclosure.get("url") or "").strip()
+            media_type = str(enclosure.get("type") or "")
+            if url and (not media_type or media_type.startswith("image/")):
+                media.append({"type": "photo", "url": url})
+        url = str(entry.get("link") or "").strip()
+        items.append(
+            ParsedItem(
+                external_id=str(entry.get("id") or entry.get("guid") or url).strip(),
+                url=url,
+                title=title,
+                text=text,
+                html=html,
+                published_at=iso_date(entry.get("published_parsed") or entry.get("updated_parsed")),
+                author=str(entry.get("author") or "").strip() or None,
+                media=list({item["url"]: item for item in media}.values()),
+            )
+        )
+    return items
+
+
+async def fetch_in_browser(
+    url: str,
+    rucaptcha_token: str,
+    browser_state: dict[str, Any] | None,
+) -> tuple[str, dict[str, Any]]:
+    # ponytail: one browser at a time; use a queue only when parallel source parsing is needed.
+    async with browser_lock:
+        async with async_playwright() as playwright:
+            browser = await playwright.chromium.launch(channel="chrome", headless=False)
+            context = await browser.new_context(storage_state=browser_state) if browser_state else await browser.new_context()
+            page = await context.new_page()
+            async with TwoCaptchaSolver(
+                framework=FrameworkType.PLAYWRIGHT,
+                page=page,
+                async_two_captcha_client=AsyncTwoCaptcha(rucaptcha_token),
+                max_attempts=1,
+            ) as solver:
+                response = await page.goto(url, wait_until="domcontentloaded", timeout=60_000)
+                if response and response.status in {403, 429, 503} and "Just a moment" in await page.title():
+                    await solver.solve_captcha(
+                        captcha_container=page,
+                        captcha_type=CaptchaType.CLOUDFLARE_INTERSTITIAL,
+                    )
+                try:
+                    await page.locator("pre").wait_for(state="visible", timeout=60_000)
+                except Exception as exc:
+                    raise RuntimeError(f"RSS did not load after Cloudflare challenge: {await page.title()}") from exc
+                xml = await page.locator("pre").inner_text()
+                state = await context.storage_state()
+            await browser.close()
+    return xml, state
+
+
+@app.get("/health")
+async def health() -> dict[str, bool]:
+    return {"ok": True}
+
+
+@app.post("/v1/parse")
+async def parse_source(
+    request: ParseRequest,
+    worker_token: Annotated[str | None, Header(alias="X-Worker-Token")] = None,
+) -> dict[str, Any]:
+    require_token(worker_token)
+    config = request.config
+    if not config:
+        raise HTTPException(status_code=422, detail="config is required and cannot be empty")
+    if config.get("format") != "rss":
+        raise HTTPException(status_code=422, detail="Only config.format=rss is supported")
+    access = str(config.get("access") or "auto")
+    if access not in {"auto", "http", "cloudflare"}:
+        raise HTTPException(status_code=422, detail="config.access must be auto, http or cloudflare")
+    try:
+        max_items = int(config.get("max_items", 20))
+    except (TypeError, ValueError) as exc:
+        raise HTTPException(status_code=422, detail="config.max_items must be an integer") from exc
+    if not 1 <= max_items <= 100:
+        raise HTTPException(status_code=422, detail="config.max_items must be between 1 and 100")
+    url = str(request.url)
+    xml = ""
+    state = request.browser_state
+    fetched_via = "http"
+    if access != "cloudflare":
+        try:
+            async with httpx.AsyncClient(follow_redirects=True, timeout=30) as client:
+                response = await client.get(url, headers={"User-Agent": "Mozilla/5.0 SiteParser/0.1"})
+        except httpx.HTTPError as exc:
+            raise HTTPException(status_code=502, detail=f"Source request failed: {exc}") from exc
+        if not is_cloudflare_challenge(response.status_code, response.text):
+            try:
+                response.raise_for_status()
+            except httpx.HTTPStatusError as exc:
+                raise HTTPException(status_code=502, detail=f"Source returned HTTP {response.status_code}") from exc
+            xml = response.text
+        elif access == "http":
+            raise HTTPException(status_code=502, detail="Cloudflare challenge received in http-only mode")
+    if not xml:
+        if not request.rucaptcha_token:
+            raise HTTPException(status_code=422, detail="rucaptcha_token is required for Cloudflare")
+        fetched_via = "cloudflare"
+        try:
+            xml, state = await fetch_in_browser(url, request.rucaptcha_token.get_secret_value(), state)
+        except Exception as exc:
+            raise HTTPException(status_code=502, detail=str(exc)) from exc
+    try:
+        items = parse_rss(xml, max_items)
+    except ValueError as exc:
+        raise HTTPException(status_code=502, detail=str(exc)) from exc
+    return {
+        "source_url": url,
+        "fetched_via": fetched_via,
+        "items": [item.model_dump() for item in items],
+        "browser_state": state,
+    }

site_parser_worker/requirements.txt

@@ -0,0 +1,8 @@
+2captcha-python-async==1.5.1
+beautifulsoup4==4.15.0
+fastapi==0.141.1
+feedparser==6.0.14
+httpx==0.28.1
+playwright==1.62.0
+playwright-captcha==0.1.5
+uvicorn==0.52.1

site_parser_worker/test_parser.py

@@ -0,0 +1,16 @@
+from app import parse_rss
+
+
+def test_parse_rss() -> None:
+    xml = """<rss><channel><item><guid>1</guid><title>Title</title>
+    <link>https://example.test/1</link>
+    <description>&lt;p&gt;Body&lt;/p&gt;&lt;img src="https://example.test/1.jpg"&gt;</description>
+    <pubDate>Mon, 10 Aug 2026 10:00:00 +0000</pubDate></item></channel></rss>"""
+    item = parse_rss(xml, 20)[0]
+    assert item.external_id == "1"
+    assert item.text == "Body"
+    assert item.media[0]["url"] == "https://example.test/1.jpg"
+
+
+if __name__ == "__main__":
+    test_parse_rss()