Your Name
310 lines
13 KiB
Python
310 lines
13 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
import re
|
|
import secrets
|
|
from datetime import datetime, timezone
|
|
from time import struct_time
|
|
from typing import Annotated, Any
|
|
from urllib.parse import parse_qs, urljoin, urlparse
|
|
|
|
import feedparser
|
|
import httpx
|
|
from bs4 import BeautifulSoup
|
|
from fastapi import FastAPI, Header, HTTPException
|
|
from playwright.async_api import async_playwright
|
|
from playwright_captcha import CaptchaType, FrameworkType, TwoCaptchaSolver
|
|
from pydantic import BaseModel, Field, HttpUrl, SecretStr
|
|
from twocaptcha import AsyncTwoCaptcha
|
|
|
|
app = FastAPI(title="Site Parser Worker", version="0.1.0")
|
|
browser_lock = asyncio.Lock()
|
|
|
|
|
|
class ParseRequest(BaseModel):
|
|
url: HttpUrl
|
|
config: dict[str, Any]
|
|
rucaptcha_token: SecretStr | None = None
|
|
browser_state: dict[str, Any] | None = None
|
|
browser_user_agent: str | None = None
|
|
|
|
|
|
class ParsedItem(BaseModel):
|
|
external_id: str
|
|
url: str
|
|
title: str
|
|
text: str
|
|
html: str
|
|
published_at: str | None
|
|
author: str | None
|
|
media: list[dict[str, str]] = Field(default_factory=list)
|
|
|
|
|
|
def require_token(worker_token: str | None) -> None:
|
|
expected = os.getenv("WORKER_TOKEN", "")
|
|
if not expected:
|
|
raise HTTPException(status_code=503, detail="WORKER_TOKEN is not configured")
|
|
if not worker_token or not secrets.compare_digest(worker_token, expected):
|
|
raise HTTPException(status_code=401, detail="Invalid worker token")
|
|
|
|
|
|
def is_cloudflare_challenge(status: int, body: str) -> bool:
|
|
sample = body[:20_000].lower()
|
|
return status in {403, 429, 503} and (
|
|
"just a moment" in sample or "cf-chl" in sample or "challenge-platform" in sample
|
|
)
|
|
|
|
|
|
def iso_date(value: struct_time | None) -> str | None:
|
|
if not value:
|
|
return None
|
|
return datetime(*value[:6], tzinfo=timezone.utc).isoformat()
|
|
|
|
|
|
def youtube_url(value: str, base_url: str = "") -> str | None:
|
|
url = urljoin(base_url, str(value or "").strip())
|
|
parsed = urlparse(url)
|
|
host = (parsed.hostname or "").lower().removeprefix("www.").removeprefix("m.")
|
|
video_id = ""
|
|
if host == "youtu.be":
|
|
video_id = parsed.path.strip("/").split("/", 1)[0]
|
|
elif host in {"youtube.com", "youtube-nocookie.com"}:
|
|
if parsed.path == "/watch":
|
|
video_id = (parse_qs(parsed.query).get("v") or [""])[0]
|
|
elif parsed.path.startswith(("/embed/", "/shorts/", "/live/")):
|
|
video_id = parsed.path.strip("/").split("/", 1)[1]
|
|
if not re.fullmatch(r"[A-Za-z0-9_-]{6,20}", video_id):
|
|
return None
|
|
return f"https://www.youtube.com/watch?v={video_id}"
|
|
|
|
|
|
def clean_html(value: Any, base_url: str = "") -> tuple[str, str, list[dict[str, str]]]:
|
|
html = str(value or "").strip()
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
for element in soup.find_all(["script", "style", "noscript"]):
|
|
element.decompose()
|
|
text = soup.get_text("\n", strip=True)
|
|
media = [
|
|
{"type": "photo", "url": urljoin(base_url, str(image["src"]))}
|
|
for image in soup.find_all("img", src=True)
|
|
]
|
|
for element in soup.find_all(["video", "source"], src=True):
|
|
media.append({"type": "video", "url": urljoin(base_url, str(element["src"]))})
|
|
for element in soup.find_all(["iframe", "a"]):
|
|
url = youtube_url(str(element.get("src") or element.get("href") or ""), base_url)
|
|
if url:
|
|
media.append({"type": "video", "url": url})
|
|
return html, text, media
|
|
|
|
|
|
def parse_rss(xml: str, max_items: int) -> list[ParsedItem]:
|
|
feed = feedparser.parse(xml, sanitize_html=False)
|
|
if feed.bozo and not feed.entries:
|
|
raise ValueError(f"Invalid RSS: {feed.bozo_exception}")
|
|
items = []
|
|
for entry in feed.entries[:max_items]:
|
|
title = BeautifulSoup(str(entry.get("title") or ""), "html.parser").get_text(" ", strip=True)
|
|
url = str(entry.get("link") or "").strip()
|
|
content = (entry.get("content") or [{}])[0].get("value") or entry.get("description") or entry.get("summary") or ""
|
|
html, text, media = clean_html(content, url)
|
|
for enclosure in entry.get("enclosures") or []:
|
|
enclosure_url = urljoin(url, str(enclosure.get("href") or enclosure.get("url") or "").strip())
|
|
media_type = str(enclosure.get("type") or "")
|
|
if enclosure_url and (not media_type or media_type.startswith("image/")):
|
|
media.append({"type": "photo", "url": enclosure_url})
|
|
elif enclosure_url and media_type.startswith("video/"):
|
|
media.append({"type": "video", "url": enclosure_url})
|
|
items.append(
|
|
ParsedItem(
|
|
external_id=str(entry.get("id") or entry.get("guid") or url).strip(),
|
|
url=url,
|
|
title=title,
|
|
text=text,
|
|
html=html,
|
|
published_at=iso_date(entry.get("published_parsed") or entry.get("updated_parsed")),
|
|
author=str(entry.get("author") or "").strip() or None,
|
|
media=list({item["url"]: item for item in media}.values()),
|
|
)
|
|
)
|
|
return items
|
|
|
|
|
|
async def fetch_in_browser(
|
|
url: str,
|
|
rucaptcha_token: str,
|
|
browser_state: dict[str, Any] | None,
|
|
browser_user_agent: str | None,
|
|
) -> tuple[str, dict[str, Any], str]:
|
|
# ponytail: one browser at a time; use a queue only when parallel source parsing is needed.
|
|
async with browser_lock:
|
|
async with async_playwright() as playwright:
|
|
browser = await playwright.chromium.launch(channel="chrome", headless=False)
|
|
context = await browser.new_context(
|
|
storage_state=browser_state,
|
|
user_agent=browser_user_agent,
|
|
)
|
|
page = await context.new_page()
|
|
async with TwoCaptchaSolver(
|
|
framework=FrameworkType.PLAYWRIGHT,
|
|
page=page,
|
|
async_two_captcha_client=AsyncTwoCaptcha(rucaptcha_token),
|
|
max_attempts=1,
|
|
) as solver:
|
|
await page.goto(url, wait_until="domcontentloaded", timeout=60_000)
|
|
if "just a moment" in (await page.title()).lower():
|
|
await solver.solve_captcha(
|
|
captcha_container=page,
|
|
captcha_type=CaptchaType.CLOUDFLARE_INTERSTITIAL,
|
|
)
|
|
try:
|
|
await page.locator("pre").wait_for(state="visible", timeout=60_000)
|
|
except Exception as exc:
|
|
raise RuntimeError(f"RSS did not load after Cloudflare challenge: {await page.title()}") from exc
|
|
xml = await page.locator("pre").inner_text()
|
|
state = await context.storage_state()
|
|
user_agent = await page.evaluate("navigator.userAgent")
|
|
await browser.close()
|
|
return xml, state, user_agent
|
|
|
|
|
|
async def enrich_items_in_browser(
|
|
items: list[ParsedItem],
|
|
rucaptcha_token: str,
|
|
browser_state: dict[str, Any] | None,
|
|
browser_user_agent: str | None,
|
|
content_selector: str,
|
|
text_selector: str,
|
|
) -> tuple[list[ParsedItem], dict[str, Any], str]:
|
|
async with browser_lock:
|
|
async with async_playwright() as playwright:
|
|
browser = await playwright.chromium.launch(channel="chrome", headless=False)
|
|
context = await browser.new_context(
|
|
storage_state=browser_state,
|
|
user_agent=browser_user_agent,
|
|
)
|
|
page = await context.new_page()
|
|
async with TwoCaptchaSolver(
|
|
framework=FrameworkType.PLAYWRIGHT,
|
|
page=page,
|
|
async_two_captcha_client=AsyncTwoCaptcha(rucaptcha_token),
|
|
max_attempts=1,
|
|
) as solver:
|
|
for item in items:
|
|
if item.text and item.media:
|
|
continue
|
|
await page.goto(item.url, wait_until="domcontentloaded", timeout=60_000)
|
|
if "just a moment" in (await page.title()).lower():
|
|
await solver.solve_captcha(
|
|
captcha_container=page,
|
|
captcha_type=CaptchaType.CLOUDFLARE_INTERSTITIAL,
|
|
)
|
|
try:
|
|
content = page.locator(content_selector).first
|
|
await content.wait_for(state="visible", timeout=60_000)
|
|
except Exception as exc:
|
|
raise RuntimeError(f"Item content not found: {item.url} ({content_selector})") from exc
|
|
html, text, media = clean_html(await content.inner_html(), item.url)
|
|
if text_selector:
|
|
text_content = content.locator(text_selector).first
|
|
text = clean_html(await text_content.inner_html())[1] if await text_content.count() else ""
|
|
item.html = html
|
|
item.text = text
|
|
item.media = list({entry["url"]: entry for entry in [*item.media, *media]}.values())
|
|
state = await context.storage_state()
|
|
user_agent = await page.evaluate("navigator.userAgent")
|
|
await browser.close()
|
|
return items, state, user_agent
|
|
|
|
|
|
@app.get("/health")
|
|
async def health() -> dict[str, bool]:
|
|
return {"ok": True}
|
|
|
|
|
|
@app.post("/v1/parse")
|
|
async def parse_source(
|
|
request: ParseRequest,
|
|
worker_token: Annotated[str | None, Header(alias="X-Worker-Token")] = None,
|
|
) -> dict[str, Any]:
|
|
require_token(worker_token)
|
|
config = request.config
|
|
if not config:
|
|
raise HTTPException(status_code=422, detail="config is required and cannot be empty")
|
|
if config.get("format") != "rss":
|
|
raise HTTPException(status_code=422, detail="Only config.format=rss is supported")
|
|
access = str(config.get("access") or "auto")
|
|
if access not in {"auto", "http", "cloudflare"}:
|
|
raise HTTPException(status_code=422, detail="config.access must be auto, http or cloudflare")
|
|
try:
|
|
max_items = int(config.get("max_items", 20))
|
|
except (TypeError, ValueError) as exc:
|
|
raise HTTPException(status_code=422, detail="config.max_items must be an integer") from exc
|
|
if not 1 <= max_items <= 100:
|
|
raise HTTPException(status_code=422, detail="config.max_items must be between 1 and 100")
|
|
follow_links = config.get("follow_links", False)
|
|
if not isinstance(follow_links, bool):
|
|
raise HTTPException(status_code=422, detail="config.follow_links must be true or false")
|
|
content_selector = str(config.get("content_selector") or "").strip()
|
|
text_selector = str(config.get("text_selector") or "").strip()
|
|
if follow_links and not content_selector:
|
|
raise HTTPException(status_code=422, detail="config.content_selector is required when follow_links=true")
|
|
url = str(request.url)
|
|
xml = ""
|
|
state = request.browser_state
|
|
browser_user_agent = request.browser_user_agent
|
|
fetched_via = "http"
|
|
if access != "cloudflare":
|
|
try:
|
|
async with httpx.AsyncClient(follow_redirects=True, timeout=30) as client:
|
|
response = await client.get(url, headers={"User-Agent": "Mozilla/5.0 SiteParser/0.1"})
|
|
except httpx.HTTPError as exc:
|
|
raise HTTPException(status_code=502, detail=f"Source request failed: {exc}") from exc
|
|
if not is_cloudflare_challenge(response.status_code, response.text):
|
|
try:
|
|
response.raise_for_status()
|
|
except httpx.HTTPStatusError as exc:
|
|
raise HTTPException(status_code=502, detail=f"Source returned HTTP {response.status_code}") from exc
|
|
xml = response.text
|
|
elif access == "http":
|
|
raise HTTPException(status_code=502, detail="Cloudflare challenge received in http-only mode")
|
|
if not xml:
|
|
if not request.rucaptcha_token:
|
|
raise HTTPException(status_code=422, detail="rucaptcha_token is required for Cloudflare")
|
|
fetched_via = "cloudflare"
|
|
try:
|
|
xml, state, browser_user_agent = await fetch_in_browser(
|
|
url,
|
|
request.rucaptcha_token.get_secret_value(),
|
|
state,
|
|
browser_user_agent,
|
|
)
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=502, detail=str(exc)) from exc
|
|
try:
|
|
items = parse_rss(xml, max_items)
|
|
except ValueError as exc:
|
|
raise HTTPException(status_code=502, detail=str(exc)) from exc
|
|
if follow_links:
|
|
if not request.rucaptcha_token:
|
|
raise HTTPException(status_code=422, detail="rucaptcha_token is required when follow_links=true")
|
|
try:
|
|
items, state, browser_user_agent = await enrich_items_in_browser(
|
|
items,
|
|
request.rucaptcha_token.get_secret_value(),
|
|
state,
|
|
browser_user_agent,
|
|
content_selector,
|
|
text_selector,
|
|
)
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=502, detail=str(exc)) from exc
|
|
fetched_via += "+item-pages"
|
|
return {
|
|
"source_url": url,
|
|
"fetched_via": fetched_via,
|
|
"items": [item.model_dump() for item in items],
|
|
"browser_state": state,
|
|
"browser_user_agent": browser_user_agent,
|
|
}
|