feat: add external site parser worker

src/vk_parser_app/source_adapters.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any
+
+import aiohttp
+
+from .constants import PLATFORM_SITE, PLATFORM_VK
+
+
+@dataclass
+class SourceMedia:
+    url: str
+    media_type: str = "photo"
+
+
+@dataclass
+class SourceItem:
+    external_id: str
+    url: str
+    text: str
+    posted_at: datetime
+    media: list[SourceMedia] = field(default_factory=list)
+    raw: dict[str, Any] = field(default_factory=dict)
+
+
+def validate_source_config(platform: str, config: dict[str, Any]) -> None:
+    if platform == PLATFORM_VK:
+        return
+    if platform != PLATFORM_SITE:
+        raise ValueError("Неподдерживаемая площадка")
+    if not config:
+        raise ValueError("Для сайта нужен конфиг JSON")
+    if config.get("format") != "rss":
+        raise ValueError('Сейчас поддерживается только "format": "rss"')
+    if str(config.get("access") or "auto") not in {"auto", "http", "cloudflare"}:
+        raise ValueError('access должен быть "auto", "http" или "cloudflare"')
+    try:
+        max_items = int(config.get("max_items", 20))
+    except (TypeError, ValueError) as exc:
+        raise ValueError("max_items должен быть целым числом") from exc
+    if not 1 <= max_items <= 100:
+        raise ValueError("max_items должен быть от 1 до 100")
+
+
+def _posted_at(value: Any) -> datetime:
+    try:
+        parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+    except (TypeError, ValueError):
+        return datetime.now(timezone.utc)
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=timezone.utc)
+    return parsed.astimezone(timezone.utc)
+
+
+class SiteParserClient:
+    def __init__(
+        self,
+        session: aiohttp.ClientSession,
+        base_url: str,
+        token: str,
+        rucaptcha_token: str,
+        timeout_sec: int,
+    ) -> None:
+        self.session = session
+        self.base_url = base_url.rstrip("/")
+        self.token = token
+        self.rucaptcha_token = rucaptcha_token
+        self.timeout = aiohttp.ClientTimeout(total=max(10, timeout_sec))
+
+    async def fetch(self, source: dict) -> tuple[list[SourceItem], dict[str, Any] | None]:
+        if not self.base_url or not self.token:
+            raise RuntimeError("Site Parser URL или токен не настроены")
+        config = dict(source.get("settings_json") or {})
+        validate_source_config(PLATFORM_SITE, config)
+        runtime_state = dict(source.get("runtime_state_json") or {})
+        payload = {
+            "url": source["url"],
+            "config": config,
+            "rucaptcha_token": self.rucaptcha_token or None,
+            "browser_state": runtime_state.get("browser_state"),
+        }
+        try:
+            async with self.session.post(
+                f"{self.base_url}/v1/parse",
+                json=payload,
+                headers={"X-Worker-Token": self.token},
+                timeout=self.timeout,
+            ) as response:
+                data = await response.json(content_type=None)
+                if response.status >= 400:
+                    raise RuntimeError(f"Site Parser HTTP {response.status}: {data.get('detail', data)}")
+        except TimeoutError as exc:
+            raise RuntimeError(f"Site Parser превысил таймаут {int(self.timeout.total)} сек") from exc
+        except aiohttp.ClientError as exc:
+            raise RuntimeError(f"Site Parser недоступен: {exc}") from exc
+
+        items = []
+        for raw in data.get("items") or []:
+            title = str(raw.get("title") or "").strip()
+            body = str(raw.get("text") or "").strip()
+            text = "\n\n".join(part for part in (title, body) if part)
+            url = str(raw.get("url") or source["url"]).strip()
+            external_id = str(raw.get("external_id") or url).strip()
+            if not external_id:
+                continue
+            media = [
+                SourceMedia(str(item["url"]), str(item.get("type") or "photo"))
+                for item in raw.get("media") or []
+                if isinstance(item, dict) and item.get("url")
+            ]
+            items.append(SourceItem(external_id, url, text, _posted_at(raw.get("published_at")), media, raw))
+        state = data.get("browser_state")
+        return items, ({"browser_state": state} if isinstance(state, dict) else None)