from __future__ import annotations import json from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Any import aiohttp from .constants import PLATFORM_SITE, PLATFORM_VK def json_object(value: Any) -> dict[str, Any]: if isinstance(value, dict): return dict(value) if isinstance(value, str): try: parsed = json.loads(value) except json.JSONDecodeError: return {} return parsed if isinstance(parsed, dict) else {} return {} @dataclass class SourceMedia: url: str media_type: str = "photo" @dataclass class SourceItem: external_id: str url: str text: str posted_at: datetime media: list[SourceMedia] = field(default_factory=list) raw: dict[str, Any] = field(default_factory=dict) @property def body_text(self) -> str: return str(self.raw.get("text") or "").strip() def validate_source_config(platform: str, config: dict[str, Any]) -> None: if platform == PLATFORM_VK: return if platform != PLATFORM_SITE: raise ValueError("Неподдерживаемая площадка") if not config: raise ValueError("Для сайта нужен конфиг JSON") if config.get("format") != "rss": raise ValueError('Сейчас поддерживается только "format": "rss"') if str(config.get("access") or "auto") not in {"auto", "http", "cloudflare"}: raise ValueError('access должен быть "auto", "http" или "cloudflare"') try: max_items = int(config.get("max_items", 20)) except (TypeError, ValueError) as exc: raise ValueError("max_items должен быть целым числом") from exc if not 1 <= max_items <= 100: raise ValueError("max_items должен быть от 1 до 100") follow_links = config.get("follow_links", False) if not isinstance(follow_links, bool): raise ValueError("follow_links должен быть true или false") if follow_links and not str(config.get("content_selector") or "").strip(): raise ValueError("При follow_links=true нужен content_selector") try: min_text_length = int(config.get("min_text_length", 0)) except (TypeError, ValueError) as exc: raise ValueError("min_text_length должен быть целым числом") from exc if not 0 <= min_text_length <= 100_000: raise ValueError("min_text_length должен быть от 0 до 100000") def _posted_at(value: Any) -> datetime: try: parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) except (TypeError, ValueError): return datetime.now(timezone.utc) if parsed.tzinfo is None: parsed = parsed.replace(tzinfo=timezone.utc) return parsed.astimezone(timezone.utc) class SiteParserClient: def __init__( self, session: aiohttp.ClientSession, base_url: str, token: str, rucaptcha_token: str, timeout_sec: int, ) -> None: self.session = session self.base_url = base_url.rstrip("/") self.token = token self.rucaptcha_token = rucaptcha_token self.timeout = aiohttp.ClientTimeout(total=max(10, timeout_sec)) async def fetch(self, source: dict) -> tuple[list[SourceItem], dict[str, Any] | None]: if not self.base_url or not self.token: raise RuntimeError("Site Parser URL или токен не настроены") config = json_object(source.get("settings_json")) validate_source_config(PLATFORM_SITE, config) runtime_state = json_object(source.get("runtime_state_json")) payload = { "url": source["url"], "config": config, "rucaptcha_token": self.rucaptcha_token or None, "browser_state": runtime_state.get("browser_state"), "browser_user_agent": runtime_state.get("browser_user_agent"), } try: async with self.session.post( f"{self.base_url}/v1/parse", json=payload, headers={"X-Worker-Token": self.token}, timeout=self.timeout, ) as response: data = await response.json(content_type=None) if response.status >= 400: raise RuntimeError(f"Site Parser HTTP {response.status}: {data.get('detail', data)}") except TimeoutError as exc: raise RuntimeError(f"Site Parser превысил таймаут {int(self.timeout.total)} сек") from exc except aiohttp.ClientError as exc: raise RuntimeError(f"Site Parser недоступен: {exc}") from exc items = [] for raw in data.get("items") or []: title = str(raw.get("title") or "").strip() body = str(raw.get("text") or "").strip() body_starts_with_title = title and ( body.casefold() == title.casefold() or body.casefold().startswith(f"{title}\n".casefold()) ) text = body if body_starts_with_title else "\n\n".join(part for part in (title, body) if part) url = str(raw.get("url") or source["url"]).strip() external_id = str(raw.get("external_id") or url).strip() if not external_id: continue media = [ SourceMedia(str(item["url"]), str(item.get("type") or "photo")) for item in raw.get("media") or [] if isinstance(item, dict) and item.get("url") ] items.append(SourceItem(external_id, url, text, _posted_at(raw.get("published_at")), media, raw)) state = data.get("browser_state") return items, ( { "browser_state": state, "browser_user_agent": str(data.get("browser_user_agent") or "") or None, } if isinstance(state, dict) else None )