feat: ingest site videos through raw pipeline

This commit is contained in:
Your Name
2026-08-10 22:35:33 +05:00
parent 7889d4d376
commit faded0132b
9 changed files with 231 additions and 42 deletions
+12 -1
View File
@@ -55,6 +55,13 @@ def validate_source_config(platform: str, config: dict[str, Any]) -> None:
raise ValueError("max_items должен быть целым числом") from exc
if not 1 <= max_items <= 100:
raise ValueError("max_items должен быть от 1 до 100")
follow_links = config.get("follow_links", False)
if not isinstance(follow_links, bool):
raise ValueError("follow_links должен быть true или false")
if follow_links and not str(config.get("content_selector") or "").strip():
raise ValueError("При follow_links=true нужен content_selector")
def _posted_at(value: Any) -> datetime:
try:
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
@@ -111,7 +118,11 @@ class SiteParserClient:
for raw in data.get("items") or []:
title = str(raw.get("title") or "").strip()
body = str(raw.get("text") or "").strip()
text = "\n\n".join(part for part in (title, body) if part)
body_starts_with_title = title and (
body.casefold() == title.casefold()
or body.casefold().startswith(f"{title}\n".casefold())
)
text = body if body_starts_with_title else "\n\n".join(part for part in (title, body) if part)
url = str(raw.get("url") or source["url"]).strip()
external_id = str(raw.get("external_id") or url).strip()
if not external_id: