feat: ingest site videos through raw pipeline
This commit is contained in:
@@ -55,6 +55,13 @@ def validate_source_config(platform: str, config: dict[str, Any]) -> None:
|
||||
raise ValueError("max_items должен быть целым числом") from exc
|
||||
if not 1 <= max_items <= 100:
|
||||
raise ValueError("max_items должен быть от 1 до 100")
|
||||
follow_links = config.get("follow_links", False)
|
||||
if not isinstance(follow_links, bool):
|
||||
raise ValueError("follow_links должен быть true или false")
|
||||
if follow_links and not str(config.get("content_selector") or "").strip():
|
||||
raise ValueError("При follow_links=true нужен content_selector")
|
||||
|
||||
|
||||
def _posted_at(value: Any) -> datetime:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
||||
@@ -111,7 +118,11 @@ class SiteParserClient:
|
||||
for raw in data.get("items") or []:
|
||||
title = str(raw.get("title") or "").strip()
|
||||
body = str(raw.get("text") or "").strip()
|
||||
text = "\n\n".join(part for part in (title, body) if part)
|
||||
body_starts_with_title = title and (
|
||||
body.casefold() == title.casefold()
|
||||
or body.casefold().startswith(f"{title}\n".casefold())
|
||||
)
|
||||
text = body if body_starts_with_title else "\n\n".join(part for part in (title, body) if part)
|
||||
url = str(raw.get("url") or source["url"]).strip()
|
||||
external_id = str(raw.get("external_id") or url).strip()
|
||||
if not external_id:
|
||||
|
||||
Reference in New Issue
Block a user