feat: ingest site videos through raw pipeline

src/vk_parser_app/source_adapters.py

@@ -55,6 +55,13 @@ def validate_source_config(platform: str, config: dict[str, Any]) -> None:
         raise ValueError("max_items должен быть целым числом") from exc
     if not 1 <= max_items <= 100:
         raise ValueError("max_items должен быть от 1 до 100")
+    follow_links = config.get("follow_links", False)
+    if not isinstance(follow_links, bool):
+        raise ValueError("follow_links должен быть true или false")
+    if follow_links and not str(config.get("content_selector") or "").strip():
+        raise ValueError("При follow_links=true нужен content_selector")
+
+
 def _posted_at(value: Any) -> datetime:
     try:
         parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
@@ -111,7 +118,11 @@ class SiteParserClient:
         for raw in data.get("items") or []:
             title = str(raw.get("title") or "").strip()
             body = str(raw.get("text") or "").strip()
-            text = "\n\n".join(part for part in (title, body) if part)
+            body_starts_with_title = title and (
+                body.casefold() == title.casefold()
+                or body.casefold().startswith(f"{title}\n".casefold())
+            )
+            text = body if body_starts_with_title else "\n\n".join(part for part in (title, body) if part)
             url = str(raw.get("url") or source["url"]).strip()
             external_id = str(raw.get("external_id") or url).strip()
             if not external_id:

src/vk_parser_app/templates/source_form.html

@@ -52,6 +52,7 @@
   "max_items": 20
 }</pre>
             <p class="mt-2"><code>access</code>: <code>auto</code> сначала пробует обычный запрос и при Cloudflare использует RuCaptcha; <code>http</code> запрещает браузер; <code>cloudflare</code> сразу запускает браузер.</p>
+            <p class="mt-2">Если RSS не содержит текст или медиа, добавьте <code>"follow_links": true</code> и CSS-селектор содержимого страницы, например <code>"content_selector": "article.full"</code>.</p>
           </details>
         </div>
       </div>

src/vk_parser_app/workers/vk_storage_uploader.py

@@ -7,6 +7,7 @@ import re
 import sys
 import time
 from pathlib import Path
+from urllib.parse import urlparse
 
 import aiohttp
 from aiogram import Bot
@@ -52,6 +53,20 @@ def parse_vk_video_url(vk_url: str) -> tuple[int, int] | None:
     return int(match.group(1)), int(match.group(2))
 
 
+def video_provider(url: str) -> str | None:
+    host = (urlparse(str(url or "")).hostname or "").lower().removeprefix("www.").removeprefix("m.")
+    if host == "vk.com" and parse_vk_video_url(url):
+        return "vk"
+    if host in {"youtube.com", "youtube-nocookie.com", "youtu.be"}:
+        return "youtube"
+    return None
+
+
+def remove_download_files(output_path: str) -> None:
+    for path in TMP_DIR.glob(f"{Path(output_path).name}*"):
+        path.unlink(missing_ok=True)
+
+
 def split_message_chunks(text: str, limit: int) -> list[str]:
     text = str(text or "").strip()
     if not text:
@@ -320,37 +335,43 @@ class TelegramStorageUploader:
             error[:1000],
         )
 
-    async def download_video(self, vk_url: str, output_path: str) -> dict | None:
-        parsed = parse_vk_video_url(vk_url)
-        if not parsed:
+    async def download_video(self, video_url: str, output_path: str) -> dict | None:
+        provider = video_provider(video_url)
+        if not provider:
             return None
-        owner_id, video_id = parsed
         max_size = self.video_max_size_mb * 1024 * 1024
-        netrc_path = f"{output_path}.netrc"
-        fd = os.open(netrc_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
-        with os.fdopen(fd, "w", encoding="utf-8") as fh:
-            fh.write(f"machine vk.com login vk_token password {settings.vk_access_token}\n")
+        target_url = video_url
         cmd = [
             sys.executable,
             "-m",
             "yt_dlp",
-            "--netrc-location",
-            netrc_path,
-            f"https://vk.com/video{owner_id}_{video_id}",
-            "-o",
-            output_path,
+        ]
+        netrc_path = None
+        if provider == "vk":
+            owner_id, video_id = parse_vk_video_url(video_url) or (0, 0)
+            target_url = f"https://vk.com/video{owner_id}_{video_id}"
+            netrc_path = f"{output_path}.netrc"
+            fd = os.open(netrc_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+            with os.fdopen(fd, "w", encoding="utf-8") as fh:
+                fh.write(f"machine vk.com login vk_token password {settings.vk_access_token}\n")
+            cmd.extend(["--netrc-location", netrc_path])
+        cmd.extend([
+            target_url,
+            "-o", output_path,
             "--no-playlist",
-            "-f",
-            (
+            "--match-filter", f"duration <= {self.video_max_duration_sec}",
+            "--merge-output-format", "mp4",
+            "-f", (
                 f"best[height<={self.video_max_height}][filesize<{max_size}]"
                 f"/best[height<={self.video_max_height}]"
                 f"/bestvideo[height<={self.video_max_height}][filesize<{max_size}]+bestaudio/best"
                 f"/bestvideo[height<={self.video_max_height}]+bestaudio/best"
                 f"/best[filesize<{max_size}]"
             ),
-            "--quiet",
-            "--no-warnings",
-        ]
+            "--quiet", "--no-warnings",
+        ])
+        proc = None
+        stderr = b""
         try:
             proc = await asyncio.create_subprocess_exec(
                 *cmd,
@@ -359,20 +380,24 @@ class TelegramStorageUploader:
             )
             _, stderr = await asyncio.wait_for(proc.communicate(), timeout=self.yt_dlp_timeout_sec)
         except asyncio.TimeoutError:
-            proc.kill()
-            await proc.communicate()
+            if proc:
+                proc.kill()
+                await proc.communicate()
+            remove_download_files(output_path)
             return {"error": "video download timeout", "permanent": False}
         finally:
-            try:
-                os.unlink(netrc_path)
-            except FileNotFoundError:
-                pass
+            if netrc_path:
+                Path(netrc_path).unlink(missing_ok=True)
         if proc.returncode != 0 or not os.path.exists(output_path):
             err = (stderr or b"").decode("utf-8", errors="ignore").lower()
-            permanent = any(marker in err for marker in ("removed", "unavailable", "private", "access denied"))
+            permanent = any(marker in err for marker in (
+                "removed", "unavailable", "private", "access denied", "does not pass filter", "sign in",
+            ))
+            remove_download_files(output_path)
             return {"error": "video unavailable or download failed", "permanent": permanent}
         size = os.path.getsize(output_path)
         if size > max_size:
+            remove_download_files(output_path)
             return {"error": "video too large", "permanent": True}
         return {"path": output_path, "size_bytes": size}
 
@@ -581,8 +606,7 @@ class TelegramStorageUploader:
                 tmp_path = item.get("tmp_path")
                 if tmp_path:
                     try:
-                        if os.path.exists(tmp_path):
-                            os.remove(tmp_path)
+                        remove_download_files(tmp_path)
                     except Exception:
                         pass
             if not blocking_error.startswith("media still pending:"):
@@ -600,8 +624,7 @@ class TelegramStorageUploader:
                 tmp_path = item.get("tmp_path")
                 if tmp_path:
                     try:
-                        if os.path.exists(tmp_path):
-                            os.remove(tmp_path)
+                        remove_download_files(tmp_path)
                     except Exception:
                         pass
         await self.mark_post_ready(raw_post_id, message_ids, meta_message_id)