fix: reuse Cloudflare session for site media

src/vk_parser_app/source_adapters.py

@@ -108,6 +108,7 @@ class SiteParserClient:
             "config": config,
             "rucaptcha_token": self.rucaptcha_token or None,
             "browser_state": runtime_state.get("browser_state"),
+            "browser_user_agent": runtime_state.get("browser_user_agent"),
         }
         try:
             async with self.session.post(
@@ -144,4 +145,11 @@ class SiteParserClient:
             ]
             items.append(SourceItem(external_id, url, text, _posted_at(raw.get("published_at")), media, raw))
         state = data.get("browser_state")
-        return items, ({"browser_state": state} if isinstance(state, dict) else None)
+        return items, (
+            {
+                "browser_state": state,
+                "browser_user_agent": str(data.get("browser_user_agent") or "") or None,
+            }
+            if isinstance(state, dict)
+            else None
+        )

src/vk_parser_app/workers/media_uploader.py

@@ -30,6 +30,7 @@ from ..constants import (
 from ..db import fetch_float_setting, fetch_int_setting, fetch_setting, get_pool
 from ..heartbeat import HeartbeatReporter
 from ..jobs import ack_done, ack_retry, claim_job, is_worker_enabled, recover_stale_jobs
+from ..source_adapters import json_object
 
 TMP_DIR = Path("/tmp")
 TMP_PREFIX = "vkparser_tg_media_"
@@ -210,10 +211,12 @@ class MediaUploader:
     async def load_media(self, raw_post_id: int) -> list[dict]:
         rows = await self.pool.fetch(
             """
-            SELECT *
-            FROM raw_post_media
-            WHERE raw_post_id=$1
-            ORDER BY sort_order ASC, id ASC
+            SELECT m.*, s.url AS source_url, s.runtime_state_json AS source_runtime_state
+            FROM raw_post_media m
+            JOIN raw_posts rp ON rp.id=m.raw_post_id
+            JOIN sources s ON s.id=rp.source_id
+            WHERE m.raw_post_id=$1
+            ORDER BY m.sort_order ASC, m.id ASC
             """,
             raw_post_id,
         )
@@ -277,9 +280,35 @@ class MediaUploader:
             return f"media not uploaded: {details}"
         return None
 
-    async def download_bytes(self, session: aiohttp.ClientSession, url: str) -> bytes | None:
+    def site_request_options(self, media: dict) -> dict:
+        if str(media.get("platform") or "") != "site":
+            return {}
+        state = json_object(media.get("source_runtime_state"))
+        browser_state = json_object(state.get("browser_state"))
+        cookies = {
+            str(cookie["name"]): str(cookie["value"])
+            for cookie in browser_state.get("cookies") or []
+            if isinstance(cookie, dict) and cookie.get("name") and cookie.get("value")
+        }
+        headers = {}
+        if state.get("browser_user_agent"):
+            headers["User-Agent"] = str(state["browser_user_agent"])
+        if media.get("source_url"):
+            headers["Referer"] = str(media["source_url"])
+        return {"headers": headers, "cookies": cookies}
+
+    async def download_bytes(
+        self,
+        session: aiohttp.ClientSession,
+        url: str,
+        request_options: dict | None = None,
+    ) -> bytes | None:
         try:
-            async with session.get(url, timeout=aiohttp.ClientTimeout(total=self.download_timeout_sec)) as response:
+            async with session.get(
+                url,
+                timeout=aiohttp.ClientTimeout(total=self.download_timeout_sec),
+                **(request_options or {}),
+            ) as response:
                 if response.status != 200:
                     return None
                 return await response.read()
@@ -291,10 +320,15 @@ class MediaUploader:
         session: aiohttp.ClientSession,
         url: str,
         output_path: str,
+        request_options: dict | None = None,
     ) -> dict:
         max_size = self.video_max_size_mb * 1024 * 1024
         try:
-            async with session.get(url, timeout=aiohttp.ClientTimeout(total=self.download_timeout_sec)) as response:
+            async with session.get(
+                url,
+                timeout=aiohttp.ClientTimeout(total=self.download_timeout_sec),
+                **(request_options or {}),
+            ) as response:
                 if response.status != 200:
                     return {"error": f"video returned HTTP {response.status}", "permanent": False}
                 size = 0
@@ -432,6 +466,7 @@ class MediaUploader:
                 media_id = int(item["id"])
                 media_type = str(item["media_type"])
                 url = str(item.get("original_url") or "")
+                request_options = self.site_request_options(item)
 
                 if item.get("tg_file_id"):
                     prepared.append({"media_id": media_id, "media_type": media_type, "media": item["tg_file_id"]})
@@ -440,7 +475,7 @@ class MediaUploader:
                     continue
 
                 if media_type == "photo":
-                    data = await self.download_bytes(session, url)
+                    data = await self.download_bytes(session, url, request_options)
                     if not data:
                         await self.mark_media_failed_attempt(media_id, "photo download failed")
                         continue
@@ -462,7 +497,7 @@ class MediaUploader:
                     info = (
                         await self.download_video(url, temp_path)
                         if video_provider(url)
-                        else await self.download_http_video(session, url, temp_path)
+                        else await self.download_http_video(session, url, temp_path, request_options)
                     )
                     if not info:
                         await self.mark_media_failed_attempt(media_id, "video download failed")