fix: reuse Cloudflare session for site media

site_parser_worker/app.py

@@ -27,6 +27,7 @@ class ParseRequest(BaseModel):
     config: dict[str, Any]
     rucaptcha_token: SecretStr | None = None
     browser_state: dict[str, Any] | None = None
+    browser_user_agent: str | None = None
 
 
 class ParsedItem(BaseModel):
@@ -133,12 +134,16 @@ async def fetch_in_browser(
     url: str,
     rucaptcha_token: str,
     browser_state: dict[str, Any] | None,
-) -> tuple[str, dict[str, Any]]:
+    browser_user_agent: str | None,
+) -> tuple[str, dict[str, Any], str]:
     # ponytail: one browser at a time; use a queue only when parallel source parsing is needed.
     async with browser_lock:
         async with async_playwright() as playwright:
             browser = await playwright.chromium.launch(channel="chrome", headless=False)
-            context = await browser.new_context(storage_state=browser_state) if browser_state else await browser.new_context()
+            context = await browser.new_context(
+                storage_state=browser_state,
+                user_agent=browser_user_agent,
+            )
             page = await context.new_page()
             async with TwoCaptchaSolver(
                 framework=FrameworkType.PLAYWRIGHT,
@@ -158,21 +163,26 @@ async def fetch_in_browser(
                     raise RuntimeError(f"RSS did not load after Cloudflare challenge: {await page.title()}") from exc
                 xml = await page.locator("pre").inner_text()
                 state = await context.storage_state()
+                user_agent = await page.evaluate("navigator.userAgent")
             await browser.close()
-    return xml, state
+    return xml, state, user_agent
 
 
 async def enrich_items_in_browser(
     items: list[ParsedItem],
     rucaptcha_token: str,
     browser_state: dict[str, Any] | None,
+    browser_user_agent: str | None,
     content_selector: str,
     text_selector: str,
-) -> tuple[list[ParsedItem], dict[str, Any]]:
+) -> tuple[list[ParsedItem], dict[str, Any], str]:
     async with browser_lock:
         async with async_playwright() as playwright:
             browser = await playwright.chromium.launch(channel="chrome", headless=False)
-            context = await browser.new_context(storage_state=browser_state) if browser_state else await browser.new_context()
+            context = await browser.new_context(
+                storage_state=browser_state,
+                user_agent=browser_user_agent,
+            )
             page = await context.new_page()
             async with TwoCaptchaSolver(
                 framework=FrameworkType.PLAYWRIGHT,
@@ -202,8 +212,9 @@ async def enrich_items_in_browser(
                     item.text = text
                     item.media = list({entry["url"]: entry for entry in [*item.media, *media]}.values())
                 state = await context.storage_state()
+                user_agent = await page.evaluate("navigator.userAgent")
             await browser.close()
-    return items, state
+    return items, state, user_agent
 
 
 @app.get("/health")
@@ -241,6 +252,7 @@ async def parse_source(
     url = str(request.url)
     xml = ""
     state = request.browser_state
+    browser_user_agent = request.browser_user_agent
     fetched_via = "http"
     if access != "cloudflare":
         try:
@@ -261,7 +273,12 @@ async def parse_source(
             raise HTTPException(status_code=422, detail="rucaptcha_token is required for Cloudflare")
         fetched_via = "cloudflare"
         try:
-            xml, state = await fetch_in_browser(url, request.rucaptcha_token.get_secret_value(), state)
+            xml, state, browser_user_agent = await fetch_in_browser(
+                url,
+                request.rucaptcha_token.get_secret_value(),
+                state,
+                browser_user_agent,
+            )
         except Exception as exc:
             raise HTTPException(status_code=502, detail=str(exc)) from exc
     try:
@@ -272,10 +289,11 @@ async def parse_source(
         if not request.rucaptcha_token:
             raise HTTPException(status_code=422, detail="rucaptcha_token is required when follow_links=true")
         try:
-            items, state = await enrich_items_in_browser(
+            items, state, browser_user_agent = await enrich_items_in_browser(
                 items,
                 request.rucaptcha_token.get_secret_value(),
                 state,
+                browser_user_agent,
                 content_selector,
                 text_selector,
             )
@@ -287,4 +305,5 @@ async def parse_source(
         "fetched_via": fetched_via,
         "items": [item.model_dump() for item in items],
         "browser_state": state,
+        "browser_user_agent": browser_user_agent,
     }

site_parser_worker/docker-compose.yml

@@ -0,0 +1,21 @@
+version: "3.8"
+
+services:
+  site-parser:
+    build: .
+    container_name: site-parser-worker
+    restart: unless-stopped
+    env_file: .env
+    ports:
+      - "8080:8080"
+    shm_size: 512mb
+    healthcheck:
+      test:
+        - CMD
+        - python
+        - -c
+        - import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health', timeout=5)
+      interval: 10s
+      timeout: 5s
+      retries: 6
+      start_period: 20s