feat: filter site bodies before media upload

site_parser_worker/app.py

@@ -167,6 +167,7 @@ async def enrich_items_in_browser(
     rucaptcha_token: str,
     browser_state: dict[str, Any] | None,
     content_selector: str,
+    text_selector: str,
 ) -> tuple[list[ParsedItem], dict[str, Any]]:
     async with browser_lock:
         async with async_playwright() as playwright:
@@ -194,9 +195,11 @@ async def enrich_items_in_browser(
                     except Exception as exc:
                         raise RuntimeError(f"Item content not found: {item.url} ({content_selector})") from exc
                     html, text, media = clean_html(await content.inner_html(), item.url)
-                    if text:
-                        item.html = html
-                        item.text = text
+                    if text_selector:
+                        text_content = content.locator(text_selector).first
+                        text = clean_html(await text_content.inner_html())[1] if await text_content.count() else ""
+                    item.html = html
+                    item.text = text
                     item.media = list({entry["url"]: entry for entry in [*item.media, *media]}.values())
                 state = await context.storage_state()
             await browser.close()
@@ -232,6 +235,7 @@ async def parse_source(
     if not isinstance(follow_links, bool):
         raise HTTPException(status_code=422, detail="config.follow_links must be true or false")
     content_selector = str(config.get("content_selector") or "").strip()
+    text_selector = str(config.get("text_selector") or "").strip()
     if follow_links and not content_selector:
         raise HTTPException(status_code=422, detail="config.content_selector is required when follow_links=true")
     url = str(request.url)
@@ -273,6 +277,7 @@ async def parse_source(
                 request.rucaptcha_token.get_secret_value(),
                 state,
                 content_selector,
+                text_selector,
             )
         except Exception as exc:
             raise HTTPException(status_code=502, detail=str(exc)) from exc