exostring/new_vk_parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: ingest site videos through raw pipeline

Browse Source
This commit is contained in:
Your Name
2026-08-10 22:35:33 +05:00
parent 7889d4d376
commit faded0132b
9 changed files with 231 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
site_parser_worker/app.py
+104 -11
View File
@@ -2,10 +2,12 @@ from __future__ import annotations
import asyncio
import os
import re
import secrets
from datetime import datetime, timezone
from time import struct_time
from typing import Annotated, Any
from urllib.parse import parse_qs, urljoin, urlparse
import feedparser
import httpx
@@ -59,28 +61,59 @@ def iso_date(value: struct_time | None) -> str | None:
return datetime(*value[:6], tzinfo=timezone.utc).isoformat()
def clean_html(value: Any) -> tuple[str, str, list[dict[str, str]]]:
def youtube_url(value: str, base_url: str = "") -> str | None:
url = urljoin(base_url, str(value or "").strip())
parsed = urlparse(url)
host = (parsed.hostname or "").lower().removeprefix("www.").removeprefix("m.")
video_id = ""
if host == "youtu.be":
video_id = parsed.path.strip("/").split("/", 1)[0]
elif host in {"youtube.com", "youtube-nocookie.com"}:
if parsed.path == "/watch":
video_id = (parse_qs(parsed.query).get("v") or [""])[0]
elif parsed.path.startswith(("/embed/", "/shorts/", "/live/")):
video_id = parsed.path.strip("/").split("/", 1)[1]
if not re.fullmatch(r"[A-Za-z0-9_-]{6,20}", video_id):
return None
return f"https://www.youtube.com/watch?v={video_id}"
def clean_html(value: Any, base_url: str = "") -> tuple[str, str, list[dict[str, str]]]:
html = str(value or "").strip()
soup = BeautifulSoup(html, "html.parser")
for element in soup.find_all(["script", "style", "noscript"]):
element.decompose()
text = soup.get_text("\n", strip=True)
media = [{"type": "photo", "url": str(image["src"])} for image in soup.find_all("img", src=True)]
media = [
{"type": "photo", "url": urljoin(base_url, str(image["src"]))}
for image in soup.find_all("img", src=True)
]
for element in soup.find_all(["video", "source"], src=True):
media.append({"type": "video", "url": urljoin(base_url, str(element["src"]))})
for element in soup.find_all(["iframe", "a"]):
url = youtube_url(str(element.get("src") or element.get("href") or ""), base_url)
if url:
media.append({"type": "video", "url": url})
return html, text, media
def parse_rss(xml: str, max_items: int) -> list[ParsedItem]:
feed = feedparser.parse(xml)
feed = feedparser.parse(xml, sanitize_html=False)
if feed.bozo and not feed.entries:
raise ValueError(f"Invalid RSS: {feed.bozo_exception}")
items = []
for entry in feed.entries[:max_items]:
title = BeautifulSoup(str(entry.get("title") or ""), "html.parser").get_text(" ", strip=True)
html, text, media = clean_html(entry.get("description") or entry.get("summary") or "")
for enclosure in entry.get("enclosures") or []:
url = str(enclosure.get("href") or enclosure.get("url") or "").strip()
media_type = str(enclosure.get("type") or "")
if url and (not media_type or media_type.startswith("image/")):
media.append({"type": "photo", "url": url})
url = str(entry.get("link") or "").strip()
content = (entry.get("content") or [{}])[0].get("value") or entry.get("description") or entry.get("summary") or ""
html, text, media = clean_html(content, url)
for enclosure in entry.get("enclosures") or []:
enclosure_url = urljoin(url, str(enclosure.get("href") or enclosure.get("url") or "").strip())
media_type = str(enclosure.get("type") or "")
if enclosure_url and (not media_type or media_type.startswith("image/")):
media.append({"type": "photo", "url": enclosure_url})
elif enclosure_url and media_type.startswith("video/"):
media.append({"type": "video", "url": enclosure_url})
items.append(
ParsedItem(
external_id=str(entry.get("id") or entry.get("guid") or url).strip(),
@@ -113,8 +146,8 @@ async def fetch_in_browser(
async_two_captcha_client=AsyncTwoCaptcha(rucaptcha_token),
max_attempts=1,
) as solver:
response = await page.goto(url, wait_until="domcontentloaded", timeout=60_000)
if response and response.status in {403, 429, 503} and "Just a moment" in await page.title():
await page.goto(url, wait_until="domcontentloaded", timeout=60_000)
if "just a moment" in (await page.title()).lower():
await solver.solve_captcha(
captcha_container=page,
captcha_type=CaptchaType.CLOUDFLARE_INTERSTITIAL,
@@ -129,6 +162,47 @@ async def fetch_in_browser(
return xml, state
async def enrich_items_in_browser(
items: list[ParsedItem],
rucaptcha_token: str,
browser_state: dict[str, Any] | None,
content_selector: str,
) -> tuple[list[ParsedItem], dict[str, Any]]:
async with browser_lock:
async with async_playwright() as playwright:
browser = await playwright.chromium.launch(channel="chrome", headless=False)
context = await browser.new_context(storage_state=browser_state) if browser_state else await browser.new_context()
page = await context.new_page()
async with TwoCaptchaSolver(
framework=FrameworkType.PLAYWRIGHT,
page=page,
async_two_captcha_client=AsyncTwoCaptcha(rucaptcha_token),
max_attempts=1,
) as solver:
for item in items:
if item.text and item.media:
continue
await page.goto(item.url, wait_until="domcontentloaded", timeout=60_000)
if "just a moment" in (await page.title()).lower():
await solver.solve_captcha(
captcha_container=page,
captcha_type=CaptchaType.CLOUDFLARE_INTERSTITIAL,
)
try:
content = page.locator(content_selector).first
await content.wait_for(state="visible", timeout=60_000)
except Exception as exc:
raise RuntimeError(f"Item content not found: {item.url} ({content_selector})") from exc
html, text, media = clean_html(await content.inner_html(), item.url)
if text:
item.html = html
item.text = text
item.media = list({entry["url"]: entry for entry in [*item.media, *media]}.values())
state = await context.storage_state()
await browser.close()
return items, state
@app.get("/health")
async def health() -> dict[str, bool]:
return {"ok": True}
@@ -154,6 +228,12 @@ async def parse_source(
raise HTTPException(status_code=422, detail="config.max_items must be an integer") from exc
if not 1 <= max_items <= 100:
raise HTTPException(status_code=422, detail="config.max_items must be between 1 and 100")
follow_links = config.get("follow_links", False)
if not isinstance(follow_links, bool):
raise HTTPException(status_code=422, detail="config.follow_links must be true or false")
content_selector = str(config.get("content_selector") or "").strip()
if follow_links and not content_selector:
raise HTTPException(status_code=422, detail="config.content_selector is required when follow_links=true")
url = str(request.url)
xml = ""
state = request.browser_state
@@ -184,6 +264,19 @@ async def parse_source(
items = parse_rss(xml, max_items)
except ValueError as exc:
raise HTTPException(status_code=502, detail=str(exc)) from exc
if follow_links:
if not request.rucaptcha_token:
raise HTTPException(status_code=422, detail="rucaptcha_token is required when follow_links=true")
try:
items, state = await enrich_items_in_browser(
items,
request.rucaptcha_token.get_secret_value(),
state,
content_selector,
)
except Exception as exc:
raise HTTPException(status_code=502, detail=str(exc)) from exc
fetched_via += "+item-pages"
return {
"source_url": url,
"fetched_via": fetched_via,
site_parser_worker/test_parser.py
+3 -1
View File
@@ -4,12 +4,14 @@ from app import parse_rss
def test_parse_rss() -> None:
xml = """<rss><channel><item><guid>1</guid><title>Title</title>
<link>https://example.test/1</link>
<description>&lt;p&gt;Body&lt;/p&gt;&lt;img src="https://example.test/1.jpg"&gt;</description>
<description>&lt;p&gt;Body&lt;/p&gt;&lt;img src="/1.jpg"&gt;
&lt;iframe src="https://www.youtube.com/embed/dQw4w9WgXcQ?feature=oembed"&gt;&lt;/iframe&gt;</description>
<pubDate>Mon, 10 Aug 2026 10:00:00 +0000</pubDate></item></channel></rss>"""
item = parse_rss(xml, 20)[0]
assert item.external_id == "1"
assert item.text == "Body"
assert item.media[0]["url"] == "https://example.test/1.jpg"
assert item.media[1] == {"type": "video", "url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ"}
if __name__ == "__main__":