exostring/new_vk_parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add declarative site parser engine

Browse Source
This commit is contained in:
Your Name
2026-08-11 00:46:11 +05:00
parent 01eea99c88
commit 2d6fa3c99a
13 changed files with 1202 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_source_adapters.py
+25 -3
View File
@@ -61,7 +61,7 @@ class EmptyBodySession(FakeSession):
class SourceAdapterTests(unittest.IsolatedAsyncioTestCase):
async def test_worker_response_is_normalized(self) -> None:
client = SiteParserClient(FakeSession(), "http://worker", "token", "captcha", 30)
items, state = await client.fetch({
items, state, warning = await client.fetch({
"url": "https://example.test/rss.xml",
"settings_json": '{"format":"rss","access":"auto"}',
"runtime_state_json": '{}',
@@ -71,10 +71,11 @@ class SourceAdapterTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(items[0].media[0].url, "https://example.test/1.jpg")
self.assertEqual(state["browser_state"]["cookies"][0]["name"], "cf_clearance")
self.assertEqual(state["browser_user_agent"], "Test Browser")
self.assertIsNone(warning)
async def test_followed_page_does_not_repeat_title(self) -> None:
client = SiteParserClient(FollowedPageSession(), "http://worker", "token", "captcha", 30)
items, _ = await client.fetch({
items, _, _ = await client.fetch({
"url": "https://example.test/rss.xml",
"settings_json": '{"format":"rss","follow_links":true,"content_selector":"article.full"}',
"runtime_state_json": "{}",
@@ -84,7 +85,7 @@ class SourceAdapterTests(unittest.IsolatedAsyncioTestCase):
async def test_title_is_not_counted_as_site_body(self) -> None:
client = SiteParserClient(EmptyBodySession(), "http://worker", "token", "captcha", 30)
items, _ = await client.fetch({
items, _, _ = await client.fetch({
"url": "https://example.test/rss.xml",
"settings_json": '{"format":"rss"}',
"runtime_state_json": "{}",
@@ -104,6 +105,27 @@ class SourceAdapterTests(unittest.IsolatedAsyncioTestCase):
with self.assertRaisesRegex(ValueError, "min_text_length"):
validate_source_config("site", {"format": "rss", "min_text_length": "many"})
def test_versioned_site_config_is_validated(self) -> None:
validate_source_config("site", {
"version": 1,
"discovery": {"type": "rss", "limit": 10},
"detail": {
"enabled": True,
"root_selector": "article",
"fields": {"text": {"selector": ".body", "extract": "text"}},
},
"access": {"type": "cloudflare"},
"retry": {"attempts": 3, "delay_seconds": 5, "timeout_seconds": 90},
"min_text_length": 50,
})
def test_html_discovery_requires_item_selector(self) -> None:
with self.assertRaisesRegex(ValueError, "item_selector"):
validate_source_config("site", {
"version": 1,
"discovery": {"type": "html"},
})
if __name__ == "__main__":
unittest.main()