feat: add declarative site parser engine
This commit is contained in:
Your Name
@@ -0,0 +1,171 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from extractor import detail_from_html, extract_fields, normalize_config
|
||||
|
||||
|
||||
POPULARAIRSOFT_CONFIG = {
|
||||
"version": 1,
|
||||
"discovery": {
|
||||
"type": "html",
|
||||
"item_selector": "#block-views-block-latest-news-list-block-2 .feature-contents, #block-views-block-latest-news-list-block-1 .lt-teasure",
|
||||
"limit": 10,
|
||||
"fields": {
|
||||
"url": {"selector": "a.link-title", "extract": "attr", "attribute": "href", "required": True},
|
||||
"external_id": {"selector": "a.link-title", "extract": "attr", "attribute": "href", "required": True},
|
||||
"title": {"selector": "a.link-title", "extract": "text", "required": True},
|
||||
"published_at": {
|
||||
"selector": "time[datetime]",
|
||||
"extract": "attr",
|
||||
"attribute": "datetime",
|
||||
"required": True,
|
||||
},
|
||||
},
|
||||
"media": [
|
||||
{"type": "photo", "selector": ".site-image img", "attributes": ["src", "data-src", "srcset"]}
|
||||
],
|
||||
},
|
||||
"detail": {
|
||||
"enabled": True,
|
||||
"always": True,
|
||||
"transport": "http",
|
||||
"root_selector": "article.news.full",
|
||||
"fields": {
|
||||
"title": {
|
||||
"selector": ".feature-contents > .news-story-texts:first-child h2",
|
||||
"extract": "text",
|
||||
"required": True,
|
||||
},
|
||||
"published_at": {
|
||||
"candidates": [
|
||||
{
|
||||
"selector": ".feature-contents > .news-story-texts:first-child .news-story-date",
|
||||
"extract": "text",
|
||||
"formats": ["%d %b %Y"],
|
||||
"timezone": "UTC",
|
||||
},
|
||||
{"source": "list.published_at"},
|
||||
],
|
||||
"required": True,
|
||||
},
|
||||
"text": {
|
||||
"selector": ".feature-contents > .news-story-texts:last-child .field--name-body",
|
||||
"extract": "text",
|
||||
"required": True,
|
||||
"min_length": 50,
|
||||
},
|
||||
},
|
||||
"media": [
|
||||
{
|
||||
"type": "photo",
|
||||
"selector": ".feature-contents > .site-image .field--name-field-image img",
|
||||
"attributes": ["src", "data-src", "srcset"],
|
||||
},
|
||||
{
|
||||
"type": "video",
|
||||
"selector": ".field--name-body iframe, .field--name-body video, .field--name-body source",
|
||||
"attributes": ["src", "data-src"],
|
||||
},
|
||||
],
|
||||
},
|
||||
"access": {"type": "cloudflare", "wait_for": "#block-views-block-latest-news-list-block-1"},
|
||||
"retry": {"attempts": 3, "delay_seconds": 5, "timeout_seconds": 90},
|
||||
"min_text_length": 50,
|
||||
}
|
||||
|
||||
|
||||
PAGE = """
|
||||
<html><body>
|
||||
<img src="/logo.png">
|
||||
<article class="news full">
|
||||
<div class="feature-contents">
|
||||
<div class="news-story-texts">
|
||||
<h2>Double Bell M16A2</h2>
|
||||
<h4>OptimusPrime</h4>
|
||||
<p class="news-story-date">10 Aug 2026</p>
|
||||
</div>
|
||||
<div class="site-image">
|
||||
<div class="field--name-field-image"><img src="/cover.jpg"></div>
|
||||
</div>
|
||||
<div class="news-story-texts">
|
||||
<div class="field--name-body">
|
||||
<p>This is the complete article body with enough useful text to pass validation safely.</p>
|
||||
<iframe src="https://www.youtube-nocookie.com/embed/t6mvlySpXNk?si=test"></iframe>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
<div class="related"><img src="/garbage.jpg"></div>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
LIST = """
|
||||
<section id="block-views-block-latest-news-list-block-2">
|
||||
<div class="feature-contents">
|
||||
<a class="link-title" href="/news/vfc-vityaz">VFC Vityaz</a>
|
||||
<time datetime="2026-08-10T06:06:31+00:00">10 Aug 2026</time>
|
||||
</div>
|
||||
</section>
|
||||
<section id="block-views-block-latest-news-list-block-1">
|
||||
<div class="lt-teasure">
|
||||
<a class="link-title" href="/news/double-bell">Double Bell</a>
|
||||
<time datetime="2026-08-10T06:05:49+00:00">10 Aug 2026</time>
|
||||
</div>
|
||||
</section>
|
||||
"""
|
||||
|
||||
|
||||
class ExtractorTests(unittest.TestCase):
|
||||
def test_popularairsoft_discovery_selects_news_links(self) -> None:
|
||||
config = normalize_config(POPULARAIRSOFT_CONFIG)
|
||||
soup = BeautifulSoup(LIST, "html.parser")
|
||||
cards = soup.select(config["discovery"]["item_selector"])
|
||||
values = [extract_fields(card, config["discovery"]["fields"])[0] for card in cards]
|
||||
self.assertEqual([item["url"] for item in values], ["/news/vfc-vityaz", "/news/double-bell"])
|
||||
self.assertEqual(values[1]["published_at"], "2026-08-10T06:05:49+00:00")
|
||||
|
||||
def test_popularairsoft_extracts_only_article_fields(self) -> None:
|
||||
config = normalize_config(POPULARAIRSOFT_CONFIG)
|
||||
item, errors = detail_from_html(
|
||||
PAGE,
|
||||
"https://popularairsoft.com/news/example",
|
||||
config,
|
||||
{"list": {"published_at": "2026-08-10T06:05:49+00:00"}},
|
||||
)
|
||||
self.assertEqual(errors, [])
|
||||
self.assertEqual(item["title"], "Double Bell M16A2")
|
||||
self.assertEqual(item["published_at"], "2026-08-10T00:00:00+00:00")
|
||||
self.assertNotIn("garbage", item["text"])
|
||||
self.assertEqual(item["media"][0]["url"], "https://popularairsoft.com/cover.jpg")
|
||||
self.assertEqual(item["media"][1], {
|
||||
"type": "video",
|
||||
"url": "https://www.youtube.com/watch?v=t6mvlySpXNk",
|
||||
"provider": "youtube",
|
||||
})
|
||||
|
||||
def test_date_falls_back_to_rss(self) -> None:
|
||||
fields = {
|
||||
"published_at": {
|
||||
"candidates": [
|
||||
{"selector": "time", "extract": "attr", "attribute": "datetime"},
|
||||
{"source": "rss.published"},
|
||||
],
|
||||
"required": True,
|
||||
}
|
||||
}
|
||||
values, errors = extract_fields(None, fields, {"rss": {"published": "Mon, 10 Aug 2026 06:05:49 +0000"}})
|
||||
self.assertEqual(errors, [])
|
||||
self.assertEqual(values["published_at"], "2026-08-10T06:05:49+00:00")
|
||||
|
||||
def test_missing_required_field_is_an_item_error(self) -> None:
|
||||
config = normalize_config(POPULARAIRSOFT_CONFIG)
|
||||
item, errors = detail_from_html("<article class='news full'></article>", "https://example.test/1", config)
|
||||
self.assertEqual(item["media"], [])
|
||||
self.assertTrue(any(error["field"] == "text" for error in errors))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user