Your Name
172 lines
6.4 KiB
Python
172 lines
6.4 KiB
Python
from __future__ import annotations
|
|
|
|
import unittest
|
|
|
|
from bs4 import BeautifulSoup
|
|
|
|
from extractor import detail_from_html, extract_fields, normalize_config
|
|
|
|
|
|
POPULARAIRSOFT_CONFIG = {
|
|
"version": 1,
|
|
"discovery": {
|
|
"type": "html",
|
|
"item_selector": "#block-views-block-latest-news-list-block-2 .feature-contents, #block-views-block-latest-news-list-block-1 .lt-teasure",
|
|
"limit": 10,
|
|
"fields": {
|
|
"url": {"selector": "a.link-title", "extract": "attr", "attribute": "href", "required": True},
|
|
"external_id": {"selector": "a.link-title", "extract": "attr", "attribute": "href", "required": True},
|
|
"title": {"selector": "a.link-title", "extract": "text", "required": True},
|
|
"published_at": {
|
|
"selector": "time[datetime]",
|
|
"extract": "attr",
|
|
"attribute": "datetime",
|
|
"required": True,
|
|
},
|
|
},
|
|
"media": [
|
|
{"type": "photo", "selector": ".site-image img", "attributes": ["src", "data-src", "srcset"]}
|
|
],
|
|
},
|
|
"detail": {
|
|
"enabled": True,
|
|
"always": True,
|
|
"transport": "http",
|
|
"root_selector": "article.news.full",
|
|
"fields": {
|
|
"title": {
|
|
"selector": ".feature-contents > .news-story-texts:first-child h2",
|
|
"extract": "text",
|
|
"required": True,
|
|
},
|
|
"published_at": {
|
|
"candidates": [
|
|
{
|
|
"selector": ".feature-contents > .news-story-texts:first-child .news-story-date",
|
|
"extract": "text",
|
|
"formats": ["%d %b %Y"],
|
|
"timezone": "UTC",
|
|
},
|
|
{"source": "list.published_at"},
|
|
],
|
|
"required": True,
|
|
},
|
|
"text": {
|
|
"selector": ".feature-contents > .news-story-texts:last-child .field--name-body",
|
|
"extract": "text",
|
|
"required": True,
|
|
"min_length": 50,
|
|
},
|
|
},
|
|
"media": [
|
|
{
|
|
"type": "photo",
|
|
"selector": ".feature-contents > .site-image .field--name-field-image img",
|
|
"attributes": ["src", "data-src", "srcset"],
|
|
},
|
|
{
|
|
"type": "video",
|
|
"selector": ".field--name-body iframe, .field--name-body video, .field--name-body source",
|
|
"attributes": ["src", "data-src"],
|
|
},
|
|
],
|
|
},
|
|
"access": {"type": "cloudflare", "wait_for": "#block-views-block-latest-news-list-block-1"},
|
|
"retry": {"attempts": 3, "delay_seconds": 5, "timeout_seconds": 90},
|
|
"min_text_length": 50,
|
|
}
|
|
|
|
|
|
PAGE = """
|
|
<html><body>
|
|
<img src="/logo.png">
|
|
<article class="news full">
|
|
<div class="feature-contents">
|
|
<div class="news-story-texts">
|
|
<h2>Double Bell M16A2</h2>
|
|
<h4>OptimusPrime</h4>
|
|
<p class="news-story-date">10 Aug 2026</p>
|
|
</div>
|
|
<div class="site-image">
|
|
<div class="field--name-field-image"><img src="/cover.jpg"></div>
|
|
</div>
|
|
<div class="news-story-texts">
|
|
<div class="field--name-body">
|
|
<p>This is the complete article body with enough useful text to pass validation safely.</p>
|
|
<iframe src="https://www.youtube-nocookie.com/embed/t6mvlySpXNk?si=test"></iframe>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</article>
|
|
<div class="related"><img src="/garbage.jpg"></div>
|
|
</body></html>
|
|
"""
|
|
|
|
LIST = """
|
|
<section id="block-views-block-latest-news-list-block-2">
|
|
<div class="feature-contents">
|
|
<a class="link-title" href="/news/vfc-vityaz">VFC Vityaz</a>
|
|
<time datetime="2026-08-10T06:06:31+00:00">10 Aug 2026</time>
|
|
</div>
|
|
</section>
|
|
<section id="block-views-block-latest-news-list-block-1">
|
|
<div class="lt-teasure">
|
|
<a class="link-title" href="/news/double-bell">Double Bell</a>
|
|
<time datetime="2026-08-10T06:05:49+00:00">10 Aug 2026</time>
|
|
</div>
|
|
</section>
|
|
"""
|
|
|
|
|
|
class ExtractorTests(unittest.TestCase):
|
|
def test_popularairsoft_discovery_selects_news_links(self) -> None:
|
|
config = normalize_config(POPULARAIRSOFT_CONFIG)
|
|
soup = BeautifulSoup(LIST, "html.parser")
|
|
cards = soup.select(config["discovery"]["item_selector"])
|
|
values = [extract_fields(card, config["discovery"]["fields"])[0] for card in cards]
|
|
self.assertEqual([item["url"] for item in values], ["/news/vfc-vityaz", "/news/double-bell"])
|
|
self.assertEqual(values[1]["published_at"], "2026-08-10T06:05:49+00:00")
|
|
|
|
def test_popularairsoft_extracts_only_article_fields(self) -> None:
|
|
config = normalize_config(POPULARAIRSOFT_CONFIG)
|
|
item, errors = detail_from_html(
|
|
PAGE,
|
|
"https://popularairsoft.com/news/example",
|
|
config,
|
|
{"list": {"published_at": "2026-08-10T06:05:49+00:00"}},
|
|
)
|
|
self.assertEqual(errors, [])
|
|
self.assertEqual(item["title"], "Double Bell M16A2")
|
|
self.assertEqual(item["published_at"], "2026-08-10T00:00:00+00:00")
|
|
self.assertNotIn("garbage", item["text"])
|
|
self.assertEqual(item["media"][0]["url"], "https://popularairsoft.com/cover.jpg")
|
|
self.assertEqual(item["media"][1], {
|
|
"type": "video",
|
|
"url": "https://www.youtube.com/watch?v=t6mvlySpXNk",
|
|
"provider": "youtube",
|
|
})
|
|
|
|
def test_date_falls_back_to_rss(self) -> None:
|
|
fields = {
|
|
"published_at": {
|
|
"candidates": [
|
|
{"selector": "time", "extract": "attr", "attribute": "datetime"},
|
|
{"source": "rss.published"},
|
|
],
|
|
"required": True,
|
|
}
|
|
}
|
|
values, errors = extract_fields(None, fields, {"rss": {"published": "Mon, 10 Aug 2026 06:05:49 +0000"}})
|
|
self.assertEqual(errors, [])
|
|
self.assertEqual(values["published_at"], "2026-08-10T06:05:49+00:00")
|
|
|
|
def test_missing_required_field_is_an_item_error(self) -> None:
|
|
config = normalize_config(POPULARAIRSOFT_CONFIG)
|
|
item, errors = detail_from_html("<article class='news full'></article>", "https://example.test/1", config)
|
|
self.assertEqual(item["media"], [])
|
|
self.assertTrue(any(error["field"] == "text" for error in errors))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|