from __future__ import annotations import unittest from bs4 import BeautifulSoup from extractor import detail_from_html, extract_fields, normalize_config POPULARAIRSOFT_CONFIG = { "version": 1, "discovery": { "type": "html", "item_selector": "#block-views-block-latest-news-list-block-2 .feature-contents, #block-views-block-latest-news-list-block-1 .lt-teasure", "limit": 10, "fields": { "url": {"selector": "a.link-title", "extract": "attr", "attribute": "href", "required": True}, "external_id": {"selector": "a.link-title", "extract": "attr", "attribute": "href", "required": True}, "title": {"selector": "a.link-title", "extract": "text", "required": True}, "published_at": { "selector": "time[datetime]", "extract": "attr", "attribute": "datetime", "required": True, }, }, "media": [ {"type": "photo", "selector": ".site-image img", "attributes": ["src", "data-src", "srcset"]} ], }, "detail": { "enabled": True, "always": True, "transport": "http", "root_selector": "article.news.full", "fields": { "title": { "selector": ".feature-contents > .news-story-texts:first-child h2", "extract": "text", "required": True, }, "published_at": { "candidates": [ { "selector": ".feature-contents > .news-story-texts:first-child .news-story-date", "extract": "text", "formats": ["%d %b %Y"], "timezone": "UTC", }, {"source": "list.published_at"}, ], "required": True, }, "text": { "selector": ".feature-contents > .news-story-texts:last-child .field--name-body", "extract": "text", "required": True, "min_length": 50, }, }, "media": [ { "type": "photo", "selector": ".feature-contents > .site-image .field--name-field-image img", "attributes": ["src", "data-src", "srcset"], }, { "type": "video", "selector": ".field--name-body iframe, .field--name-body video, .field--name-body source", "attributes": ["src", "data-src"], }, ], }, "access": {"type": "cloudflare", "wait_for": "#block-views-block-latest-news-list-block-1"}, "retry": {"attempts": 3, "delay_seconds": 5, "timeout_seconds": 90}, "min_text_length": 50, } PAGE = """

Double Bell M16A2

OptimusPrime

10 Aug 2026

This is the complete article body with enough useful text to pass validation safely.

""" LIST = """
VFC Vityaz
Double Bell
""" class ExtractorTests(unittest.TestCase): def test_popularairsoft_discovery_selects_news_links(self) -> None: config = normalize_config(POPULARAIRSOFT_CONFIG) soup = BeautifulSoup(LIST, "html.parser") cards = soup.select(config["discovery"]["item_selector"]) values = [extract_fields(card, config["discovery"]["fields"])[0] for card in cards] self.assertEqual([item["url"] for item in values], ["/news/vfc-vityaz", "/news/double-bell"]) self.assertEqual(values[1]["published_at"], "2026-08-10T06:05:49+00:00") def test_popularairsoft_extracts_only_article_fields(self) -> None: config = normalize_config(POPULARAIRSOFT_CONFIG) item, errors = detail_from_html( PAGE, "https://popularairsoft.com/news/example", config, {"list": {"published_at": "2026-08-10T06:05:49+00:00"}}, ) self.assertEqual(errors, []) self.assertEqual(item["title"], "Double Bell M16A2") self.assertEqual(item["published_at"], "2026-08-10T00:00:00+00:00") self.assertNotIn("garbage", item["text"]) self.assertEqual(item["media"][0]["url"], "https://popularairsoft.com/cover.jpg") self.assertEqual(item["media"][1], { "type": "video", "url": "https://www.youtube.com/watch?v=t6mvlySpXNk", "provider": "youtube", }) def test_date_falls_back_to_rss(self) -> None: fields = { "published_at": { "candidates": [ {"selector": "time", "extract": "attr", "attribute": "datetime"}, {"source": "rss.published"}, ], "required": True, } } values, errors = extract_fields(None, fields, {"rss": {"published": "Mon, 10 Aug 2026 06:05:49 +0000"}}) self.assertEqual(errors, []) self.assertEqual(values["published_at"], "2026-08-10T06:05:49+00:00") def test_missing_required_field_is_an_item_error(self) -> None: config = normalize_config(POPULARAIRSOFT_CONFIG) item, errors = detail_from_html("
", "https://example.test/1", config) self.assertEqual(item["media"], []) self.assertTrue(any(error["field"] == "text" for error in errors)) if __name__ == "__main__": unittest.main()