feat: add declarative site parser engine

2026-08-11 00:46:11 +05:00
parent 01eea99c88
commit 2d6fa3c99a
13 changed files with 1202 additions and 244 deletions
@@ -0,0 +1,146 @@
+# Site Parser Worker
+
+The worker is a stateless executor. The main application owns source configs,
+schedules, database state, and queues. Each request gives the worker a source
+URL, a versioned JSON instruction, and optional Cloudflare session state. The
+worker returns normalized items plus per-item diagnostics.
+
+## Config v1
+
+- `discovery.type`: `rss` or `html`.
+- `discovery.fields`: where list/RSS values are located.
+- `detail.enabled`: open every discovered item when enabled.
+- `detail.root_selector`: limits extraction to the actual article.
+- `detail.fields`: exact field rules. A rule may contain ordered `candidates`.
+- `detail.media`: exact selectors and fallback attributes for media.
+- `detail.transport`: `http`, `fetch`, or `browser`.
+- `access.type`: `auto`, `http`, `browser`, or `cloudflare`.
+- `retry`: bounded attempts, delay, and per-attempt timeout.
+
+Field extraction modes are `text`, `html`, `attr`, and `json`. Date rules can
+provide `formats` and `timezone`. Relative links are resolved against the page
+URL. Unknown iframe/video providers are returned as `external_video`; they do
+not fail the item.
+
+Errors are isolated to one item. Successful items are returned with
+`status=partial`; the source receives a visible diagnostic in the main app.
+Only discovery/network failure prevents the whole request from returning a
+batch.
+
+## PopularAirsoft
+
+Use `https://popularairsoft.com/` as the source URL. Its RSS mixes full news
+articles with short video pages, so the reliable discovery surface is the
+`The Latest News` HTML block. The worker opens the main page through
+Cloudflare, collects `/news/...` links, and downloads each article using the
+same clearance and an impersonated browser TLS fingerprint.
+
+```json
+{
+  "version": 1,
+  "discovery": {
+    "type": "html",
+    "item_selector": "#block-views-block-latest-news-list-block-2 .feature-contents, #block-views-block-latest-news-list-block-1 .lt-teasure",
+    "limit": 10,
+    "fields": {
+      "url": {
+        "selector": "a.link-title",
+        "extract": "attr",
+        "attribute": "href",
+        "required": true
+      },
+      "external_id": {
+        "selector": "a.link-title",
+        "extract": "attr",
+        "attribute": "href",
+        "required": true
+      },
+      "title": {
+        "selector": "a.link-title",
+        "extract": "text",
+        "required": true
+      },
+      "published_at": {
+        "selector": "time[datetime]",
+        "extract": "attr",
+        "attribute": "datetime",
+        "required": true
+      }
+    },
+    "media": [
+      {
+        "type": "photo",
+        "selector": ".site-image img",
+        "attributes": ["src", "data-src", "srcset"]
+      }
+    ]
+  },
+  "detail": {
+    "enabled": true,
+    "always": true,
+    "transport": "http",
+    "root_selector": "article.news.full",
+    "fields": {
+      "title": {
+        "candidates": [
+          {
+            "selector": ".feature-contents > .news-story-texts:first-child h2",
+            "extract": "text"
+          },
+          {"source": "list.title"}
+        ],
+        "required": true
+      },
+      "published_at": {
+        "candidates": [
+          {
+            "selector": ".feature-contents > .news-story-texts:first-child .news-story-date",
+            "extract": "text",
+            "formats": ["%d %b %Y"],
+            "timezone": "UTC"
+          },
+          {"source": "list.published_at"}
+        ],
+        "required": true
+      },
+      "author": {
+        "selector": ".feature-contents > .news-story-texts:first-child h4",
+        "extract": "text"
+      },
+      "text": {
+        "selector": ".feature-contents > .news-story-texts:last-child .field--name-body",
+        "extract": "text",
+        "required": true,
+        "min_length": 50
+      }
+    },
+    "media": [
+      {
+        "type": "photo",
+        "selector": ".feature-contents > .site-image .field--name-field-image img, .field--name-body img",
+        "attributes": ["src", "data-src", "srcset"]
+      },
+      {
+        "type": "video",
+        "selector": ".field--name-body iframe, .field--name-body video, .field--name-body source",
+        "attributes": ["src", "data-src"]
+      }
+    ]
+  },
+  "access": {
+    "type": "cloudflare",
+    "wait_for": "#block-views-block-latest-news-list-block-1"
+  },
+  "retry": {
+    "attempts": 2,
+    "delay_seconds": 2,
+    "timeout_seconds": 90
+  },
+  "min_text_length": 50
+}
+```
+
+Verified against the live site on 2026-08-11: nine items, nine successful
+details, zero errors. The VFC sample returned 5,041 text characters, one photo,
+and two Twitch links. The Double Bell sample returned 1,293 characters, one
+photo, and one normalized YouTube URL.