fix: keep site parsing interval global

db/migrations/047_site_parser_schedule.sql

@@ -3,8 +3,8 @@ VALUES (
   'site_parser_default_interval_minutes',
   '30'::jsonb,
   'int',
-  'Интервал по умолчанию, мин',
+  'Интервал парсинга сайтов, мин',
-  'Как часто проверять сайт, если у источника не задан свой интервал.',
+  'Как часто Site Parser проверяет все активные сайты.',
   'Site Parser'
 )
 ON CONFLICT (key) DO NOTHING;

db/migrations/048_site_parser_global_interval.sql

@@ -0,0 +1,14 @@
+INSERT INTO app_settings(key, value_json, value_type, title, description, category, updated_by)
+SELECT
+  'site_parser_interval_minutes',
+  value_json,
+  'int',
+  'Интервал парсинга сайтов, мин',
+  'Как часто Site Parser проверяет все активные сайты.',
+  'Site Parser',
+  updated_by
+FROM app_settings
+WHERE key='site_parser_default_interval_minutes'
+ON CONFLICT (key) DO NOTHING;
+
+DELETE FROM app_settings WHERE key='site_parser_default_interval_minutes';

src/vk_parser_app/admin.py

@@ -371,7 +371,7 @@ SETTING_ORDER = {
         "site_parser_token",
         "site_parser_rucaptcha_token",
         "site_parser_timeout_sec",
-        "site_parser_default_interval_minutes",
+        "site_parser_interval_minutes",
     ],
     "VK": [
         "vk_requests_per_second",
@@ -2359,14 +2359,7 @@ async def source_new(request: Request):
         return redirect("/login")
     return templates.TemplateResponse(
         "source_form.html",
-        base_context(
+        base_context(request, user, source=None, action="/sources/new", title="Новый источник"),
-            request,
-            user,
-            source=None,
-            action="/sources/new",
-            title="Новый источник",
-            site_default_interval=await fetch_int_setting("site_parser_default_interval_minutes", 30),
-        ),
     )
 
 
@@ -2381,7 +2374,6 @@ async def source_create(
     active: str = Form("off"),
     priority: int = Form(100),
     settings_json: str = Form("{}"),
-    interval_minutes: int = Form(30),
 ):
     user = await get_current_user(request)
     if not user:
@@ -2394,8 +2386,6 @@ async def source_create(
         source_settings = json.loads(settings_json or "{}")
         if not isinstance(source_settings, dict):
             raise ValueError("Настройки должны быть JSON-объектом")
-        if platform == PLATFORM_SITE:
-            source_settings["interval_minutes"] = interval_minutes
         validate_source_config(platform, source_settings)
     except (json.JSONDecodeError, ValueError) as exc:
         raise HTTPException(status_code=422, detail=str(exc)) from exc
@@ -2461,7 +2451,6 @@ async def source_edit(request: Request, source_id: int):
             source={**dict(source), "settings_json": json_object(source["settings_json"])},
             action=f"/sources/{source_id}/edit",
             title=f"Источник #{source_id}",
-            site_default_interval=await fetch_int_setting("site_parser_default_interval_minutes", 30),
         ),
     )
 
@@ -2478,7 +2467,6 @@ async def source_update(
     active: str = Form("off"),
     priority: int = Form(100),
     settings_json: str = Form("{}"),
-    interval_minutes: int = Form(30),
 ):
     user = await get_current_user(request)
     if not user:
@@ -2491,8 +2479,6 @@ async def source_update(
         source_settings = json.loads(settings_json or "{}")
         if not isinstance(source_settings, dict):
             raise ValueError("Настройки должны быть JSON-объектом")
-        if platform == PLATFORM_SITE:
-            source_settings["interval_minutes"] = interval_minutes
         validate_source_config(platform, source_settings)
     except (json.JSONDecodeError, ValueError) as exc:
         raise HTTPException(status_code=422, detail=str(exc)) from exc

src/vk_parser_app/source_adapters.py

@@ -55,14 +55,6 @@ def validate_source_config(platform: str, config: dict[str, Any]) -> None:
         raise ValueError("max_items должен быть целым числом") from exc
     if not 1 <= max_items <= 100:
         raise ValueError("max_items должен быть от 1 до 100")
-    try:
-        interval_minutes = int(config.get("interval_minutes", 30))
-    except (TypeError, ValueError) as exc:
-        raise ValueError("interval_minutes должен быть целым числом") from exc
-    if not 1 <= interval_minutes <= 10080:
-        raise ValueError("interval_minutes должен быть от 1 до 10080")
-
-
 def _posted_at(value: Any) -> datetime:
     try:
         parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))

src/vk_parser_app/templates/source_form.html

@@ -42,8 +42,6 @@
         </div>
 
         <div id="site-config" class="form-control md:col-span-2">
-          <label class="label"><span class="label-text font-bold">Проверять каждые, минут</span></label>
-          <input name="interval_minutes" type="number" min="1" max="10080" value="{{ source.settings_json.get('interval_minutes', site_default_interval) if source and source.settings_json else site_default_interval }}" class="input input-bordered w-full max-w-xs mb-4">
           <label class="label"><span class="label-text font-bold">Конфигурация JSON</span></label>
           <textarea name="settings_json" rows="8" class="textarea textarea-bordered w-full font-mono" placeholder='{"format":"rss","access":"auto","max_items":20}'>{{ source.settings_json | tojson(indent=2) if source else '{}' }}</textarea>
           <details class="mt-2 text-sm text-base-content/70">
@@ -51,8 +49,7 @@
             <pre class="mt-2 p-3 bg-base-200 overflow-x-auto">{
   "format": "rss",
   "access": "auto",
-  "max_items": 20,
+  "max_items": 20
-  "interval_minutes": 30
 }</pre>
             <p class="mt-2"><code>access</code>: <code>auto</code> сначала пробует обычный запрос и при Cloudflare использует RuCaptcha; <code>http</code> запрещает браузер; <code>cloudflare</code> сразу запускает браузер.</p>
           </details>

src/vk_parser_app/templates/source_row.html

@@ -7,7 +7,6 @@
     </div>
     <div class="sm:hidden text-[10px] font-mono text-app-textMuted mt-1">#{{ s.id }}</div>
     <div class="text-[10px] font-mono text-app-textMuted mt-1 truncate" title="External ID">{{ s.external_id or "" }}</div>
-    {% if s.platform == "site" %}<div class="text-[10px] text-app-textMuted mt-1">Каждые {{ s.settings_json.get('interval_minutes', 30) }} мин.</div>{% endif %}
     <div class="sm:hidden text-[10px] font-mono text-blue-400 mt-1 truncate">#{{ s.tag or "—" }}</div>
     <a href="{{ s.url }}" target="_blank" class="lg:hidden text-[10px] text-app-textMuted hover:text-white hover:underline truncate block mt-1" title="{{ s.url }}">{{ s.url }}</a>
   </td>

src/vk_parser_app/workers/parser.py

@@ -55,7 +55,7 @@ class VKParserWorker:
     async def init(self) -> None:
         self.pool = await get_pool()
 
-    async def active_sources(self, vk_interval_sec: int, site_default_interval_minutes: int) -> list[dict]:
+    async def active_sources(self, vk_interval_sec: int, site_interval_minutes: int) -> list[dict]:
         rows = await self.pool.fetch(
             """
             SELECT *
@@ -68,13 +68,7 @@ class VKParserWorker:
                 OR (platform=$2 AND last_checked_at <= NOW() - $3::double precision * INTERVAL '1 second')
                 OR (
                   platform=$4
-                  AND last_checked_at <= NOW() - (
+                  AND last_checked_at <= NOW() - $5::double precision * INTERVAL '1 minute'
-                    CASE
-                      WHEN settings_json->>'interval_minutes' ~ '^[0-9]+$'
-                      THEN (settings_json->>'interval_minutes')::double precision
-                      ELSE $5::double precision
-                    END
-                  ) * INTERVAL '1 minute'
                 )
               )
             ORDER BY last_checked_at NULLS FIRST, priority ASC, id ASC
@@ -83,7 +77,7 @@ class VKParserWorker:
             PLATFORM_VK,
             vk_interval_sec,
             PLATFORM_SITE,
-            site_default_interval_minutes,
+            site_interval_minutes,
         )
         return [dict(r) for r in rows]
 
@@ -531,7 +525,7 @@ class VKParserWorker:
             return
 
         parser_interval_sec = max(10, await fetch_int_setting("parser_interval_sec", 300))
-        site_default_interval = max(1, await fetch_int_setting("site_parser_default_interval_minutes", 30))
+        site_interval = max(1, await fetch_int_setting("site_parser_interval_minutes", 30))
         rps = await fetch_int_setting("vk_requests_per_second", 3)
         timeout_total = await fetch_int_setting("vk_api_timeout_total_sec", 15)
         timeout_connect = await fetch_int_setting("vk_api_timeout_connect_sec", 5)
@@ -540,7 +534,7 @@ class VKParserWorker:
         retry_min_delay = await fetch_float_setting("vk_api_retry_min_delay_sec", 2.0)
         retry_max_delay = await fetch_float_setting("vk_api_retry_max_delay_sec", 10.0)
         source_pause = max(0.0, await fetch_float_setting("parser_source_pause_sec", 0.0))
-        sources = await self.active_sources(parser_interval_sec, site_default_interval)
+        sources = await self.active_sources(parser_interval_sec, site_interval)
         await self.heartbeat.beat(self.pool, meta={"sources": len(sources)})
         if not sources:
             logger.debug("No sources due for parsing")

tests/test_source_adapters.py

@@ -38,7 +38,7 @@ class SourceAdapterTests(unittest.IsolatedAsyncioTestCase):
         client = SiteParserClient(FakeSession(), "http://worker", "token", "captcha", 30)
         items, state = await client.fetch({
             "url": "https://example.test/rss.xml",
-            "settings_json": '{"format":"rss","access":"auto","interval_minutes":30}',
+            "settings_json": '{"format":"rss","access":"auto"}',
             "runtime_state_json": '{}',
         })