diff --git a/db/migrations/047_site_parser_schedule.sql b/db/migrations/047_site_parser_schedule.sql index 72af16d..60f3b03 100644 --- a/db/migrations/047_site_parser_schedule.sql +++ b/db/migrations/047_site_parser_schedule.sql @@ -3,8 +3,8 @@ VALUES ( 'site_parser_default_interval_minutes', '30'::jsonb, 'int', - 'Интервал по умолчанию, мин', - 'Как часто проверять сайт, если у источника не задан свой интервал.', + 'Интервал парсинга сайтов, мин', + 'Как часто Site Parser проверяет все активные сайты.', 'Site Parser' ) ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/048_site_parser_global_interval.sql b/db/migrations/048_site_parser_global_interval.sql new file mode 100644 index 0000000..9df2420 --- /dev/null +++ b/db/migrations/048_site_parser_global_interval.sql @@ -0,0 +1,14 @@ +INSERT INTO app_settings(key, value_json, value_type, title, description, category, updated_by) +SELECT + 'site_parser_interval_minutes', + value_json, + 'int', + 'Интервал парсинга сайтов, мин', + 'Как часто Site Parser проверяет все активные сайты.', + 'Site Parser', + updated_by +FROM app_settings +WHERE key='site_parser_default_interval_minutes' +ON CONFLICT (key) DO NOTHING; + +DELETE FROM app_settings WHERE key='site_parser_default_interval_minutes'; diff --git a/src/vk_parser_app/admin.py b/src/vk_parser_app/admin.py index 33bac20..e4bbcf5 100644 --- a/src/vk_parser_app/admin.py +++ b/src/vk_parser_app/admin.py @@ -371,7 +371,7 @@ SETTING_ORDER = { "site_parser_token", "site_parser_rucaptcha_token", "site_parser_timeout_sec", - "site_parser_default_interval_minutes", + "site_parser_interval_minutes", ], "VK": [ "vk_requests_per_second", @@ -2359,14 +2359,7 @@ async def source_new(request: Request): return redirect("/login") return templates.TemplateResponse( "source_form.html", - base_context( - request, - user, - source=None, - action="/sources/new", - title="Новый источник", - site_default_interval=await fetch_int_setting("site_parser_default_interval_minutes", 30), - ), + base_context(request, user, source=None, action="/sources/new", title="Новый источник"), ) @@ -2381,7 +2374,6 @@ async def source_create( active: str = Form("off"), priority: int = Form(100), settings_json: str = Form("{}"), - interval_minutes: int = Form(30), ): user = await get_current_user(request) if not user: @@ -2394,8 +2386,6 @@ async def source_create( source_settings = json.loads(settings_json or "{}") if not isinstance(source_settings, dict): raise ValueError("Настройки должны быть JSON-объектом") - if platform == PLATFORM_SITE: - source_settings["interval_minutes"] = interval_minutes validate_source_config(platform, source_settings) except (json.JSONDecodeError, ValueError) as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc @@ -2461,7 +2451,6 @@ async def source_edit(request: Request, source_id: int): source={**dict(source), "settings_json": json_object(source["settings_json"])}, action=f"/sources/{source_id}/edit", title=f"Источник #{source_id}", - site_default_interval=await fetch_int_setting("site_parser_default_interval_minutes", 30), ), ) @@ -2478,7 +2467,6 @@ async def source_update( active: str = Form("off"), priority: int = Form(100), settings_json: str = Form("{}"), - interval_minutes: int = Form(30), ): user = await get_current_user(request) if not user: @@ -2491,8 +2479,6 @@ async def source_update( source_settings = json.loads(settings_json or "{}") if not isinstance(source_settings, dict): raise ValueError("Настройки должны быть JSON-объектом") - if platform == PLATFORM_SITE: - source_settings["interval_minutes"] = interval_minutes validate_source_config(platform, source_settings) except (json.JSONDecodeError, ValueError) as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc diff --git a/src/vk_parser_app/source_adapters.py b/src/vk_parser_app/source_adapters.py index 09c132d..6855a34 100644 --- a/src/vk_parser_app/source_adapters.py +++ b/src/vk_parser_app/source_adapters.py @@ -55,14 +55,6 @@ def validate_source_config(platform: str, config: dict[str, Any]) -> None: raise ValueError("max_items должен быть целым числом") from exc if not 1 <= max_items <= 100: raise ValueError("max_items должен быть от 1 до 100") - try: - interval_minutes = int(config.get("interval_minutes", 30)) - except (TypeError, ValueError) as exc: - raise ValueError("interval_minutes должен быть целым числом") from exc - if not 1 <= interval_minutes <= 10080: - raise ValueError("interval_minutes должен быть от 1 до 10080") - - def _posted_at(value: Any) -> datetime: try: parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) diff --git a/src/vk_parser_app/templates/source_form.html b/src/vk_parser_app/templates/source_form.html index f35698a..76988d7 100644 --- a/src/vk_parser_app/templates/source_form.html +++ b/src/vk_parser_app/templates/source_form.html @@ -42,8 +42,6 @@
- -
@@ -51,8 +49,7 @@ 
{
   "format": "rss",
   "access": "auto",
-  "max_items": 20,
-  "interval_minutes": 30
+  "max_items": 20
 }

access: auto сначала пробует обычный запрос и при Cloudflare использует RuCaptcha; http запрещает браузер; cloudflare сразу запускает браузер.

diff --git a/src/vk_parser_app/templates/source_row.html b/src/vk_parser_app/templates/source_row.html index 8d16a62..c642117 100644 --- a/src/vk_parser_app/templates/source_row.html +++ b/src/vk_parser_app/templates/source_row.html @@ -7,7 +7,6 @@
#{{ s.id }}
{{ s.external_id or "" }}
- {% if s.platform == "site" %}
Каждые {{ s.settings_json.get('interval_minutes', 30) }} мин.
{% endif %}
#{{ s.tag or "—" }}
{{ s.url }} diff --git a/src/vk_parser_app/workers/parser.py b/src/vk_parser_app/workers/parser.py index 53831fd..5c421cb 100644 --- a/src/vk_parser_app/workers/parser.py +++ b/src/vk_parser_app/workers/parser.py @@ -55,7 +55,7 @@ class VKParserWorker: async def init(self) -> None: self.pool = await get_pool() - async def active_sources(self, vk_interval_sec: int, site_default_interval_minutes: int) -> list[dict]: + async def active_sources(self, vk_interval_sec: int, site_interval_minutes: int) -> list[dict]: rows = await self.pool.fetch( """ SELECT * @@ -68,13 +68,7 @@ class VKParserWorker: OR (platform=$2 AND last_checked_at <= NOW() - $3::double precision * INTERVAL '1 second') OR ( platform=$4 - AND last_checked_at <= NOW() - ( - CASE - WHEN settings_json->>'interval_minutes' ~ '^[0-9]+$' - THEN (settings_json->>'interval_minutes')::double precision - ELSE $5::double precision - END - ) * INTERVAL '1 minute' + AND last_checked_at <= NOW() - $5::double precision * INTERVAL '1 minute' ) ) ORDER BY last_checked_at NULLS FIRST, priority ASC, id ASC @@ -83,7 +77,7 @@ class VKParserWorker: PLATFORM_VK, vk_interval_sec, PLATFORM_SITE, - site_default_interval_minutes, + site_interval_minutes, ) return [dict(r) for r in rows] @@ -531,7 +525,7 @@ class VKParserWorker: return parser_interval_sec = max(10, await fetch_int_setting("parser_interval_sec", 300)) - site_default_interval = max(1, await fetch_int_setting("site_parser_default_interval_minutes", 30)) + site_interval = max(1, await fetch_int_setting("site_parser_interval_minutes", 30)) rps = await fetch_int_setting("vk_requests_per_second", 3) timeout_total = await fetch_int_setting("vk_api_timeout_total_sec", 15) timeout_connect = await fetch_int_setting("vk_api_timeout_connect_sec", 5) @@ -540,7 +534,7 @@ class VKParserWorker: retry_min_delay = await fetch_float_setting("vk_api_retry_min_delay_sec", 2.0) retry_max_delay = await fetch_float_setting("vk_api_retry_max_delay_sec", 10.0) source_pause = max(0.0, await fetch_float_setting("parser_source_pause_sec", 0.0)) - sources = await self.active_sources(parser_interval_sec, site_default_interval) + sources = await self.active_sources(parser_interval_sec, site_interval) await self.heartbeat.beat(self.pool, meta={"sources": len(sources)}) if not sources: logger.debug("No sources due for parsing") diff --git a/tests/test_source_adapters.py b/tests/test_source_adapters.py index 122df24..ff519a9 100644 --- a/tests/test_source_adapters.py +++ b/tests/test_source_adapters.py @@ -38,7 +38,7 @@ class SourceAdapterTests(unittest.IsolatedAsyncioTestCase): client = SiteParserClient(FakeSession(), "http://worker", "token", "captcha", 30) items, state = await client.fetch({ "url": "https://example.test/rss.xml", - "settings_json": '{"format":"rss","access":"auto","interval_minutes":30}', + "settings_json": '{"format":"rss","access":"auto"}', "runtime_state_json": '{}', })