exostring/new_vk_parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: keep site parsing interval global

Browse Source
This commit is contained in:
Your Name
2026-08-10 21:32:46 +05:00
parent 5417737c08
commit 7889d4d376
8 changed files with 25 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
db/migrations/047_site_parser_schedule.sql
+2 -2
View File
@@ -3,8 +3,8 @@ VALUES (
'site_parser_default_interval_minutes',
'30'::jsonb,
'int',
'Интервал по умолчанию, мин',
'Как часто проверять сайт, если у источника не задан свой интервал.',
'Интервал парсинга сайтов, мин',
'Как часто Site Parser проверяет все активные сайты.',
'Site Parser'
)
ON CONFLICT (key) DO NOTHING;
db/migrations/048_site_parser_global_interval.sql
+14
View File
@@ -0,0 +1,14 @@
INSERT INTO app_settings(key, value_json, value_type, title, description, category, updated_by)
SELECT
'site_parser_interval_minutes',
value_json,
'int',
'Интервал парсинга сайтов, мин',
'Как часто Site Parser проверяет все активные сайты.',
'Site Parser',
updated_by
FROM app_settings
WHERE key='site_parser_default_interval_minutes'
ON CONFLICT (key) DO NOTHING;
DELETE FROM app_settings WHERE key='site_parser_default_interval_minutes';
src/vk_parser_app/admin.py
+2 -16
View File
@@ -371,7 +371,7 @@ SETTING_ORDER = {
"site_parser_token",
"site_parser_rucaptcha_token",
"site_parser_timeout_sec",
"site_parser_default_interval_minutes",
"site_parser_interval_minutes",
],
"VK": [
"vk_requests_per_second",
@@ -2359,14 +2359,7 @@ async def source_new(request: Request):
return redirect("/login")
return templates.TemplateResponse(
"source_form.html",
base_context(
request,
user,
source=None,
action="/sources/new",
title="Новый источник",
site_default_interval=await fetch_int_setting("site_parser_default_interval_minutes", 30),
),
base_context(request, user, source=None, action="/sources/new", title="Новый источник"),
)
@@ -2381,7 +2374,6 @@ async def source_create(
active: str = Form("off"),
priority: int = Form(100),
settings_json: str = Form("{}"),
interval_minutes: int = Form(30),
):
user = await get_current_user(request)
if not user:
@@ -2394,8 +2386,6 @@ async def source_create(
source_settings = json.loads(settings_json or "{}")
if not isinstance(source_settings, dict):
raise ValueError("Настройки должны быть JSON-объектом")
if platform == PLATFORM_SITE:
source_settings["interval_minutes"] = interval_minutes
validate_source_config(platform, source_settings)
except (json.JSONDecodeError, ValueError) as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
@@ -2461,7 +2451,6 @@ async def source_edit(request: Request, source_id: int):
source={**dict(source), "settings_json": json_object(source["settings_json"])},
action=f"/sources/{source_id}/edit",
title=f"Источник #{source_id}",
site_default_interval=await fetch_int_setting("site_parser_default_interval_minutes", 30),
),
)
@@ -2478,7 +2467,6 @@ async def source_update(
active: str = Form("off"),
priority: int = Form(100),
settings_json: str = Form("{}"),
interval_minutes: int = Form(30),
):
user = await get_current_user(request)
if not user:
@@ -2491,8 +2479,6 @@ async def source_update(
source_settings = json.loads(settings_json or "{}")
if not isinstance(source_settings, dict):
raise ValueError("Настройки должны быть JSON-объектом")
if platform == PLATFORM_SITE:
source_settings["interval_minutes"] = interval_minutes
validate_source_config(platform, source_settings)
except (json.JSONDecodeError, ValueError) as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
src/vk_parser_app/source_adapters.py
-8
View File
@@ -55,14 +55,6 @@ def validate_source_config(platform: str, config: dict[str, Any]) -> None:
raise ValueError("max_items должен быть целым числом") from exc
if not 1 <= max_items <= 100:
raise ValueError("max_items должен быть от 1 до 100")
try:
interval_minutes = int(config.get("interval_minutes", 30))
except (TypeError, ValueError) as exc:
raise ValueError("interval_minutes должен быть целым числом") from exc
if not 1 <= interval_minutes <= 10080:
raise ValueError("interval_minutes должен быть от 1 до 10080")
def _posted_at(value: Any) -> datetime:
try:
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
src/vk_parser_app/templates/source_form.html
+1 -4
View File
@@ -42,8 +42,6 @@
</div>
<div id="site-config" class="form-control md:col-span-2">
<label class="label"><span class="label-text font-bold">Проверять каждые, минут</span></label>
<input name="interval_minutes" type="number" min="1" max="10080" value="{{ source.settings_json.get('interval_minutes', site_default_interval) if source and source.settings_json else site_default_interval }}" class="input input-bordered w-full max-w-xs mb-4">
<label class="label"><span class="label-text font-bold">Конфигурация JSON</span></label>
<textarea name="settings_json" rows="8" class="textarea textarea-bordered w-full font-mono" placeholder='{"format":"rss","access":"auto","max_items":20}'>{{ source.settings_json | tojson(indent=2) if source else '{}' }}</textarea>
<details class="mt-2 text-sm text-base-content/70">
@@ -51,8 +49,7 @@
<pre class="mt-2 p-3 bg-base-200 overflow-x-auto">{
"format": "rss",
"access": "auto",
"max_items": 20,
"interval_minutes": 30
"max_items": 20
}</pre>
<p class="mt-2"><code>access</code>: <code>auto</code> сначала пробует обычный запрос и при Cloudflare использует RuCaptcha; <code>http</code> запрещает браузер; <code>cloudflare</code> сразу запускает браузер.</p>
</details>
src/vk_parser_app/templates/source_row.html
-1
View File
@@ -7,7 +7,6 @@
</div>
<div class="sm:hidden text-[10px] font-mono text-app-textMuted mt-1">#{{ s.id }}</div>
<div class="text-[10px] font-mono text-app-textMuted mt-1 truncate" title="External ID">{{ s.external_id or "" }}</div>
{% if s.platform == "site" %}<div class="text-[10px] text-app-textMuted mt-1">Каждые {{ s.settings_json.get('interval_minutes', 30) }} мин.</div>{% endif %}
<div class="sm:hidden text-[10px] font-mono text-blue-400 mt-1 truncate">#{{ s.tag or "—" }}</div>
<a href="{{ s.url }}" target="_blank" class="lg:hidden text-[10px] text-app-textMuted hover:text-white hover:underline truncate block mt-1" title="{{ s.url }}">{{ s.url }}</a>
</td>
src/vk_parser_app/workers/parser.py
+5 -11
View File
@@ -55,7 +55,7 @@ class VKParserWorker:
async def init(self) -> None:
self.pool = await get_pool()
async def active_sources(self, vk_interval_sec: int, site_default_interval_minutes: int) -> list[dict]:
async def active_sources(self, vk_interval_sec: int, site_interval_minutes: int) -> list[dict]:
rows = await self.pool.fetch(
"""
SELECT *
@@ -68,13 +68,7 @@ class VKParserWorker:
OR (platform=$2 AND last_checked_at <= NOW() - $3::double precision * INTERVAL '1 second')
OR (
platform=$4
AND last_checked_at <= NOW() - (
CASE
WHEN settings_json->>'interval_minutes' ~ '^[0-9]+$'
THEN (settings_json->>'interval_minutes')::double precision
ELSE $5::double precision
END
) * INTERVAL '1 minute'
AND last_checked_at <= NOW() - $5::double precision * INTERVAL '1 minute'
)
)
ORDER BY last_checked_at NULLS FIRST, priority ASC, id ASC
@@ -83,7 +77,7 @@ class VKParserWorker:
PLATFORM_VK,
vk_interval_sec,
PLATFORM_SITE,
site_default_interval_minutes,
site_interval_minutes,
)
return [dict(r) for r in rows]
@@ -531,7 +525,7 @@ class VKParserWorker:
return
parser_interval_sec = max(10, await fetch_int_setting("parser_interval_sec", 300))
site_default_interval = max(1, await fetch_int_setting("site_parser_default_interval_minutes", 30))
site_interval = max(1, await fetch_int_setting("site_parser_interval_minutes", 30))
rps = await fetch_int_setting("vk_requests_per_second", 3)
timeout_total = await fetch_int_setting("vk_api_timeout_total_sec", 15)
timeout_connect = await fetch_int_setting("vk_api_timeout_connect_sec", 5)
@@ -540,7 +534,7 @@ class VKParserWorker:
retry_min_delay = await fetch_float_setting("vk_api_retry_min_delay_sec", 2.0)
retry_max_delay = await fetch_float_setting("vk_api_retry_max_delay_sec", 10.0)
source_pause = max(0.0, await fetch_float_setting("parser_source_pause_sec", 0.0))
sources = await self.active_sources(parser_interval_sec, site_default_interval)
sources = await self.active_sources(parser_interval_sec, site_interval)
await self.heartbeat.beat(self.pool, meta={"sources": len(sources)})
if not sources:
logger.debug("No sources due for parsing")
tests/test_source_adapters.py
+1 -1
View File
@@ -38,7 +38,7 @@ class SourceAdapterTests(unittest.IsolatedAsyncioTestCase):
client = SiteParserClient(FakeSession(), "http://worker", "token", "captcha", 30)
items, state = await client.fetch({
"url": "https://example.test/rss.xml",
"settings_json": '{"format":"rss","access":"auto","interval_minutes":30}',
"settings_json": '{"format":"rss","access":"auto"}',
"runtime_state_json": '{}',
})