From fae611109e72d712cd91773e8324ffd756f0810d Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 5 Aug 2026 21:14:37 +0500 Subject: [PATCH] fix: poll instagram parser quickly while idle --- src/vk_parser_app/workers/insta_parser.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/vk_parser_app/workers/insta_parser.py b/src/vk_parser_app/workers/insta_parser.py index 1b2ce16..b998cac 100644 --- a/src/vk_parser_app/workers/insta_parser.py +++ b/src/vk_parser_app/workers/insta_parser.py @@ -404,16 +404,16 @@ class InstaParserWorker: logger.info("Parsed Instagram source {}: fetched={} known={} saved={}", username, len(posts), len(known_ids), saved) return saved - async def run_once(self) -> None: + async def run_once(self) -> bool: enabled = await is_worker_enabled(self.pool, WORKER_INSTA_PARSER) if not enabled: await self.heartbeat.beat(self.pool, status="disabled", force=True) - return + return False until = await self.cooldown_until() if until and until > datetime.now(tz=timezone.utc): await self.heartbeat.beat(self.pool, status="cooldown", meta={"until": until.isoformat()}) - return + return False login = str(await fetch_setting("insta_login", "") or "").strip() password = str(await fetch_setting("insta_password", "") or "").strip() @@ -425,26 +425,28 @@ class InstaParserWorker: if not login or not password: await self.heartbeat.beat(self.pool, status="missing_credentials") - return + return False try: await asyncio.to_thread(self.setup_client, login, password, proxy, session_path) except Exception as exc: await self.set_cooldown(cooldown_hours, f"login failed: {exc}") await send_parser_error_alert(WORKER_INSTA_PARSER, "login", str(exc)) - return + return False source = await self.active_source() if not source: await self.heartbeat.beat(self.pool, status="idle") - return + return False try: await self.heartbeat.beat(self.pool, status="running", meta={"source": source.get("external_id")}) await self.parse_source(source, fetch_count, request_pause_sec) + return True except (self.exceptions.get("user_not_found", NeverRaised), self.exceptions.get("private_account", NeverRaised)) as exc: await self.deactivate_source(int(source["id"]), str(exc)) await send_parser_error_alert(WORKER_INSTA_PARSER, str(source.get("external_id") or ""), str(exc)) + return True except ( self.exceptions.get("challenge", NeverRaised), self.exceptions.get("login_required", NeverRaised), @@ -454,18 +456,24 @@ class InstaParserWorker: await self.mark_source_error(int(source["id"]), str(exc)) await self.set_cooldown(cooldown_hours, str(exc)) await send_parser_error_alert(WORKER_INSTA_PARSER, str(source.get("external_id") or ""), str(exc)) + return True except Exception as exc: await self.mark_source_error(int(source["id"]), str(exc)) logger.exception("Unexpected Instagram source error {}: {}", source.get("external_id"), exc) + return True async def run_loop(self) -> None: await self.init() logger.info("{} started", WORKER_INSTA_PARSER) while True: try: - await self.run_once() + visited_source = await self.run_once() except Exception as exc: logger.exception("Instagram parser loop error: {}", exc) + visited_source = False + if not visited_source: + await asyncio.sleep(10) + continue base = max(1, await fetch_int_setting("insta_delay_base_minutes", 35)) spread = max(0, await fetch_int_setting("insta_delay_random_minutes", 5)) delay = max(60.0, base * 60 + random.uniform(-spread * 60, spread * 60))