Keep AI workers running after failed batch

PROJECT.md

@@ -794,11 +794,11 @@ AI получает JSON-массив:
 2026-07-30 один пост (`id=2348`) был платно отправлен в Sonnet 4.6 983 раза,
 потому что failed снова попадал в claim и цикл сразу продолжал работу. Failed
 посты нужно возвращать в `pending` только явным ручным действием.
-После любого failed AI batch worker дополнительно выключает свой флаг
+После failed AI batch worker помечает конкретные посты как `failed` и продолжает
-`ai_writer_enabled=false` / `ai_qualifier_enabled=false`; включать обратно
+обрабатывать другие `pending` посты. Ошибка показывается на `/workers` из
-только после просмотра ошибки. Ошибка показывается на `/workers` из heartbeat
+heartbeat meta и отправляется Telegram-уведомлением получателям
-meta и отправляется Telegram-уведомлением получателям `daily_report_recipient_ids`
+`daily_report_recipient_ids` через `daily_report_bot_token`, затем
-через `daily_report_bot_token`, затем `tg_poster_bot_token`, затем `TG_BOT_TOKEN`.
+`tg_poster_bot_token`, затем `TG_BOT_TOKEN`.
 
 ### 10.2. Вход в модель
 

STATE.md

@@ -102,7 +102,7 @@ As of 2026-07-29:
 
 - Current live writer model setting checked on 2026-07-29: `ai_writer_model = anthropic/claude-sonnet-4-6`.
 - Sonnet 4.6 can return `rewrites` as a JSON string instead of a list. `validate_rewrites()` in `src/vk_parser_app/workers/ai_writer.py` now parses stringified `rewrites` before validation.
-- 2026-07-30 incident: post `2348` was retried 983 times by AI writer because automatic claim included `rewrite_status='failed'`; estimated internal cost was `$16.465977`. Live setting `ai_writer_enabled` was switched to `false` to stop spend. Automatic AI qualifier/writer claims must not include `failed`; failed posts require an explicit manual reset to `pending`. AI writer/qualifier now also set their own `*_enabled=false` after any failed AI batch, show the error on `/workers`, and send a Telegram alert via daily-report/TG-poster bot recipients, so one broken prompt/model/post cannot burn money indefinitely or fail silently.
+- 2026-07-30 incident: post `2348` was retried 983 times by AI writer because automatic claim included `rewrite_status='failed'`; estimated internal cost was `$16.465977`. Live setting `ai_writer_enabled` was switched to `false` manually to stop spend. Automatic AI qualifier/writer claims must not include `failed`; failed posts require an explicit manual reset to `pending`. AI writer/qualifier keep running after a failed batch, show the error on `/workers`, and send a Telegram alert via daily-report/TG-poster bot recipients.
 
 ## Prompt Architecture
 

src/vk_parser_app/workers/ai_alerts.py

@@ -39,7 +39,7 @@ def parse_recipients(value: Any) -> list[int]:
     return recipients
 
 
-async def send_ai_worker_disabled_alert(worker_name: str, model: str, post_ids: list[int], error: str) -> None:
+async def send_ai_worker_error_alert(worker_name: str, model: str, post_ids: list[int], error: str) -> None:
     token = (
         str(await fetch_setting("daily_report_bot_token", "") or "").strip()
         or str(await fetch_setting("tg_poster_bot_token", "") or "").strip()
@@ -52,7 +52,7 @@ async def send_ai_worker_disabled_alert(worker_name: str, model: str, post_ids:
 
     post_part = ", ".join(str(post_id) for post_id in post_ids) if post_ids else "-"
     text = (
-        "AI worker auto-disabled\n"
+        "AI worker batch failed\n"
         f"worker: {worker_name}\n"
         f"model: {model or '-'}\n"
         f"posts: {post_part}\n"

src/vk_parser_app/workers/ai_qualifier.py

@@ -15,7 +15,7 @@ from ..constants import WORKER_AI_QUALIFIER
 from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool
 from ..heartbeat import HeartbeatReporter
 from ..jobs import is_worker_enabled
-from .ai_alerts import send_ai_worker_disabled_alert
+from .ai_alerts import send_ai_worker_error_alert
 
 
 def now_utc() -> datetime:
@@ -292,19 +292,11 @@ class AIQualifierWorker:
             error[:1000],
         )
 
-    async def disable_after_error(self, model: str, post_ids: list[int], error: str) -> None:
+    async def report_failed_batch(self, model: str, post_ids: list[int], error: str) -> None:
-        await self.pool.execute(
-            """
-            UPDATE app_settings
-            SET value_json='false'::jsonb,
-                updated_at=NOW()
-            WHERE key='ai_qualifier_enabled'
-            """
-        )
         meta = {"error": error[:300], "post_ids": post_ids, "model": model}
-        await self.heartbeat.beat(self.pool, status="disabled_after_error", meta=meta, force=True)
+        await self.heartbeat.beat(self.pool, status="error", meta=meta, force=True)
         try:
-            await send_ai_worker_disabled_alert(WORKER_AI_QUALIFIER, model, post_ids, error)
+            await send_ai_worker_error_alert(WORKER_AI_QUALIFIER, model, post_ids, error)
         except Exception as alert_exc:
             logger.warning("AI qualifier alert failed: {}", alert_exc)
 
@@ -429,7 +421,7 @@ class AIQualifierWorker:
         except Exception as exc:
             await self.mark_posts_failed(post_ids, str(exc))
             await self.finish_batch(batch_id, "failed", error=str(exc))
-            await self.disable_after_error(normalize_model(provider, model), post_ids, str(exc))
+            await self.report_failed_batch(normalize_model(provider, model), post_ids, str(exc))
             logger.exception("AI qualifier batch failed: id={} error={}", batch_id, exc)
             return True
 

src/vk_parser_app/workers/ai_writer.py

@@ -17,7 +17,7 @@ from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fet
 from ..heartbeat import HeartbeatReporter
 from ..jobs import is_worker_enabled
 from ..text_utils import build_publication_text, normalize_hash_tag, parse_categories
-from .ai_alerts import send_ai_worker_disabled_alert
+from .ai_alerts import send_ai_worker_error_alert
 
 
 NON_TARGET_CATEGORY_ID = 18
@@ -373,19 +373,11 @@ class AIWriterWorker:
             error[:1000],
         )
 
-    async def disable_after_error(self, model: str, post_ids: list[int], error: str) -> None:
+    async def report_failed_batch(self, model: str, post_ids: list[int], error: str) -> None:
-        await self.pool.execute(
-            """
-            UPDATE app_settings
-            SET value_json='false'::jsonb,
-                updated_at=NOW()
-            WHERE key='ai_writer_enabled'
-            """
-        )
         meta = {"error": error[:300], "post_ids": post_ids, "model": model}
-        await self.heartbeat.beat(self.pool, status="disabled_after_error", meta=meta, force=True)
+        await self.heartbeat.beat(self.pool, status="error", meta=meta, force=True)
         try:
-            await send_ai_worker_disabled_alert(WORKER_AI_WRITER, model, post_ids, error)
+            await send_ai_worker_error_alert(WORKER_AI_WRITER, model, post_ids, error)
         except Exception as alert_exc:
             logger.warning("AI writer alert failed: {}", alert_exc)
 
@@ -535,7 +527,7 @@ class AIWriterWorker:
         except Exception as exc:
             await self.mark_posts_failed(post_ids, str(exc))
             await self.finish_batch(batch_id, "failed", response=response, error=str(exc), usage=usage)
-            await self.disable_after_error(normalize_model(provider, model), post_ids, str(exc))
+            await self.report_failed_batch(normalize_model(provider, model), post_ids, str(exc))
             logger.exception("AI writer batch failed: id={} error={}", batch_id, exc)
             return True