From bb67e9509bbf9cf2e6006d754a1e191d4ebdfe27 Mon Sep 17 00:00:00 2001
From: Your Name <you@example.com>
Date: Thu, 30 Jul 2026 09:01:53 +0500
Subject: [PATCH] Disable AI workers after failed batch

---
 PROJECT.md                                |  3 +++
 STATE.md                                  |  2 +-
 src/vk_parser_app/workers/ai_qualifier.py | 12 ++++++++++++
 src/vk_parser_app/workers/ai_writer.py    | 12 ++++++++++++
 4 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/PROJECT.md b/PROJECT.md
index 1973316..3ecaee3 100644
--- a/PROJECT.md
+++ b/PROJECT.md
@@ -794,6 +794,9 @@ AI получает JSON-массив:
 2026-07-30 один пост (`id=2348`) был платно отправлен в Sonnet 4.6 983 раза,
 потому что failed снова попадал в claim и цикл сразу продолжал работу. Failed
 посты нужно возвращать в `pending` только явным ручным действием.
+После любого failed AI batch worker дополнительно выключает свой флаг
+`ai_writer_enabled=false` / `ai_qualifier_enabled=false`; включать обратно
+только после просмотра ошибки.
 
 ### 10.2. Вход в модель
 
diff --git a/STATE.md b/STATE.md
index cb4ddcd..9a89212 100644
--- a/STATE.md
+++ b/STATE.md
@@ -102,7 +102,7 @@ As of 2026-07-29:
 
 - Current live writer model setting checked on 2026-07-29: `ai_writer_model = anthropic/claude-sonnet-4-6`.
 - Sonnet 4.6 can return `rewrites` as a JSON string instead of a list. `validate_rewrites()` in `src/vk_parser_app/workers/ai_writer.py` now parses stringified `rewrites` before validation.
-- 2026-07-30 incident: post `2348` was retried 983 times by AI writer because automatic claim included `rewrite_status='failed'`; estimated internal cost was `$16.465977`. Live setting `ai_writer_enabled` was switched to `false` to stop spend. Automatic AI qualifier/writer claims must not include `failed`; failed posts require an explicit manual reset to `pending`.
+- 2026-07-30 incident: post `2348` was retried 983 times by AI writer because automatic claim included `rewrite_status='failed'`; estimated internal cost was `$16.465977`. Live setting `ai_writer_enabled` was switched to `false` to stop spend. Automatic AI qualifier/writer claims must not include `failed`; failed posts require an explicit manual reset to `pending`. AI writer/qualifier now also set their own `*_enabled=false` after any failed AI batch, so one broken prompt/model/post cannot burn money indefinitely.
 
 ## Prompt Architecture
 
diff --git a/src/vk_parser_app/workers/ai_qualifier.py b/src/vk_parser_app/workers/ai_qualifier.py
index 00fed8a..5423ef5 100644
--- a/src/vk_parser_app/workers/ai_qualifier.py
+++ b/src/vk_parser_app/workers/ai_qualifier.py
@@ -291,6 +291,17 @@ class AIQualifierWorker:
             error[:1000],
         )
 
+    async def disable_after_error(self, error: str) -> None:
+        await self.pool.execute(
+            """
+            UPDATE app_settings
+            SET value_json='false'::jsonb,
+                updated_at=NOW()
+            WHERE key='ai_qualifier_enabled'
+            """
+        )
+        await self.heartbeat.beat(self.pool, status="disabled_after_error", meta={"error": error[:300]}, force=True)
+
     async def apply_results(self, batch_id: int, results: list[dict], model: str, prompt: str, min_score: int) -> None:
         for item in results:
             score = int(item["score"])
@@ -412,6 +423,7 @@ class AIQualifierWorker:
         except Exception as exc:
             await self.mark_posts_failed(post_ids, str(exc))
             await self.finish_batch(batch_id, "failed", error=str(exc))
+            await self.disable_after_error(str(exc))
             logger.exception("AI qualifier batch failed: id={} error={}", batch_id, exc)
             return True
 
diff --git a/src/vk_parser_app/workers/ai_writer.py b/src/vk_parser_app/workers/ai_writer.py
index 4d77af8..2fa2c8d 100644
--- a/src/vk_parser_app/workers/ai_writer.py
+++ b/src/vk_parser_app/workers/ai_writer.py
@@ -372,6 +372,17 @@ class AIWriterWorker:
             error[:1000],
         )
 
+    async def disable_after_error(self, error: str) -> None:
+        await self.pool.execute(
+            """
+            UPDATE app_settings
+            SET value_json='false'::jsonb,
+                updated_at=NOW()
+            WHERE key='ai_writer_enabled'
+            """
+        )
+        await self.heartbeat.beat(self.pool, status="disabled_after_error", meta={"error": error[:300]}, force=True)
+
     async def apply_rewrites(self, batch_id: int, rewrites: list[dict], model: str, prompt: str) -> None:
         for item in rewrites:
             final_text = build_publication_text(item["text"], item["category_tag"], item["source_tag"])
@@ -518,6 +529,7 @@ class AIWriterWorker:
         except Exception as exc:
             await self.mark_posts_failed(post_ids, str(exc))
             await self.finish_batch(batch_id, "failed", response=response, error=str(exc), usage=usage)
+            await self.disable_after_error(str(exc))
             logger.exception("AI writer batch failed: id={} error={}", batch_id, exc)
             return True