diff --git a/PROJECT.md b/PROJECT.md index a247912..1db6f2a 100644 --- a/PROJECT.md +++ b/PROJECT.md @@ -794,11 +794,11 @@ AI получает JSON-массив: 2026-07-30 один пост (`id=2348`) был платно отправлен в Sonnet 4.6 983 раза, потому что failed снова попадал в claim и цикл сразу продолжал работу. Failed посты нужно возвращать в `pending` только явным ручным действием. -После любого failed AI batch worker дополнительно выключает свой флаг -`ai_writer_enabled=false` / `ai_qualifier_enabled=false`; включать обратно -только после просмотра ошибки. Ошибка показывается на `/workers` из heartbeat -meta и отправляется Telegram-уведомлением получателям `daily_report_recipient_ids` -через `daily_report_bot_token`, затем `tg_poster_bot_token`, затем `TG_BOT_TOKEN`. +После failed AI batch worker помечает конкретные посты как `failed` и продолжает +обрабатывать другие `pending` посты. Ошибка показывается на `/workers` из +heartbeat meta и отправляется Telegram-уведомлением получателям +`daily_report_recipient_ids` через `daily_report_bot_token`, затем +`tg_poster_bot_token`, затем `TG_BOT_TOKEN`. ### 10.2. Вход в модель diff --git a/STATE.md b/STATE.md index 7c4cbd1..ea03f7d 100644 --- a/STATE.md +++ b/STATE.md @@ -102,7 +102,7 @@ As of 2026-07-29: - Current live writer model setting checked on 2026-07-29: `ai_writer_model = anthropic/claude-sonnet-4-6`. - Sonnet 4.6 can return `rewrites` as a JSON string instead of a list. `validate_rewrites()` in `src/vk_parser_app/workers/ai_writer.py` now parses stringified `rewrites` before validation. -- 2026-07-30 incident: post `2348` was retried 983 times by AI writer because automatic claim included `rewrite_status='failed'`; estimated internal cost was `$16.465977`. Live setting `ai_writer_enabled` was switched to `false` to stop spend. Automatic AI qualifier/writer claims must not include `failed`; failed posts require an explicit manual reset to `pending`. AI writer/qualifier now also set their own `*_enabled=false` after any failed AI batch, show the error on `/workers`, and send a Telegram alert via daily-report/TG-poster bot recipients, so one broken prompt/model/post cannot burn money indefinitely or fail silently. +- 2026-07-30 incident: post `2348` was retried 983 times by AI writer because automatic claim included `rewrite_status='failed'`; estimated internal cost was `$16.465977`. Live setting `ai_writer_enabled` was switched to `false` manually to stop spend. Automatic AI qualifier/writer claims must not include `failed`; failed posts require an explicit manual reset to `pending`. AI writer/qualifier keep running after a failed batch, show the error on `/workers`, and send a Telegram alert via daily-report/TG-poster bot recipients. ## Prompt Architecture diff --git a/src/vk_parser_app/workers/ai_alerts.py b/src/vk_parser_app/workers/ai_alerts.py index 2fdbf4f..b687b01 100644 --- a/src/vk_parser_app/workers/ai_alerts.py +++ b/src/vk_parser_app/workers/ai_alerts.py @@ -39,7 +39,7 @@ def parse_recipients(value: Any) -> list[int]: return recipients -async def send_ai_worker_disabled_alert(worker_name: str, model: str, post_ids: list[int], error: str) -> None: +async def send_ai_worker_error_alert(worker_name: str, model: str, post_ids: list[int], error: str) -> None: token = ( str(await fetch_setting("daily_report_bot_token", "") or "").strip() or str(await fetch_setting("tg_poster_bot_token", "") or "").strip() @@ -52,7 +52,7 @@ async def send_ai_worker_disabled_alert(worker_name: str, model: str, post_ids: post_part = ", ".join(str(post_id) for post_id in post_ids) if post_ids else "-" text = ( - "AI worker auto-disabled\n" + "AI worker batch failed\n" f"worker: {worker_name}\n" f"model: {model or '-'}\n" f"posts: {post_part}\n" diff --git a/src/vk_parser_app/workers/ai_qualifier.py b/src/vk_parser_app/workers/ai_qualifier.py index 4eec999..18c982d 100644 --- a/src/vk_parser_app/workers/ai_qualifier.py +++ b/src/vk_parser_app/workers/ai_qualifier.py @@ -15,7 +15,7 @@ from ..constants import WORKER_AI_QUALIFIER from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool from ..heartbeat import HeartbeatReporter from ..jobs import is_worker_enabled -from .ai_alerts import send_ai_worker_disabled_alert +from .ai_alerts import send_ai_worker_error_alert def now_utc() -> datetime: @@ -292,19 +292,11 @@ class AIQualifierWorker: error[:1000], ) - async def disable_after_error(self, model: str, post_ids: list[int], error: str) -> None: - await self.pool.execute( - """ - UPDATE app_settings - SET value_json='false'::jsonb, - updated_at=NOW() - WHERE key='ai_qualifier_enabled' - """ - ) + async def report_failed_batch(self, model: str, post_ids: list[int], error: str) -> None: meta = {"error": error[:300], "post_ids": post_ids, "model": model} - await self.heartbeat.beat(self.pool, status="disabled_after_error", meta=meta, force=True) + await self.heartbeat.beat(self.pool, status="error", meta=meta, force=True) try: - await send_ai_worker_disabled_alert(WORKER_AI_QUALIFIER, model, post_ids, error) + await send_ai_worker_error_alert(WORKER_AI_QUALIFIER, model, post_ids, error) except Exception as alert_exc: logger.warning("AI qualifier alert failed: {}", alert_exc) @@ -429,7 +421,7 @@ class AIQualifierWorker: except Exception as exc: await self.mark_posts_failed(post_ids, str(exc)) await self.finish_batch(batch_id, "failed", error=str(exc)) - await self.disable_after_error(normalize_model(provider, model), post_ids, str(exc)) + await self.report_failed_batch(normalize_model(provider, model), post_ids, str(exc)) logger.exception("AI qualifier batch failed: id={} error={}", batch_id, exc) return True diff --git a/src/vk_parser_app/workers/ai_writer.py b/src/vk_parser_app/workers/ai_writer.py index 843cc72..39c29f0 100644 --- a/src/vk_parser_app/workers/ai_writer.py +++ b/src/vk_parser_app/workers/ai_writer.py @@ -17,7 +17,7 @@ from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fet from ..heartbeat import HeartbeatReporter from ..jobs import is_worker_enabled from ..text_utils import build_publication_text, normalize_hash_tag, parse_categories -from .ai_alerts import send_ai_worker_disabled_alert +from .ai_alerts import send_ai_worker_error_alert NON_TARGET_CATEGORY_ID = 18 @@ -373,19 +373,11 @@ class AIWriterWorker: error[:1000], ) - async def disable_after_error(self, model: str, post_ids: list[int], error: str) -> None: - await self.pool.execute( - """ - UPDATE app_settings - SET value_json='false'::jsonb, - updated_at=NOW() - WHERE key='ai_writer_enabled' - """ - ) + async def report_failed_batch(self, model: str, post_ids: list[int], error: str) -> None: meta = {"error": error[:300], "post_ids": post_ids, "model": model} - await self.heartbeat.beat(self.pool, status="disabled_after_error", meta=meta, force=True) + await self.heartbeat.beat(self.pool, status="error", meta=meta, force=True) try: - await send_ai_worker_disabled_alert(WORKER_AI_WRITER, model, post_ids, error) + await send_ai_worker_error_alert(WORKER_AI_WRITER, model, post_ids, error) except Exception as alert_exc: logger.warning("AI writer alert failed: {}", alert_exc) @@ -535,7 +527,7 @@ class AIWriterWorker: except Exception as exc: await self.mark_posts_failed(post_ids, str(exc)) await self.finish_batch(batch_id, "failed", response=response, error=str(exc), usage=usage) - await self.disable_after_error(normalize_model(provider, model), post_ids, str(exc)) + await self.report_failed_batch(normalize_model(provider, model), post_ids, str(exc)) logger.exception("AI writer batch failed: id={} error={}", batch_id, exc) return True