Retry AI qualifier failures once
This commit is contained in:
@@ -804,6 +804,10 @@ heartbeat meta и отправляется Telegram-уведомлением п
|
|||||||
`daily_report_recipient_ids` через `daily_report_bot_token`, затем
|
`daily_report_recipient_ids` через `daily_report_bot_token`, затем
|
||||||
`tg_poster_bot_token`, затем `TG_BOT_TOKEN`.
|
`tg_poster_bot_token`, затем `TG_BOT_TOKEN`.
|
||||||
|
|
||||||
|
AI-квалификатор делает ровно один автоматический retry: после первого failed
|
||||||
|
batch пост возвращается в `qualification_status='pending'`, после второго failed
|
||||||
|
batch остаётся `failed` и больше не попадает в автоматический claim.
|
||||||
|
|
||||||
### 10.2. Вход в модель
|
### 10.2. Вход в модель
|
||||||
|
|
||||||
AI получает JSON-объект:
|
AI получает JSON-объект:
|
||||||
|
|||||||
@@ -104,6 +104,7 @@ As of 2026-07-29:
|
|||||||
- Sonnet 4.6 can return `rewrites` as a JSON string instead of a list. `validate_rewrites()` in `src/vk_parser_app/workers/ai_writer.py` now parses stringified `rewrites` before validation.
|
- Sonnet 4.6 can return `rewrites` as a JSON string instead of a list. `validate_rewrites()` in `src/vk_parser_app/workers/ai_writer.py` now parses stringified `rewrites` before validation.
|
||||||
- 2026-07-30 incident: post `2348` was retried 983 times by AI writer because automatic claim included `rewrite_status='failed'`; estimated internal cost was `$16.465977`. Live setting `ai_writer_enabled` was switched to `false` manually to stop spend. Automatic AI qualifier/writer claims must not include `failed`; failed posts require an explicit manual reset to `pending`. AI writer/qualifier keep running after a failed batch, show the error on `/workers`, and send a Telegram alert via daily-report/TG-poster bot recipients.
|
- 2026-07-30 incident: post `2348` was retried 983 times by AI writer because automatic claim included `rewrite_status='failed'`; estimated internal cost was `$16.465977`. Live setting `ai_writer_enabled` was switched to `false` manually to stop spend. Automatic AI qualifier/writer claims must not include `failed`; failed posts require an explicit manual reset to `pending`. AI writer/qualifier keep running after a failed batch, show the error on `/workers`, and send a Telegram alert via daily-report/TG-poster bot recipients.
|
||||||
- Sonnet 4.6 may return `category_id=18` (`Не целевой контент`) with empty `text` and the explanation in `notes`. This is valid AI rejection, not a writer error; `validate_rewrites()` allows short/empty text only for non-target category.
|
- Sonnet 4.6 may return `category_id=18` (`Не целевой контент`) with empty `text` and the explanation in `notes`. This is valid AI rejection, not a writer error; `validate_rewrites()` allows short/empty text only for non-target category.
|
||||||
|
- AI qualifier has exactly one automatic retry: after the first failed qualification batch, affected posts are returned to `qualification_status='pending'`; after the second failed batch for the same post, they stay `failed` and are not claimed again automatically.
|
||||||
|
|
||||||
## Prompt Architecture
|
## Prompt Architecture
|
||||||
|
|
||||||
|
|||||||
@@ -277,20 +277,40 @@ class AIQualifierWorker:
|
|||||||
usage.get("estimated_cost_usd"),
|
usage.get("estimated_cost_usd"),
|
||||||
)
|
)
|
||||||
|
|
||||||
async def mark_posts_failed(self, post_ids: list[int], error: str) -> None:
|
async def mark_posts_after_failed_batch(self, post_ids: list[int], error: str, max_attempts: int = 2) -> tuple[list[int], list[int]]:
|
||||||
if not post_ids:
|
if not post_ids:
|
||||||
return
|
return [], []
|
||||||
await self.pool.execute(
|
rows = await self.pool.fetch(
|
||||||
"""
|
"""
|
||||||
|
WITH target AS (
|
||||||
|
SELECT unnest($1::bigint[]) AS id
|
||||||
|
),
|
||||||
|
attempts AS (
|
||||||
|
SELECT t.id, COUNT(b.id) AS failed_batches
|
||||||
|
FROM target t
|
||||||
|
LEFT JOIN ai_qualification_batches b
|
||||||
|
ON b.status='failed'
|
||||||
|
AND b.request_json @> jsonb_build_array(jsonb_build_object('id', t.id))
|
||||||
|
GROUP BY t.id
|
||||||
|
)
|
||||||
UPDATE raw_posts
|
UPDATE raw_posts
|
||||||
SET qualification_status='failed',
|
SET qualification_status=CASE WHEN attempts.failed_batches < $3 THEN 'pending' ELSE 'failed' END,
|
||||||
qualification_reason=$2,
|
qualification_reason=CASE
|
||||||
|
WHEN attempts.failed_batches < $3 THEN $2 || ' (retry once)'
|
||||||
|
ELSE $2
|
||||||
|
END,
|
||||||
updated_at=NOW()
|
updated_at=NOW()
|
||||||
WHERE id=ANY($1::bigint[])
|
FROM attempts
|
||||||
|
WHERE raw_posts.id=attempts.id
|
||||||
|
RETURNING raw_posts.id, raw_posts.qualification_status
|
||||||
""",
|
""",
|
||||||
post_ids,
|
post_ids,
|
||||||
error[:1000],
|
error[:900],
|
||||||
|
max_attempts,
|
||||||
)
|
)
|
||||||
|
retry_ids = [int(row["id"]) for row in rows if row["qualification_status"] == "pending"]
|
||||||
|
failed_ids = [int(row["id"]) for row in rows if row["qualification_status"] == "failed"]
|
||||||
|
return retry_ids, failed_ids
|
||||||
|
|
||||||
async def report_failed_batch(self, model: str, post_ids: list[int], error: str) -> None:
|
async def report_failed_batch(self, model: str, post_ids: list[int], error: str) -> None:
|
||||||
meta = {"error": error[:300], "post_ids": post_ids, "model": model}
|
meta = {"error": error[:300], "post_ids": post_ids, "model": model}
|
||||||
@@ -419,9 +439,17 @@ class AIQualifierWorker:
|
|||||||
logger.info("AI qualifier batch done: id={} posts={} usage={}", batch_id, len(results), usage)
|
logger.info("AI qualifier batch done: id={} posts={} usage={}", batch_id, len(results), usage)
|
||||||
return True
|
return True
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
await self.mark_posts_failed(post_ids, str(exc))
|
|
||||||
await self.finish_batch(batch_id, "failed", error=str(exc))
|
await self.finish_batch(batch_id, "failed", error=str(exc))
|
||||||
await self.report_failed_batch(normalize_model(provider, model), post_ids, str(exc))
|
retry_ids, failed_ids = await self.mark_posts_after_failed_batch(post_ids, str(exc))
|
||||||
|
if failed_ids:
|
||||||
|
await self.report_failed_batch(normalize_model(provider, model), failed_ids, str(exc))
|
||||||
|
else:
|
||||||
|
await self.heartbeat.beat(
|
||||||
|
self.pool,
|
||||||
|
status="retrying_after_error",
|
||||||
|
meta={"error": str(exc)[:300], "post_ids": retry_ids, "model": normalize_model(provider, model)},
|
||||||
|
force=True,
|
||||||
|
)
|
||||||
logger.exception("AI qualifier batch failed: id={} error={}", batch_id, exc)
|
logger.exception("AI qualifier batch failed: id={} error={}", batch_id, exc)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user