Fix duplicate-post and timeout bugs in TG/MAX posting pipeline
- Stop blindly retrying send_photo/send_video/send_media_group and MAX send_message on ambiguous network timeouts - a timeout doesn't prove the message wasn't delivered, and retrying risked posting duplicates (observed live: a video posted 3-5x after repeated timeout retries). - Raise/rework timeouts that were too short for real large-file transfer speeds: TG media upload timeout, and yt-dlp download now uses stall detection (killed only on true silence) instead of a flat ceiling that was cutting off legitimately slow-but-successful video downloads. - Fix sendRichMessage: ok=true with an unparseable message_id no longer triggers a fallback send (Telegram already created the message). - MAX: videos over the documented 250MB cap are now sent as a "watch via link" note instead of silently failing the upload. - Add PRAGMA busy_timeout to all DB connections. - Remove unused MAX_MEDIA_CHANNEL_ID (MAX's /uploads returns a portable token directly, no staging channel needed, unlike Telegram). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
+25
-13
@@ -2,8 +2,8 @@ from __future__ import annotations
|
||||
from loguru import logger
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Any, AsyncIterator, Optional
|
||||
import aiosqlite
|
||||
try:
|
||||
from .config import settings
|
||||
@@ -11,13 +11,26 @@ except (ImportError, ValueError):
|
||||
from config import settings
|
||||
|
||||
|
||||
# Each method opens its own short-lived connection (no pooling). busy_timeout
|
||||
# means a second connection opened concurrently (e.g. inspecting the DB by
|
||||
# hand with sqlite3 while the service runs) waits instead of immediately
|
||||
# failing with "database is locked".
|
||||
_BUSY_TIMEOUT_MS = 5000
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self, db_path: Optional[str] = None) -> None:
|
||||
self.db_path = str(settings.db_path if db_path is None else db_path)
|
||||
|
||||
@asynccontextmanager
|
||||
async def _connect(self) -> AsyncIterator[aiosqlite.Connection]:
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
await db.execute(f"PRAGMA busy_timeout = {_BUSY_TIMEOUT_MS};")
|
||||
yield db
|
||||
|
||||
async def init(self) -> None:
|
||||
logger.info("Initializing database at {}", self.db_path)
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
async with self._connect() as db:
|
||||
await db.execute("PRAGMA journal_mode=WAL;")
|
||||
await db.execute(
|
||||
"""
|
||||
@@ -58,7 +71,7 @@ class Database:
|
||||
await db.commit()
|
||||
|
||||
async def has_any_posts(self, owner_id: int) -> bool:
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
async with self._connect() as db:
|
||||
cursor = await db.execute(
|
||||
"SELECT 1 FROM posts WHERE vk_owner_id = ? LIMIT 1",
|
||||
(owner_id,),
|
||||
@@ -76,7 +89,7 @@ class Database:
|
||||
reason: str = "bootstrap_initial_skip",
|
||||
) -> None:
|
||||
raw_json = json.dumps(raw_data, ensure_ascii=False)
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
async with self._connect() as db:
|
||||
await db.execute(
|
||||
"""
|
||||
INSERT INTO posts (vk_owner_id, vk_post_id, posted_at, text, raw_json, tg_status, max_status, tg_error, max_error)
|
||||
@@ -90,23 +103,22 @@ class Database:
|
||||
await db.commit()
|
||||
|
||||
async def is_post_processed(self, owner_id: int, post_id: int) -> bool:
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
async with self._connect() as db:
|
||||
db.row_factory = aiosqlite.Row
|
||||
cursor = await db.execute(
|
||||
"SELECT id, tg_status, max_status FROM posts WHERE vk_owner_id = ? AND vk_post_id = ?",
|
||||
"SELECT tg_status, max_status FROM posts WHERE vk_owner_id = ? AND vk_post_id = ?",
|
||||
(owner_id, post_id),
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
if not row:
|
||||
return False
|
||||
# If already published on both or marked skipped, it's processed
|
||||
return bool(
|
||||
row["tg_status"] in ("published", "skipped")
|
||||
and row["max_status"] in ("published", "skipped")
|
||||
)
|
||||
|
||||
async def get_post(self, owner_id: int, post_id: int) -> Optional[dict[str, Any]]:
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
async with self._connect() as db:
|
||||
db.row_factory = aiosqlite.Row
|
||||
cursor = await db.execute(
|
||||
"SELECT * FROM posts WHERE vk_owner_id = ? AND vk_post_id = ?",
|
||||
@@ -124,7 +136,7 @@ class Database:
|
||||
raw_data: dict[str, Any],
|
||||
) -> int:
|
||||
raw_json = json.dumps(raw_data, ensure_ascii=False)
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
async with self._connect() as db:
|
||||
cursor = await db.execute(
|
||||
"""
|
||||
INSERT INTO posts (vk_owner_id, vk_post_id, posted_at, text, raw_json)
|
||||
@@ -149,7 +161,7 @@ class Database:
|
||||
error: Optional[str] = None,
|
||||
) -> None:
|
||||
msg_str = ",".join(str(m) for m in message_ids) if message_ids else None
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
async with self._connect() as db:
|
||||
await db.execute(
|
||||
"""
|
||||
UPDATE posts
|
||||
@@ -173,7 +185,7 @@ class Database:
|
||||
error: Optional[str] = None,
|
||||
) -> None:
|
||||
msg_str = ",".join(message_ids) if message_ids else None
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
async with self._connect() as db:
|
||||
await db.execute(
|
||||
"""
|
||||
UPDATE posts
|
||||
@@ -196,7 +208,7 @@ class Database:
|
||||
status: str = "ok",
|
||||
error: Optional[str] = None,
|
||||
) -> None:
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
async with self._connect() as db:
|
||||
await db.execute(
|
||||
"""
|
||||
INSERT INTO publication_runs (found_count, published_tg_count, published_max_count, status, error)
|
||||
|
||||
Reference in New Issue
Block a user