exostring/redairsoft_vk_gt_max_sender
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add multi-route donor->recipient support, fix VK hashtag truncation

Browse Source 
Posts can now be sourced from multiple VK groups, each routed to its own
Telegram/MAX destination(s) with independent on/off switches, configured
via data/routes.json (supports // line comments). Falls back to a single
route auto-generated from the legacy VK_SOURCE/TG_CHAT_ID/MAX_CHAT_ID env
vars if routes.json doesn't exist yet, so existing deployments keep working.

Routes are processed strictly sequentially within a cycle (no concurrency)
to keep flood control on VK/TG/MAX correct, since bot tokens are shared
across routes. DB schema gains route_id in the posts uniqueness key so the
same VK donor can safely feed multiple routes without status collisions.

Also removes the trailing-hashtag-stripping logic in text_formatter, which
was silently deleting VK posts' own hashtags whenever COMMON_TAGS wasn't
configured (it always wasn't) - posts are now forwarded unchanged.
This commit is contained in:
exostring
2026-08-18 14:48:22 +05:00
parent ec65aaf57d
commit 20135263c4
11 changed files with 596 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/database.py
+103 -21
View File
@@ -17,6 +17,12 @@ except (ImportError, ValueError):
# failing with "database is locked".
_BUSY_TIMEOUT_MS = 5000
# route_id used to backfill rows written before multi-route support existed,
# and the id load_routes() assigns to the auto-generated legacy single route -
# keeping these in sync means an upgrade from the old single-group deployment
# doesn't lose "already published" history for that group.
_LEGACY_ROUTE_ID = "default"
class Database:
def __init__(self, db_path: Optional[str] = None) -> None:
@@ -28,6 +34,73 @@ class Database:
await db.execute(f"PRAGMA busy_timeout = {_BUSY_TIMEOUT_MS};")
yield db
async def _table_columns(self, db: aiosqlite.Connection, table: str) -> set[str]:
cursor = await db.execute(f"PRAGMA table_info({table})")
rows = await cursor.fetchall()
return {row[1] for row in rows}
async def _ensure_column(self, db: aiosqlite.Connection, table: str, column: str, ddl: str) -> None:
cols = await self._table_columns(db, table)
if column not in cols:
await db.execute(f"ALTER TABLE {table} ADD COLUMN {ddl}")
async def _migrate_posts_table(self, db: aiosqlite.Connection) -> None:
"""Adds route_id to a pre-multi-route posts table and rebuilds the UNIQUE
constraint to (route_id, vk_owner_id, vk_post_id). A plain ALTER TABLE ADD
COLUMN can't change a table-level UNIQUE constraint in SQLite, so this does
the standard rename/recreate/copy/drop dance. Existing rows are backfilled
with _LEGACY_ROUTE_ID, matching the route id routes.load_routes() assigns
to the auto-generated single route - so "already published" history for
the pre-existing group survives the upgrade.
"""
cols = await self._table_columns(db, "posts")
if "route_id" in cols:
return
logger.info("Migrating 'posts' table: adding route_id (backfilled as '{}')", _LEGACY_ROUTE_ID)
await db.execute("ALTER TABLE posts RENAME TO posts_old;")
await db.execute(
"""
CREATE TABLE posts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
route_id TEXT NOT NULL DEFAULT 'default',
vk_post_id INTEGER NOT NULL,
vk_owner_id INTEGER NOT NULL,
posted_at INTEGER,
text TEXT,
raw_json TEXT,
tg_status TEXT DEFAULT 'pending',
tg_message_ids TEXT,
tg_url TEXT,
tg_error TEXT,
max_status TEXT DEFAULT 'pending',
max_message_ids TEXT,
max_url TEXT,
max_error TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
published_at TIMESTAMP,
UNIQUE(route_id, vk_owner_id, vk_post_id)
);
"""
)
await db.execute(
f"""
INSERT INTO posts (
id, route_id, vk_post_id, vk_owner_id, posted_at, text, raw_json,
tg_status, tg_message_ids, tg_url, tg_error,
max_status, max_message_ids, max_url, max_error,
created_at, published_at
)
SELECT id, '{_LEGACY_ROUTE_ID}', vk_post_id, vk_owner_id, posted_at, text, raw_json,
tg_status, tg_message_ids, tg_url, tg_error,
max_status, max_message_ids, max_url, max_error,
created_at, published_at
FROM posts_old;
"""
)
await db.execute("DROP TABLE posts_old;")
logger.info("Migration of 'posts' table complete.")
async def init(self) -> None:
logger.info("Initializing database at {}", self.db_path)
async with self._connect() as db:
@@ -36,6 +109,7 @@ class Database:
"""
CREATE TABLE IF NOT EXISTS posts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
route_id TEXT NOT NULL DEFAULT 'default',
vk_post_id INTEGER NOT NULL,
vk_owner_id INTEGER NOT NULL,
posted_at INTEGER,
@@ -51,10 +125,12 @@ class Database:
max_error TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
published_at TIMESTAMP,
UNIQUE(vk_owner_id, vk_post_id)
UNIQUE(route_id, vk_owner_id, vk_post_id)
);
"""
)
await self._migrate_posts_table(db)
await db.execute(
"""
CREATE TABLE IF NOT EXISTS publication_runs (
@@ -68,19 +144,22 @@ class Database:
);
"""
)
await self._ensure_column(db, "publication_runs", "route_id", "route_id TEXT")
await self._ensure_column(db, "publication_runs", "route_name", "route_name TEXT")
await db.commit()
async def has_any_posts(self, owner_id: int) -> bool:
async def has_any_posts(self, route_id: str, owner_id: int) -> bool:
async with self._connect() as db:
cursor = await db.execute(
"SELECT 1 FROM posts WHERE vk_owner_id = ? LIMIT 1",
(owner_id,),
"SELECT 1 FROM posts WHERE route_id = ? AND vk_owner_id = ? LIMIT 1",
(route_id, owner_id),
)
row = await cursor.fetchone()
return bool(row)
async def mark_post_skipped(
self,
route_id: str,
owner_id: int,
post_id: int,
posted_at: int,
@@ -92,22 +171,22 @@ class Database:
async with self._connect() as db:
await db.execute(
"""
INSERT INTO posts (vk_owner_id, vk_post_id, posted_at, text, raw_json, tg_status, max_status, tg_error, max_error)
VALUES (?, ?, ?, ?, ?, 'skipped', 'skipped', ?, ?)
ON CONFLICT(vk_owner_id, vk_post_id) DO UPDATE SET
INSERT INTO posts (route_id, vk_owner_id, vk_post_id, posted_at, text, raw_json, tg_status, max_status, tg_error, max_error)
VALUES (?, ?, ?, ?, ?, ?, 'skipped', 'skipped', ?, ?)
ON CONFLICT(route_id, vk_owner_id, vk_post_id) DO UPDATE SET
tg_status = 'skipped',
max_status = 'skipped';
""",
(owner_id, post_id, posted_at, text, raw_json, reason, reason),
(route_id, owner_id, post_id, posted_at, text, raw_json, reason, reason),
)
await db.commit()
async def is_post_processed(self, owner_id: int, post_id: int) -> bool:
async def is_post_processed(self, route_id: str, owner_id: int, post_id: int) -> bool:
async with self._connect() as db:
db.row_factory = aiosqlite.Row
cursor = await db.execute(
"SELECT tg_status, max_status FROM posts WHERE vk_owner_id = ? AND vk_post_id = ?",
(owner_id, post_id),
"SELECT tg_status, max_status FROM posts WHERE route_id = ? AND vk_owner_id = ? AND vk_post_id = ?",
(route_id, owner_id, post_id),
)
row = await cursor.fetchone()
if not row:
@@ -117,18 +196,19 @@ class Database:
and row["max_status"] in ("published", "skipped")
)
async def get_post(self, owner_id: int, post_id: int) -> Optional[dict[str, Any]]:
async def get_post(self, route_id: str, owner_id: int, post_id: int) -> Optional[dict[str, Any]]:
async with self._connect() as db:
db.row_factory = aiosqlite.Row
cursor = await db.execute(
"SELECT * FROM posts WHERE vk_owner_id = ? AND vk_post_id = ?",
(owner_id, post_id),
"SELECT * FROM posts WHERE route_id = ? AND vk_owner_id = ? AND vk_post_id = ?",
(route_id, owner_id, post_id),
)
row = await cursor.fetchone()
return dict(row) if row else None
async def save_or_update_post(
self,
route_id: str,
owner_id: int,
post_id: int,
posted_at: int,
@@ -139,14 +219,14 @@ class Database:
async with self._connect() as db:
cursor = await db.execute(
"""
INSERT INTO posts (vk_owner_id, vk_post_id, posted_at, text, raw_json)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(vk_owner_id, vk_post_id) DO UPDATE SET
INSERT INTO posts (route_id, vk_owner_id, vk_post_id, posted_at, text, raw_json)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(route_id, vk_owner_id, vk_post_id) DO UPDATE SET
text = excluded.text,
raw_json = excluded.raw_json
RETURNING id;
""",
(owner_id, post_id, posted_at, text, raw_json),
(route_id, owner_id, post_id, posted_at, text, raw_json),
)
row = await cursor.fetchone()
await db.commit()
@@ -202,6 +282,8 @@ class Database:
async def record_run(
self,
route_id: str,
route_name: str,
found_count: int,
tg_count: int,
max_count: int,
@@ -211,9 +293,9 @@ class Database:
async with self._connect() as db:
await db.execute(
"""
INSERT INTO publication_runs (found_count, published_tg_count, published_max_count, status, error)
VALUES (?, ?, ?, ?, ?);
INSERT INTO publication_runs (route_id, route_name, found_count, published_tg_count, published_max_count, status, error)
VALUES (?, ?, ?, ?, ?, ?, ?);
""",
(found_count, tg_count, max_count, status, error),
(route_id, route_name, found_count, tg_count, max_count, status, error),
)
await db.commit()