Initial RAA parser poster copy
This commit is contained in:
Your Name
@@ -0,0 +1,3 @@
|
||||
__all__ = ["__version__"]
|
||||
|
||||
__version__ = "0.1.0"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,55 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
app_env: str = "production"
|
||||
app_secret_key: str = "change-me"
|
||||
admin_site_title: str = ""
|
||||
admin_app_title: str = "Редакторская"
|
||||
admin_bootstrap_login: str = "admin"
|
||||
admin_bootstrap_password: str = ""
|
||||
|
||||
db_host: str = "localhost"
|
||||
db_port: int = 5432
|
||||
db_name: str = "vk_parser"
|
||||
db_user: str = "vk_parser_user"
|
||||
db_password: str = ""
|
||||
|
||||
vk_access_token: str = ""
|
||||
vk_group_access_token: str = ""
|
||||
vk_api_version: str = "5.199"
|
||||
vk_storage_group_id: int = 0
|
||||
|
||||
tg_bot_token: str = ""
|
||||
tg_media_channel_id: str = ""
|
||||
local_bot_api_url: str = ""
|
||||
|
||||
admin_host: str = "0.0.0.0"
|
||||
admin_port: int = 8080
|
||||
log_level: str = "INFO"
|
||||
|
||||
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
|
||||
|
||||
@property
|
||||
def db_dsn(self) -> str:
|
||||
return (
|
||||
f"postgresql://{self.db_user}:{self.db_password}"
|
||||
f"@{self.db_host}:{self.db_port}/{self.db_name}"
|
||||
)
|
||||
|
||||
@property
|
||||
def display_admin_title(self) -> str:
|
||||
return self.admin_app_title.strip() or "Редакторская"
|
||||
|
||||
@property
|
||||
def display_site_title(self) -> str:
|
||||
return self.admin_site_title.strip() or "Редакторская"
|
||||
|
||||
@property
|
||||
def vk_storage_owner_id(self) -> int:
|
||||
return -abs(int(self.vk_storage_group_id))
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -0,0 +1,40 @@
|
||||
PLATFORM_VK = "vk"
|
||||
|
||||
SOURCE_STATUS_NEW = "new"
|
||||
SOURCE_STATUS_OK = "ok"
|
||||
SOURCE_STATUS_ERROR = "error"
|
||||
SOURCE_STATUS_PAUSED = "paused"
|
||||
|
||||
POST_STATUS_RAW_SAVED = "raw_saved"
|
||||
POST_STATUS_STORAGE_PENDING = "storage_pending"
|
||||
POST_STATUS_STORAGE_READY = "storage_ready"
|
||||
POST_STATUS_SKIPPED = "skipped"
|
||||
POST_STATUS_FAILED = "failed"
|
||||
|
||||
PUBLICATION_STATUS_PENDING = "pending"
|
||||
PUBLICATION_STATUS_PUBLISHED = "published"
|
||||
PUBLICATION_STATUS_FAILED = "publish_failed"
|
||||
|
||||
MEDIA_STATUS_PENDING = "pending"
|
||||
MEDIA_STATUS_UPLOADED = "uploaded"
|
||||
MEDIA_STATUS_LINK_ONLY = "link_only"
|
||||
MEDIA_STATUS_FAILED = "failed"
|
||||
|
||||
JOB_STATUS_PENDING = "pending"
|
||||
JOB_STATUS_IN_PROGRESS = "in_progress"
|
||||
JOB_STATUS_RETRY = "retry"
|
||||
JOB_STATUS_DONE = "done"
|
||||
JOB_STATUS_DEAD = "dead"
|
||||
|
||||
JOB_TYPE_VK_STORAGE_COPY = "vk.storage.copy"
|
||||
|
||||
WORKER_PARSER = "vk-parser"
|
||||
WORKER_STORAGE_UPLOADER = "vk-storage-uploader"
|
||||
WORKER_AI_QUALIFIER = "ai-qualifier"
|
||||
WORKER_AI_WRITER = "ai-writer"
|
||||
WORKER_TG_POSTER = "tg-poster"
|
||||
WORKER_TG_REACTOR = "tg-reactor"
|
||||
WORKER_VK_POSTER = "vk-poster"
|
||||
WORKER_MAX_POSTER = "max-poster"
|
||||
WORKER_SITE_POSTER = "site-poster"
|
||||
WORKER_DAILY_REPORT = "daily-report"
|
||||
@@ -0,0 +1,99 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import asyncpg
|
||||
from loguru import logger
|
||||
|
||||
from .config import settings
|
||||
|
||||
_pool: asyncpg.Pool | None = None
|
||||
|
||||
|
||||
async def get_pool() -> asyncpg.Pool:
|
||||
global _pool
|
||||
if _pool is None:
|
||||
_pool = await asyncpg.create_pool(
|
||||
host=settings.db_host,
|
||||
port=settings.db_port,
|
||||
database=settings.db_name,
|
||||
user=settings.db_user,
|
||||
password=settings.db_password,
|
||||
min_size=1,
|
||||
max_size=10,
|
||||
command_timeout=60,
|
||||
)
|
||||
logger.info("Database pool created: {}:{}/{}", settings.db_host, settings.db_port, settings.db_name)
|
||||
return _pool
|
||||
|
||||
|
||||
async def close_pool() -> None:
|
||||
global _pool
|
||||
if _pool is not None:
|
||||
await _pool.close()
|
||||
_pool = None
|
||||
|
||||
|
||||
async def fetch_setting(key: str, default: Any = None) -> Any:
|
||||
pool = await get_pool()
|
||||
row = await pool.fetchrow("SELECT value_json FROM app_settings WHERE key=$1", key)
|
||||
if not row:
|
||||
return default
|
||||
value = row["value_json"]
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
return value
|
||||
return value
|
||||
|
||||
|
||||
async def fetch_int_setting(key: str, default: int) -> int:
|
||||
try:
|
||||
return int(await fetch_setting(key, default))
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
async def fetch_float_setting(key: str, default: float) -> float:
|
||||
try:
|
||||
return float(await fetch_setting(key, default))
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
async def fetch_bool_setting(key: str, default: bool) -> bool:
|
||||
try:
|
||||
value = await fetch_setting(key, default)
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
return value.strip().lower() in {"1", "true", "yes", "on"}
|
||||
return bool(value)
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
async def apply_migrations(migrations_dir: Path) -> None:
|
||||
pool = await get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version TEXT PRIMARY KEY,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
"""
|
||||
)
|
||||
for path in sorted(migrations_dir.glob("*.sql")):
|
||||
version = path.name
|
||||
exists = await conn.fetchval("SELECT 1 FROM schema_migrations WHERE version=$1", version)
|
||||
if exists:
|
||||
continue
|
||||
sql = path.read_text(encoding="utf-8")
|
||||
async with conn.transaction():
|
||||
await conn.execute(sql)
|
||||
await conn.execute("INSERT INTO schema_migrations(version) VALUES($1)", version)
|
||||
logger.info("Applied migration {}", version)
|
||||
@@ -0,0 +1,45 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
|
||||
class HeartbeatReporter:
|
||||
def __init__(self, name: str, interval_sec: int = 30) -> None:
|
||||
self.name = name
|
||||
self.interval_sec = max(5, int(interval_sec))
|
||||
self._last = 0.0
|
||||
|
||||
async def beat(
|
||||
self,
|
||||
pool,
|
||||
status: str = "running",
|
||||
current_job_id: int | None = None,
|
||||
meta: dict[str, Any] | None = None,
|
||||
force: bool = False,
|
||||
) -> None:
|
||||
now = time.monotonic()
|
||||
if not force and now - self._last < self.interval_sec:
|
||||
return
|
||||
self._last = now
|
||||
try:
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO worker_heartbeats(name, heartbeat_at, status, current_job_id, meta_json, updated_at)
|
||||
VALUES ($1, NOW(), $2, $3, $4::jsonb, NOW())
|
||||
ON CONFLICT (name) DO UPDATE
|
||||
SET heartbeat_at=NOW(),
|
||||
status=EXCLUDED.status,
|
||||
current_job_id=EXCLUDED.current_job_id,
|
||||
meta_json=EXCLUDED.meta_json,
|
||||
updated_at=NOW()
|
||||
""",
|
||||
self.name,
|
||||
status,
|
||||
current_job_id,
|
||||
json.dumps(meta or {}, ensure_ascii=False),
|
||||
)
|
||||
except Exception:
|
||||
return
|
||||
@@ -0,0 +1,98 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from .constants import JOB_STATUS_IN_PROGRESS, JOB_STATUS_PENDING, JOB_STATUS_RETRY
|
||||
|
||||
|
||||
async def is_worker_enabled(pool, name: str) -> bool:
|
||||
value = await pool.fetchval("SELECT enabled FROM worker_controls WHERE name=$1", name)
|
||||
return bool(value)
|
||||
|
||||
|
||||
async def claim_job(pool, job_type: str, worker_id: str) -> dict | None:
|
||||
row = await pool.fetchrow(
|
||||
"""
|
||||
WITH cte AS (
|
||||
SELECT id
|
||||
FROM jobs
|
||||
WHERE type=$1
|
||||
AND status IN ($2, $3)
|
||||
AND next_run_at <= NOW()
|
||||
ORDER BY next_run_at ASC, id ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
)
|
||||
UPDATE jobs j
|
||||
SET status=$4,
|
||||
locked_by=$5,
|
||||
locked_at=NOW(),
|
||||
updated_at=NOW()
|
||||
FROM cte
|
||||
WHERE j.id=cte.id
|
||||
RETURNING j.*
|
||||
""",
|
||||
job_type,
|
||||
JOB_STATUS_PENDING,
|
||||
JOB_STATUS_RETRY,
|
||||
JOB_STATUS_IN_PROGRESS,
|
||||
worker_id,
|
||||
)
|
||||
return dict(row) if row else None
|
||||
|
||||
|
||||
async def ack_done(pool, job_id: int) -> None:
|
||||
await pool.execute(
|
||||
"""
|
||||
UPDATE jobs
|
||||
SET status='done',
|
||||
locked_by=NULL,
|
||||
locked_at=NULL,
|
||||
last_error=NULL,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
job_id,
|
||||
)
|
||||
|
||||
|
||||
async def ack_retry(pool, job: dict, error: str, delay_sec: int = 60) -> None:
|
||||
attempt = int(job.get("attempts") or 0) + 1
|
||||
max_attempts = int(job.get("max_attempts") or 5)
|
||||
status = "dead" if attempt >= max_attempts else "retry"
|
||||
await pool.execute(
|
||||
"""
|
||||
UPDATE jobs
|
||||
SET status=$2,
|
||||
attempts=$3,
|
||||
next_run_at=CASE WHEN $2='retry' THEN NOW() + ($4 * INTERVAL '1 second') ELSE next_run_at END,
|
||||
locked_by=NULL,
|
||||
locked_at=NULL,
|
||||
last_error=$5,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
int(job["id"]),
|
||||
status,
|
||||
attempt,
|
||||
int(delay_sec),
|
||||
error[:1000],
|
||||
)
|
||||
|
||||
|
||||
async def recover_stale_jobs(pool, job_type: str, stale_minutes: int = 20) -> int:
|
||||
result = await pool.execute(
|
||||
"""
|
||||
UPDATE jobs
|
||||
SET status='retry',
|
||||
locked_by=NULL,
|
||||
locked_at=NULL,
|
||||
next_run_at=NOW(),
|
||||
last_error=COALESCE(last_error, 'recovered stale lock'),
|
||||
updated_at=NOW()
|
||||
WHERE type=$1
|
||||
AND status='in_progress'
|
||||
AND locked_at < NOW() - ($2 * INTERVAL '1 minute')
|
||||
""",
|
||||
job_type,
|
||||
stale_minutes,
|
||||
)
|
||||
return int(str(result).split()[-1])
|
||||
@@ -0,0 +1,3 @@
|
||||
from .admin import app
|
||||
|
||||
__all__ = ["app"]
|
||||
@@ -0,0 +1,37 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import secrets
|
||||
|
||||
|
||||
def hash_password(password: str, iterations: int = 390_000) -> str:
|
||||
salt = secrets.token_bytes(16)
|
||||
digest = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, iterations)
|
||||
return "pbkdf2_sha256${}${}${}".format(
|
||||
iterations,
|
||||
base64.urlsafe_b64encode(salt).decode("ascii"),
|
||||
base64.urlsafe_b64encode(digest).decode("ascii"),
|
||||
)
|
||||
|
||||
|
||||
def verify_password(password: str, encoded: str) -> bool:
|
||||
try:
|
||||
algo, iterations_raw, salt_raw, digest_raw = encoded.split("$", 3)
|
||||
if algo != "pbkdf2_sha256":
|
||||
return False
|
||||
iterations = int(iterations_raw)
|
||||
salt = base64.urlsafe_b64decode(salt_raw.encode("ascii"))
|
||||
expected = base64.urlsafe_b64decode(digest_raw.encode("ascii"))
|
||||
except Exception:
|
||||
return False
|
||||
actual = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, iterations)
|
||||
return secrets.compare_digest(actual, expected)
|
||||
|
||||
|
||||
def token_hash(token: str, secret: str) -> str:
|
||||
return hashlib.sha256((secret + ":" + token).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def new_token() -> str:
|
||||
return secrets.token_urlsafe(32)
|
||||
@@ -0,0 +1,340 @@
|
||||
<!doctype html>
|
||||
<html lang="ru" class="dark">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<link rel="icon" type="image/png" href="/favi.png">
|
||||
<title>{{ title or site_title or "Редакторская" }}</title>
|
||||
|
||||
<!-- Fonts: Fira Sans (UI) & Fira Code (Data) -->
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Fira+Code:wght@400;500;600;700&family=Fira+Sans:wght@300;400;500;600;700&display=swap" rel="stylesheet">
|
||||
|
||||
<script>
|
||||
(function() {
|
||||
const theme = localStorage.getItem('admin-theme') || 'dark';
|
||||
document.documentElement.classList.toggle('dark', theme !== 'light');
|
||||
})();
|
||||
</script>
|
||||
|
||||
<!-- Tailwind CSS via CDN -->
|
||||
<script src="https://cdn.tailwindcss.com"></script>
|
||||
<script>
|
||||
tailwind.config = {
|
||||
darkMode: 'class',
|
||||
theme: {
|
||||
extend: {
|
||||
fontFamily: {
|
||||
sans: ['"Fira Sans"', 'sans-serif'],
|
||||
mono: ['"Fira Code"', 'monospace'],
|
||||
},
|
||||
colors: {
|
||||
app: {
|
||||
bg: 'rgb(var(--app-bg) / <alpha-value>)',
|
||||
surface: 'rgb(var(--app-surface) / <alpha-value>)',
|
||||
surfaceHover: 'rgb(var(--app-surface-hover) / <alpha-value>)',
|
||||
border: 'rgb(var(--app-border) / <alpha-value>)',
|
||||
borderFocus: 'rgb(var(--app-border-focus) / <alpha-value>)',
|
||||
primary: 'rgb(var(--app-primary) / <alpha-value>)',
|
||||
primaryHover: 'rgb(var(--app-primary-hover) / <alpha-value>)',
|
||||
textMain: 'rgb(var(--app-text-main) / <alpha-value>)',
|
||||
textMuted: 'rgb(var(--app-text-muted) / <alpha-value>)',
|
||||
success: 'rgb(var(--app-success) / <alpha-value>)',
|
||||
successBg: 'var(--app-success-bg)',
|
||||
warning: 'rgb(var(--app-warning) / <alpha-value>)',
|
||||
warningBg: 'var(--app-warning-bg)',
|
||||
error: 'rgb(var(--app-error) / <alpha-value>)',
|
||||
errorBg: 'var(--app-error-bg)',
|
||||
}
|
||||
},
|
||||
boxShadow: {
|
||||
'glow': '0 0 15px rgba(59, 130, 246, 0.3)',
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<!-- HTMX & AlpineJS -->
|
||||
<script src="https://unpkg.com/htmx.org@1.9.11"></script>
|
||||
<script defer src="https://cdn.jsdelivr.net/npm/alpinejs@3.13.8/dist/cdn.min.js"></script>
|
||||
|
||||
<!-- Lucide Icons -->
|
||||
<script src="https://unpkg.com/lucide@latest"></script>
|
||||
|
||||
<style>
|
||||
:root {
|
||||
--app-bg: 248 250 252;
|
||||
--app-surface: 255 255 255;
|
||||
--app-surface-hover: 226 232 240;
|
||||
--app-border: 203 213 225;
|
||||
--app-border-focus: 148 163 184;
|
||||
--app-primary: 37 99 235;
|
||||
--app-primary-hover: 29 78 216;
|
||||
--app-text-main: 15 23 42;
|
||||
--app-text-muted: 100 116 139;
|
||||
--app-success: 5 150 105;
|
||||
--app-success-bg: rgba(5, 150, 105, 0.1);
|
||||
--app-warning: 217 119 6;
|
||||
--app-warning-bg: rgba(217, 119, 6, 0.1);
|
||||
--app-error: 220 38 38;
|
||||
--app-error-bg: rgba(220, 38, 38, 0.1);
|
||||
}
|
||||
.dark {
|
||||
--app-bg: 15 23 42;
|
||||
--app-surface: 30 41 59;
|
||||
--app-surface-hover: 51 65 85;
|
||||
--app-border: 51 65 85;
|
||||
--app-border-focus: 71 85 105;
|
||||
--app-primary: 59 130 246;
|
||||
--app-primary-hover: 37 99 235;
|
||||
--app-text-main: 248 250 252;
|
||||
--app-text-muted: 148 163 184;
|
||||
--app-success: 16 185 129;
|
||||
--app-success-bg: rgba(16, 185, 129, 0.1);
|
||||
--app-warning: 245 158 11;
|
||||
--app-warning-bg: rgba(245, 158, 11, 0.1);
|
||||
--app-error: 239 68 68;
|
||||
--app-error-bg: rgba(239, 68, 68, 0.1);
|
||||
}
|
||||
:root:not(.dark) .text-white,
|
||||
:root:not(.dark) .hover\:text-white:hover {
|
||||
color: rgb(var(--app-text-main)) !important;
|
||||
}
|
||||
:root:not(.dark) .hover\:bg-app-primary:hover.hover\:text-white,
|
||||
:root:not(.dark) .hover\:bg-app-error:hover.hover\:text-white {
|
||||
color: #fff !important;
|
||||
}
|
||||
:root:not(.dark) .media-gallery-layer .text-white,
|
||||
:root:not(.dark) .media-gallery-layer .hover\:text-app-primary:hover {
|
||||
color: #fff !important;
|
||||
}
|
||||
</style>
|
||||
|
||||
<style type="text/tailwindcss">
|
||||
@layer components {
|
||||
/* Form inputs */
|
||||
.input {
|
||||
@apply bg-app-bg border border-app-border rounded-lg px-3 py-2 text-sm text-app-textMain focus:outline-none focus:border-app-primary focus:ring-1 focus:ring-app-primary transition-colors w-full placeholder:text-app-textMuted;
|
||||
}
|
||||
.select {
|
||||
@apply input appearance-none bg-no-repeat bg-[right_0.5rem_center] bg-[length:1.5em_1.5em];
|
||||
background-image: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='none' viewBox='0 0 20 20'%3e%3cpath stroke='%2394A3B8' stroke-linecap='round' stroke-linejoin='round' stroke-width='1.5' d='M6 8l4 4 4-4'/%3e%3c/svg%3e");
|
||||
}
|
||||
.textarea {
|
||||
@apply bg-app-bg border border-app-border rounded-lg px-3 py-2 text-sm text-app-textMain focus:outline-none focus:border-app-primary focus:ring-1 focus:ring-app-primary transition-colors w-full placeholder:text-app-textMuted;
|
||||
}
|
||||
.checkbox {
|
||||
@apply w-4 h-4 rounded border-app-border bg-app-bg text-app-primary focus:ring-app-primary focus:ring-offset-app-surface;
|
||||
}
|
||||
/* Buttons */
|
||||
.btn {
|
||||
@apply inline-flex items-center justify-center gap-2 rounded-lg font-medium text-sm px-4 py-2 transition-all disabled:opacity-50 disabled:cursor-not-allowed cursor-pointer focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-offset-app-bg;
|
||||
}
|
||||
.btn-primary {
|
||||
@apply bg-app-primary text-white hover:bg-app-primaryHover focus:ring-app-primary border border-transparent;
|
||||
}
|
||||
.btn-surface {
|
||||
@apply bg-app-surface border border-app-border text-app-textMain hover:bg-app-surfaceHover focus:ring-app-border;
|
||||
}
|
||||
.btn-ghost {
|
||||
@apply text-app-textMuted hover:text-app-textMain hover:bg-app-surfaceHover border border-transparent;
|
||||
}
|
||||
.btn-danger {
|
||||
@apply bg-app-error text-white hover:bg-red-600 focus:ring-app-error border border-transparent;
|
||||
}
|
||||
.btn-danger-outline {
|
||||
@apply bg-transparent border border-app-error text-app-error hover:bg-app-error hover:text-white focus:ring-app-error;
|
||||
}
|
||||
.btn-primary-outline {
|
||||
@apply bg-transparent border border-app-primary text-app-primary hover:bg-app-primary hover:text-white focus:ring-app-primary;
|
||||
}
|
||||
.btn-sm {
|
||||
@apply px-3 py-1.5 text-xs;
|
||||
}
|
||||
.btn-xs {
|
||||
@apply px-2 py-1 text-xs;
|
||||
}
|
||||
.btn-icon {
|
||||
@apply p-2;
|
||||
}
|
||||
/* Cards */
|
||||
.card {
|
||||
@apply bg-app-surface border border-app-border rounded-xl shadow-sm overflow-hidden;
|
||||
}
|
||||
/* Badges */
|
||||
.badge {
|
||||
@apply inline-flex items-center px-2 py-0.5 rounded text-xs font-medium border;
|
||||
}
|
||||
.badge-success {
|
||||
@apply bg-app-successBg text-app-success border-app-success/20;
|
||||
}
|
||||
.badge-error {
|
||||
@apply bg-app-errorBg text-app-error border-app-error/20;
|
||||
}
|
||||
.badge-warning {
|
||||
@apply bg-app-warningBg text-app-warning border-app-warning/20;
|
||||
}
|
||||
.badge-neutral {
|
||||
@apply bg-app-bg text-app-textMuted border-app-border;
|
||||
}
|
||||
.badge-primary {
|
||||
@apply bg-app-primary/10 text-app-primary border-app-primary/20;
|
||||
}
|
||||
.badge-info {
|
||||
@apply bg-blue-500/10 text-blue-400 border-blue-500/20;
|
||||
}
|
||||
|
||||
/* Tabs */
|
||||
.tabs {
|
||||
@apply flex space-x-1 bg-app-bg p-1 rounded-lg border border-app-border overflow-x-auto;
|
||||
}
|
||||
.tab {
|
||||
@apply flex items-center px-3 py-1.5 text-sm font-medium rounded-md text-app-textMuted hover:text-app-textMain transition-colors whitespace-nowrap;
|
||||
}
|
||||
.tab-active {
|
||||
@apply bg-app-surface text-app-textMain shadow-sm border border-app-border;
|
||||
}
|
||||
|
||||
/* HTMX Indicators */
|
||||
.htmx-indicator { display:none; }
|
||||
.htmx-request .htmx-indicator { display:inline-block; }
|
||||
.htmx-request.htmx-indicator { display:inline-block; }
|
||||
[x-cloak] { display: none !important; }
|
||||
|
||||
/* Scrollbar */
|
||||
::-webkit-scrollbar { width: 8px; height: 8px; }
|
||||
::-webkit-scrollbar-track { @apply bg-transparent; }
|
||||
::-webkit-scrollbar-thumb { @apply bg-app-border rounded-full hover:bg-app-borderFocus; }
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body class="bg-app-bg text-app-textMain min-h-screen font-sans flex antialiased">
|
||||
|
||||
{% if user %}
|
||||
<!-- Sidebar -->
|
||||
<aside x-data="{ expanded: false }" :class="expanded ? 'w-64' : 'w-20'" class="flex-shrink-0 bg-app-surface border-r border-app-border flex flex-col hidden md:flex sticky top-0 h-screen transition-all duration-200">
|
||||
<div class="p-4">
|
||||
<div class="flex items-center gap-3 text-app-textMain font-bold text-lg tracking-wide" :class="expanded ? 'justify-between' : 'justify-center'">
|
||||
<div class="flex items-center gap-3 min-w-0">
|
||||
<span x-show="expanded" x-cloak class="truncate">{{ app_title or "Редакторская" }}</span>
|
||||
</div>
|
||||
<button type="button" class="btn btn-ghost btn-icon flex-shrink-0" @click="expanded = !expanded" :aria-label="expanded ? 'Свернуть меню' : 'Развернуть меню'" :title="expanded ? 'Свернуть меню' : 'Развернуть меню'">
|
||||
<i data-lucide="panel-left" class="w-5 h-5"></i>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<nav class="flex-1 px-4 space-y-1 overflow-y-auto">
|
||||
<a href="/raw" title="Сырые посты" class="flex items-center gap-3 px-3 py-2 rounded-lg text-sm font-medium transition-colors {{ 'bg-app-primary/10 text-app-primary' if 'raw' in request.url.path else 'text-app-textMuted hover:bg-app-surfaceHover hover:text-app-textMain' }}" :class="expanded ? '' : 'justify-center'">
|
||||
<i data-lucide="inbox" class="w-5 h-5 flex-shrink-0"></i>
|
||||
<span x-show="expanded" x-cloak>Сырые посты</span>
|
||||
</a>
|
||||
<a href="/editor" title="Редактор" class="flex items-center gap-3 px-3 py-2 rounded-lg text-sm font-medium transition-colors {{ 'bg-app-primary/10 text-app-primary' if 'editor' in request.url.path else 'text-app-textMuted hover:bg-app-surfaceHover hover:text-app-textMain' }}" :class="expanded ? '' : 'justify-center'">
|
||||
<i data-lucide="edit-3" class="w-5 h-5 flex-shrink-0"></i>
|
||||
<span x-show="expanded" x-cloak>Редактор</span>
|
||||
</a>
|
||||
<a href="/sources" title="Источники" class="flex items-center gap-3 px-3 py-2 rounded-lg text-sm font-medium transition-colors {{ 'bg-app-primary/10 text-app-primary' if 'sources' in request.url.path else 'text-app-textMuted hover:bg-app-surfaceHover hover:text-app-textMain' }}" :class="expanded ? '' : 'justify-center'">
|
||||
<i data-lucide="database" class="w-5 h-5 flex-shrink-0"></i>
|
||||
<span x-show="expanded" x-cloak>Источники</span>
|
||||
</a>
|
||||
{% if user.role != "editor" %}
|
||||
<a href="/workers" title="Воркеры" class="flex items-center gap-3 px-3 py-2 rounded-lg text-sm font-medium transition-colors {{ 'bg-app-primary/10 text-app-primary' if 'workers' in request.url.path else 'text-app-textMuted hover:bg-app-surfaceHover hover:text-app-textMain' }}" :class="expanded ? '' : 'justify-center'">
|
||||
<i data-lucide="cpu" class="w-5 h-5 flex-shrink-0"></i>
|
||||
<span x-show="expanded" x-cloak>Воркеры</span>
|
||||
</a>
|
||||
<a href="/users" title="Пользователи" class="flex items-center gap-3 px-3 py-2 rounded-lg text-sm font-medium transition-colors {{ 'bg-app-primary/10 text-app-primary' if 'users' in request.url.path else 'text-app-textMuted hover:bg-app-surfaceHover hover:text-app-textMain' }}" :class="expanded ? '' : 'justify-center'">
|
||||
<i data-lucide="users" class="w-5 h-5 flex-shrink-0"></i>
|
||||
<span x-show="expanded" x-cloak>Пользователи</span>
|
||||
</a>
|
||||
<a href="/logs" title="Логи" class="flex items-center gap-3 px-3 py-2 rounded-lg text-sm font-medium transition-colors {{ 'bg-app-primary/10 text-app-primary' if 'logs' in request.url.path else 'text-app-textMuted hover:bg-app-surfaceHover hover:text-app-textMain' }}" :class="expanded ? '' : 'justify-center'">
|
||||
<i data-lucide="file-text" class="w-5 h-5 flex-shrink-0"></i>
|
||||
<span x-show="expanded" x-cloak>Логи</span>
|
||||
</a>
|
||||
{% endif %}
|
||||
</nav>
|
||||
|
||||
<div class="p-4 border-t border-app-border">
|
||||
<button type="button" title="Переключить тему" class="w-full flex items-center justify-center gap-2 px-3 py-2 mb-2 rounded-lg text-sm font-medium text-app-textMuted hover:text-app-textMain hover:bg-app-surfaceHover transition-colors" onclick="toggleTheme()">
|
||||
<i data-lucide="sun-moon" class="w-4 h-4"></i>
|
||||
<span x-show="expanded" x-cloak>Тема</span>
|
||||
</button>
|
||||
<div class="flex items-center gap-3 px-3 py-2 mb-2" :class="expanded ? '' : 'justify-center'">
|
||||
<div class="w-8 h-8 rounded-full bg-app-primary/20 flex items-center justify-center text-app-primary font-bold text-sm flex-shrink-0">
|
||||
{{ user.login[0]|upper }}
|
||||
</div>
|
||||
<div x-show="expanded" x-cloak class="flex-1 min-w-0">
|
||||
<div class="text-sm font-medium text-app-textMain truncate">{{ user.login }}</div>
|
||||
<div class="text-xs text-app-textMuted truncate">{{ user.role }}</div>
|
||||
</div>
|
||||
</div>
|
||||
<form method="post" action="/logout" class="m-0">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<button type="submit" title="Выйти" class="w-full flex items-center justify-center gap-2 px-3 py-2 rounded-lg text-sm font-medium text-app-error hover:bg-app-errorBg transition-colors">
|
||||
<i data-lucide="log-out" class="w-4 h-4"></i>
|
||||
<span x-show="expanded" x-cloak>Выйти</span>
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
<!-- Mobile Topbar (Visible only on small screens) -->
|
||||
<div class="md:hidden fixed top-0 left-0 right-0 h-16 bg-app-surface border-b border-app-border z-50 flex items-center justify-between px-4">
|
||||
<div class="flex items-center gap-2 font-bold text-app-textMain">
|
||||
{{ app_title or "Редакторская" }}
|
||||
</div>
|
||||
<div class="flex items-center gap-1">
|
||||
<button class="btn btn-ghost btn-icon" type="button" onclick="toggleTheme()" title="Тема">
|
||||
<i data-lucide="sun-moon" class="w-5 h-5"></i>
|
||||
</button>
|
||||
<button class="btn btn-ghost btn-icon" onclick="document.getElementById('mobile-menu').classList.toggle('hidden')">
|
||||
<i data-lucide="menu" class="w-6 h-6"></i>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Mobile Menu Dropdown -->
|
||||
<div id="mobile-menu" class="hidden md:hidden fixed top-16 left-0 right-0 bg-app-surface border-b border-app-border z-40 p-4 space-y-2 shadow-lg">
|
||||
<a href="/raw" class="block px-3 py-2 rounded-lg text-sm {{ 'bg-app-bg text-app-textMain' if 'raw' in request.url.path else 'text-app-textMuted' }}">Сырые посты</a>
|
||||
<a href="/editor" class="block px-3 py-2 rounded-lg text-sm {{ 'bg-app-bg text-app-textMain' if 'editor' in request.url.path else 'text-app-textMuted' }}">Редактор</a>
|
||||
<a href="/sources" class="block px-3 py-2 rounded-lg text-sm {{ 'bg-app-bg text-app-textMain' if 'sources' in request.url.path else 'text-app-textMuted' }}">Источники</a>
|
||||
{% if user.role != "editor" %}
|
||||
<a href="/workers" class="block px-3 py-2 rounded-lg text-sm {{ 'bg-app-bg text-app-textMain' if 'workers' in request.url.path else 'text-app-textMuted' }}">Воркеры</a>
|
||||
{% endif %}
|
||||
<div class="pt-2 mt-2 border-t border-app-border">
|
||||
<form method="post" action="/logout" class="m-0">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<button type="submit" class="w-full text-left px-3 py-2 rounded-lg text-sm text-app-error">Выйти</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<!-- Main Content Area -->
|
||||
<main class="flex-1 min-w-0 flex flex-col h-screen overflow-y-auto {% if user %}pt-16 md:pt-0{% endif %} relative">
|
||||
<div class="p-4 md:p-8 max-w-[1600px] w-full mx-auto">
|
||||
{% block body %}{% endblock %}
|
||||
</div>
|
||||
</main>
|
||||
|
||||
<script>
|
||||
function toggleTheme() {
|
||||
const light = document.documentElement.classList.toggle('dark') === false;
|
||||
localStorage.setItem('admin-theme', light ? 'light' : 'dark');
|
||||
}
|
||||
|
||||
// Initialize Lucide icons
|
||||
lucide.createIcons();
|
||||
|
||||
// Re-initialize icons after HTMX swap
|
||||
document.body.addEventListener('htmx:afterSwap', function() {
|
||||
lucide.createIcons();
|
||||
});
|
||||
|
||||
document.body.addEventListener('htmx:responseError', function(evt) {
|
||||
alert("Ошибка при загрузке: " + evt.detail.xhr.status);
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,33 @@
|
||||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
<div class="mb-6 flex flex-col md:flex-row justify-between items-start md:items-center gap-4">
|
||||
<div>
|
||||
<h1 class="text-3xl font-bold tracking-tight text-white flex items-center gap-3">
|
||||
<i data-lucide="edit-3" class="text-app-primary w-8 h-8"></i>
|
||||
Редакторская
|
||||
</h1>
|
||||
<div class="text-app-textMuted mt-1 text-sm">Проверка, правка и публикация сгенерированных постов.</div>
|
||||
</div>
|
||||
|
||||
<!-- Status Tabs -->
|
||||
<div class="w-full md:w-auto min-w-0 overflow-x-auto">
|
||||
<div class="tabs p-1 w-max">
|
||||
{% for st in status_options %}
|
||||
<a class="tab {% if st.value in editorial_statuses %}tab-active{% endif %}" href="/editor?editorial_status={{ st.value }}">
|
||||
{{ st.label }}
|
||||
<span class="ml-2 px-1.5 py-0.5 rounded-full text-xs font-semibold {% if st.value in editorial_statuses %}bg-app-primary/20 text-app-primary{% else %}bg-app-surface text-app-textMuted{% endif %}">
|
||||
{{ counts.get(st.value, 0) }}
|
||||
</span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="content-area">
|
||||
{% include "editor_content.html" %}
|
||||
</div>
|
||||
|
||||
<!-- Modal container for HTMX forms -->
|
||||
<div id="modal-container"></div>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,128 @@
|
||||
<div class="flex flex-col xl:flex-row gap-6" x-data="{ filtersOpen: false }">
|
||||
<!-- Sidebar Filters -->
|
||||
<aside class="w-full flex-shrink-0 transition-all duration-200" :class="filtersOpen ? 'xl:w-72' : 'xl:w-16'">
|
||||
<div class="card sticky top-6" :class="filtersOpen ? 'p-4' : 'p-2'">
|
||||
<button type="button" class="btn btn-ghost btn-icon w-10 h-10 mx-auto mb-0" @click="filtersOpen = !filtersOpen; setTimeout(() => window.dispatchEvent(new Event('resize')), 250)" :aria-expanded="filtersOpen.toString()" :title="filtersOpen ? 'Свернуть фильтры' : 'Развернуть фильтры'">
|
||||
<i data-lucide="sliders-horizontal" class="w-5 h-5"></i>
|
||||
</button>
|
||||
<form id="filter-form" hx-get="/editor" hx-target="#content-area" hx-push-url="true" hx-trigger="submit">
|
||||
<input type="hidden" name="editorial_status" value="{{ status_filter }}">
|
||||
|
||||
<div x-show="filtersOpen" x-cloak class="flex justify-between items-center mt-4 mb-6">
|
||||
<h2 class="text-sm font-bold text-app-textMuted uppercase tracking-wider">Фильтры</h2>
|
||||
<a href="/editor" class="text-xs text-app-primary hover:text-app-primaryHover transition-colors" hx-boost="true" hx-target="#content-area">Сбросить</a>
|
||||
</div>
|
||||
|
||||
<div x-show="filtersOpen" x-cloak class="space-y-6">
|
||||
<!-- Sort -->
|
||||
<div>
|
||||
<label class="block text-xs font-semibold text-app-textMain mb-2">Сортировка</label>
|
||||
<select name="sort" class="select w-full">
|
||||
<option value="rewritten_desc" {% if sort == "rewritten_desc" %}selected{% endif %}>рерайт: новые</option>
|
||||
<option value="rewritten_asc" {% if sort == "rewritten_asc" %}selected{% endif %}>рерайт: старые</option>
|
||||
<option value="posted_desc" {% if sort == "posted_desc" %}selected{% endif %}>VK: новые</option>
|
||||
<option value="posted_asc" {% if sort == "posted_asc" %}selected{% endif %}>VK: старые</option>
|
||||
<option value="score_desc" {% if sort == "score_desc" %}selected{% endif %}>оценка: выше</option>
|
||||
<option value="score_asc" {% if sort == "score_asc" %}selected{% endif %}>оценка: ниже</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<!-- Score -->
|
||||
<div>
|
||||
<label class="block text-xs font-semibold text-app-textMain mb-2">Оценка AI</label>
|
||||
<div class="flex gap-2">
|
||||
<input type="number" name="score_min" placeholder="От" value="{{ score_min }}" class="input w-full" />
|
||||
<input type="number" name="score_max" placeholder="До" value="{{ score_max }}" class="input w-full" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Dates -->
|
||||
<div>
|
||||
<label class="block text-xs font-semibold text-app-textMain mb-2">Дата поста VK</label>
|
||||
<div class="space-y-2">
|
||||
<input type="date" name="date_from" value="{{ date_from }}" class="input w-full" />
|
||||
<input type="date" name="date_to" value="{{ date_to }}" class="input w-full" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Source -->
|
||||
<div>
|
||||
<label class="block text-xs font-semibold text-app-textMain mb-2">Источник</label>
|
||||
<div class="bg-app-bg border border-app-border rounded-lg p-2 max-h-48 overflow-y-auto space-y-1">
|
||||
{% for item in facets.sources %}
|
||||
<label class="flex items-center gap-2 px-2 py-1 hover:bg-app-surfaceHover rounded cursor-pointer transition-colors group">
|
||||
<input type="checkbox" name="source_id" value="{{ item.id }}" class="checkbox" {% if item.id in source_ids %}checked{% endif %} />
|
||||
<span class="text-sm text-app-textMain flex-1 truncate group-hover:text-white" title="{{ item.name }}">{{ item.name }}</span>
|
||||
<span class="badge badge-neutral bg-transparent border-none">{{ item.count }}</span>
|
||||
</label>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Category -->
|
||||
<div>
|
||||
<label class="block text-xs font-semibold text-app-textMain mb-2">Категория</label>
|
||||
<div class="bg-app-bg border border-app-border rounded-lg p-2 max-h-48 overflow-y-auto space-y-1">
|
||||
{% for item in facets.categories %}
|
||||
<label class="flex items-center gap-2 px-2 py-1 hover:bg-app-surfaceHover rounded cursor-pointer transition-colors group">
|
||||
<input type="checkbox" name="category" value="{{ item.value }}" class="checkbox" {% if item.value in categories_selected %}checked{% endif %} />
|
||||
<span class="text-sm text-app-textMain flex-1 truncate group-hover:text-white" title="{{ item.value or 'Без категории' }}">{{ item.value or "Без категории" }}</span>
|
||||
<span class="badge badge-neutral bg-transparent border-none">{{ item.count }}</span>
|
||||
</label>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<button class="btn btn-primary w-full" type="submit">Применить фильтры</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
<!-- Main Content (List) -->
|
||||
<div class="flex-1 min-w-0 flex flex-col gap-6">
|
||||
<div class="flex flex-col sm:flex-row justify-between items-start sm:items-center gap-4 bg-app-surface border border-app-border p-3 rounded-xl shadow-sm">
|
||||
<div class="text-sm text-app-textMuted flex items-center gap-2">
|
||||
<i data-lucide="layers" class="w-4 h-4"></i>
|
||||
Найдено: <span class="font-bold text-white">{{ pagination.total }}</span>
|
||||
</div>
|
||||
<div class="flex items-center gap-2">
|
||||
<span class="text-sm text-app-textMuted">На странице:</span>
|
||||
<select name="per_page" class="select select-sm w-auto py-1 pr-8" form="filter-form">
|
||||
{% for n in [10,25,50,100] %}
|
||||
<option value="{{ n }}" {% if pagination.per_page == n %}selected{% endif %}>{{ n }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="flex flex-col gap-6">
|
||||
{% for p in posts %}
|
||||
{% include "editor_post.html" %}
|
||||
{% else %}
|
||||
<div class="card p-12 flex flex-col items-center justify-center text-center">
|
||||
<i data-lucide="inbox" class="w-16 h-16 text-app-borderFocus mb-4"></i>
|
||||
<h3 class="text-lg font-bold text-white mb-1">Ничего не найдено</h3>
|
||||
<p class="text-app-textMuted text-sm">В этой очереди пока нет постов, подходящих под фильтры.</p>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
|
||||
<!-- Pagination -->
|
||||
{% if pagination.pages > 1 %}
|
||||
<div class="flex justify-center mt-4">
|
||||
<div class="tabs p-1">
|
||||
<button hx-get="?page=1" hx-include="#filter-form" hx-target="#content-area" class="btn btn-sm btn-ghost w-10 h-10 p-0 rounded-md" {% if pagination.page == 1 %}disabled{% endif %}><i data-lucide="chevrons-left" class="w-4 h-4"></i></button>
|
||||
<button hx-get="?page={{ pagination.page - 1 }}" hx-include="#filter-form" hx-target="#content-area" class="btn btn-sm btn-ghost w-10 h-10 p-0 rounded-md" {% if pagination.page == 1 %}disabled{% endif %}><i data-lucide="chevron-left" class="w-4 h-4"></i></button>
|
||||
|
||||
<span class="flex items-center justify-center px-4 text-sm font-medium text-app-textMuted bg-app-bg rounded-md border border-app-border mx-1">
|
||||
{{ pagination.page }} из {{ pagination.pages }}
|
||||
</span>
|
||||
|
||||
<button hx-get="?page={{ pagination.page + 1 }}" hx-include="#filter-form" hx-target="#content-area" class="btn btn-sm btn-ghost w-10 h-10 p-0 rounded-md" {% if pagination.page >= pagination.pages %}disabled{% endif %}><i data-lucide="chevron-right" class="w-4 h-4"></i></button>
|
||||
<button hx-get="?page={{ pagination.pages }}" hx-include="#filter-form" hx-target="#content-area" class="btn btn-sm btn-ghost w-10 h-10 p-0 rounded-md" {% if pagination.page >= pagination.pages %}disabled{% endif %}><i data-lucide="chevrons-right" class="w-4 h-4"></i></button>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
@@ -0,0 +1,213 @@
|
||||
<form hx-post="/editor/{{ p.id }}/save" hx-target="#post-actions-{{ p.id }}" hx-swap="outerHTML" hx-encoding="multipart/form-data" id="post-container-{{ p.id }}" class="card flex flex-col group relative transition-colors border border-app-border {% if p.editorial_status == 'accepted' %}bg-app-successBg/10 border-app-success/20{% elif p.editorial_status == 'rejected' %}bg-app-errorBg/10 opacity-75{% else %}hover:border-app-borderFocus hover:shadow-glow{% endif %}">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
|
||||
<div class="flex flex-col">
|
||||
<!-- Top: Meta Info & Status -->
|
||||
<div class="w-full bg-app-bg/50 border-b border-app-border p-4 flex flex-col gap-3 md:flex-row md:flex-wrap md:items-center">
|
||||
<div class="flex items-center justify-between gap-3 md:justify-start">
|
||||
<span class="text-xs font-mono text-app-textMuted tracking-wider">#{{ p.id }}</span>
|
||||
{% include "editor_post_status.html" %}
|
||||
</div>
|
||||
|
||||
<div class="flex items-center gap-2">
|
||||
<div class="w-8 h-8 rounded bg-app-surface border border-app-border flex items-center justify-center flex-shrink-0">
|
||||
<i data-lucide="link" class="w-4 h-4 text-app-textMuted"></i>
|
||||
</div>
|
||||
<div class="flex flex-col min-w-0">
|
||||
<a href="{{ p.original_url }}" target="_blank" class="text-sm font-semibold text-app-textMain hover:text-app-primary truncate transition-colors" title="{{ p.source_name }}">{{ p.source_name }}</a>
|
||||
<div class="text-xs text-app-textMuted truncate">{{ p.posted_at_fmt or p.created_at_fmt }}</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="space-y-1 md:ml-auto">
|
||||
<div class="flex justify-between gap-2 text-xs">
|
||||
<span class="text-app-textMuted">AI Оценка:</span>
|
||||
<span class="font-mono text-white font-medium">{{ p.qualification_score or "?" }}/10</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% if p.publication_platforms %}
|
||||
<div class="pt-3 border-t border-app-border/50 md:pt-0 md:border-t-0">
|
||||
<div class="flex flex-wrap items-center gap-2">
|
||||
<div class="text-xs text-app-textMuted font-semibold">Куда публикуем:</div>
|
||||
{% for pub in p.publication_platforms %}
|
||||
{% if pub.url %}
|
||||
<a href="{{ pub.url }}" target="_blank" class="badge badge-info hover:bg-blue-500/20 transition-colors" title="{{ pub.error or pub.status_label }}">
|
||||
{{ pub.label }} <i data-lucide="external-link" class="w-3 h-3 ml-1"></i>
|
||||
</a>
|
||||
{% elif pub.status == 'published' %}
|
||||
<span class="badge badge-info" title="{{ pub.error or pub.status_label }}">
|
||||
{{ pub.label }}
|
||||
</span>
|
||||
{% else %}
|
||||
<span class="badge {% if pub.status == 'error' %}badge-error{% else %}badge-neutral{% endif %}" title="{{ pub.error or pub.status_label }}">
|
||||
{{ pub.label }}
|
||||
</span>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<!-- Right: Edit Area & Spoilers -->
|
||||
<div class="flex-1 flex flex-col min-w-0 p-5">
|
||||
|
||||
<!-- Photos (Moved above text, larger preview, alpine modal gallery) -->
|
||||
{% if p.media_items %}
|
||||
{% set photo_urls = [] %}
|
||||
{% for m in p.media_items %}
|
||||
{% if m.type == 'photo' and m.url %}
|
||||
{% set _ = photo_urls.append(m.url) %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
<div class="mb-4" x-data="{ galleryOpen: false, activeIdx: 0, images: {{ photo_urls | tojson | forceescape }} }"
|
||||
@keydown.escape.window="galleryOpen = false"
|
||||
@keydown.left.window="if(galleryOpen && images.length > 1) activeIdx = (activeIdx === 0) ? images.length - 1 : activeIdx - 1"
|
||||
@keydown.right.window="if(galleryOpen && images.length > 1) activeIdx = (activeIdx === images.length - 1) ? 0 : activeIdx + 1">
|
||||
|
||||
<div class="flex flex-wrap gap-2">
|
||||
{% for m in p.media_items %}
|
||||
{% if m.type == "photo" and m.url %}
|
||||
{% set photo_idx = photo_urls.index(m.url) %}
|
||||
<div class="relative block w-32 h-32 rounded-lg overflow-hidden border border-app-border hover:border-app-primary transition-colors group/media">
|
||||
<input id="media-delete-{{ p.id }}-{{ m.id }}" type="checkbox" name="delete_media_ids" value="{{ m.id }}" class="peer sr-only">
|
||||
<img @click.prevent="activeIdx = {{ photo_idx }}; galleryOpen = true" src="{{ m.url }}" class="w-full h-full object-cover peer-checked:opacity-35" loading="lazy">
|
||||
<label for="media-delete-{{ p.id }}-{{ m.id }}" class="absolute top-1.5 right-1.5 z-10 w-7 h-7 rounded-full bg-black/70 text-white border border-white/20 flex items-center justify-center opacity-80 group-hover/media:opacity-100 peer-checked:bg-app-error peer-checked:text-white transition-colors cursor-pointer" title="Не публиковать это медиа">
|
||||
<i data-lucide="x" class="w-4 h-4"></i>
|
||||
</label>
|
||||
<span class="absolute inset-x-0 bottom-0 hidden peer-checked:block pointer-events-none bg-app-error text-white text-[10px] font-bold text-center py-1">Не публиковать</span>
|
||||
</div>
|
||||
{% elif m.url %}
|
||||
<div class="relative w-32 h-32 rounded-lg bg-app-bg border border-app-border flex items-center justify-center hover:text-app-primary transition-colors text-app-textMuted group/media">
|
||||
<input id="media-delete-{{ p.id }}-{{ m.id }}" type="checkbox" name="delete_media_ids" value="{{ m.id }}" class="peer sr-only">
|
||||
<a href="{{ m.url }}" target="_blank" class="absolute inset-0 flex items-center justify-center peer-checked:opacity-35">
|
||||
<i data-lucide="play-circle" class="w-10 h-10"></i>
|
||||
</a>
|
||||
<label for="media-delete-{{ p.id }}-{{ m.id }}" class="absolute top-1.5 right-1.5 z-10 w-7 h-7 rounded-full bg-black/70 text-white border border-white/20 flex items-center justify-center opacity-80 group-hover/media:opacity-100 peer-checked:bg-app-error peer-checked:text-white transition-colors cursor-pointer" title="Не публиковать это медиа">
|
||||
<i data-lucide="x" class="w-4 h-4"></i>
|
||||
</label>
|
||||
<span class="absolute inset-x-0 bottom-0 hidden peer-checked:block pointer-events-none bg-app-error text-white text-[10px] font-bold text-center py-1">Не публиковать</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
|
||||
<!-- Alpine Modal Gallery -->
|
||||
<div x-show="galleryOpen" style="display: none;" x-transition.opacity class="media-gallery-layer fixed inset-0 z-[100] flex items-center justify-center bg-black/90 backdrop-blur-sm p-4">
|
||||
<!-- Close button -->
|
||||
<button @click.prevent="galleryOpen = false" type="button" class="absolute top-4 right-4 text-white hover:text-app-primary transition-colors p-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" class="w-8 h-8" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2"><path stroke-linecap="round" stroke-linejoin="round" d="M6 18L18 6M6 6l12 12" /></svg>
|
||||
</button>
|
||||
|
||||
<!-- Previous -->
|
||||
<button x-show="images.length > 1" @click.prevent="activeIdx = (activeIdx === 0) ? images.length - 1 : activeIdx - 1" type="button" class="absolute left-4 top-1/2 -translate-y-1/2 text-white hover:text-app-primary p-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" class="w-10 h-10" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2"><path stroke-linecap="round" stroke-linejoin="round" d="M15 19l-7-7 7-7" /></svg>
|
||||
</button>
|
||||
|
||||
<!-- Image -->
|
||||
<img :src="images[activeIdx]" class="max-w-full max-h-[90vh] object-contain rounded-lg shadow-2xl">
|
||||
|
||||
<!-- Next -->
|
||||
<button x-show="images.length > 1" @click.prevent="activeIdx = (activeIdx === images.length - 1) ? 0 : activeIdx + 1" type="button" class="absolute right-4 top-1/2 -translate-y-1/2 text-white hover:text-app-primary p-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" class="w-10 h-10" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2"><path stroke-linecap="round" stroke-linejoin="round" d="M9 5l7 7-7 7" /></svg>
|
||||
</button>
|
||||
|
||||
<!-- Counter -->
|
||||
<div x-show="images.length > 1" class="absolute bottom-4 left-1/2 -translate-x-1/2 text-white font-mono text-sm bg-black/50 px-3 py-1 rounded-full">
|
||||
<span x-text="activeIdx + 1"></span> / <span x-text="images.length"></span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<!-- Original Spoiler -->
|
||||
<details class="group/details mb-4">
|
||||
<summary class="flex items-center gap-2 text-xs font-semibold tracking-wide text-app-textMuted hover:text-white cursor-pointer select-none">
|
||||
<i data-lucide="chevron-right" class="w-4 h-4 transition-transform group-open/details:rotate-90"></i>
|
||||
ОРИГИНАЛЬНЫЙ ТЕКСТ
|
||||
</summary>
|
||||
<div class="mt-3 bg-app-bg/50 rounded-lg p-4 border border-app-border">
|
||||
<div class="whitespace-pre-wrap text-app-textMain/80 font-mono text-xs leading-relaxed max-h-[300px] overflow-y-auto">{{ p.raw_text }}</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<!-- Edit Area (Auto resize textarea) -->
|
||||
<label class="block text-xs font-bold text-app-textMuted mb-2 uppercase tracking-wider">Финальный текст</label>
|
||||
<textarea name="final_text"
|
||||
x-data="{
|
||||
ro: null,
|
||||
resize() {
|
||||
$el.style.height = '0px';
|
||||
$el.style.height = $el.scrollHeight + 'px';
|
||||
},
|
||||
init() {
|
||||
$el.value = $el.value.replace(/\s+$/u, '');
|
||||
this.$nextTick(() => requestAnimationFrame(() => this.resize()));
|
||||
this.ro = new ResizeObserver(() => this.$nextTick(() => this.resize()));
|
||||
this.ro.observe($el);
|
||||
},
|
||||
destroy() {
|
||||
this.ro?.disconnect();
|
||||
}
|
||||
}"
|
||||
@input="resize()"
|
||||
class="textarea w-full font-sans text-sm leading-relaxed resize-none overflow-hidden p-3 bg-app-bg border border-app-border min-h-[150px]"
|
||||
placeholder="Напишите идеальный пост здесь...">{{ p.review_text.rstrip() }}</textarea>
|
||||
|
||||
<!-- Meta Inputs row -->
|
||||
<div class="grid grid-cols-1 md:grid-cols-3 gap-4 mt-4">
|
||||
<div>
|
||||
<label class="block text-xs font-bold text-app-textMuted mb-1">Категория</label>
|
||||
<select name="final_category" class="select select-sm w-full bg-app-bg">
|
||||
{% for category in categories %}
|
||||
<option value="{{ category.name }}" data-tag="{{ category.tag }}" {% if p.review_category == category.name %}selected{% endif %}>{{ category.name }}</option>
|
||||
{% endfor %}
|
||||
{% set category_names = categories | map(attribute="name") | list %}
|
||||
{% if p.review_category and p.review_category not in category_names %}
|
||||
<option value="{{ p.review_category }}" selected>{{ p.review_category }} · старая</option>
|
||||
{% endif %}
|
||||
</select>
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-xs font-bold text-app-textMuted mb-1">Тег источника</label>
|
||||
<input type="text" name="final_source_tag" value="{{ p.review_source_tag or p.source_tag or '' }}" class="input input-sm w-full bg-app-bg">
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-xs font-bold text-app-textMuted mb-1">Доп. Медиа</label>
|
||||
<input type="file" name="media_files" class="block w-full text-xs text-app-textMuted file:mr-2 file:py-1 file:px-2 file:rounded file:border-0 file:text-xs file:font-semibold file:bg-app-border file:text-white hover:file:bg-app-borderFocus cursor-pointer border border-app-border bg-app-bg rounded-md" accept="image/*,video/*,.pdf,.doc,.docx" multiple>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- AI Notes Spoiler -->
|
||||
<div class="mt-4">
|
||||
<details class="group/details">
|
||||
<summary class="flex items-center gap-2 text-xs font-semibold tracking-wide text-app-textMuted hover:text-white cursor-pointer select-none">
|
||||
<i data-lucide="chevron-right" class="w-4 h-4 transition-transform group-open/details:rotate-90"></i>
|
||||
AI ЗАМЕТКИ / РЕЗОЛЮЦИЯ
|
||||
</summary>
|
||||
<div class="mt-3 bg-app-bg/50 rounded-lg p-4 border border-app-border text-sm">
|
||||
<div class="space-y-4">
|
||||
<div>
|
||||
<div class="text-xs text-app-textMuted font-bold mb-2 uppercase tracking-wider">AI заметки / резолюция</div>
|
||||
<div class="text-app-textMain/80 whitespace-pre-wrap italic">{{ p.rewrite_notes or p.qualification_reason or "—" }}</div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="text-xs text-app-textMuted font-bold mb-2 uppercase tracking-wider">AI рерайт</div>
|
||||
<div class="whitespace-pre-wrap text-app-textMain/90 font-mono text-xs leading-relaxed max-h-[360px] overflow-y-auto">{{ p.rewritten_text or "—" }}</div>
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-xs font-bold text-app-textMuted mb-1 uppercase tracking-wider">Заметка редактора (внутренняя)</label>
|
||||
<input type="text" name="editor_notes" value="{{ p.editor_notes or '' }}" class="input input-sm w-full text-app-warning placeholder:text-app-warning/50 bg-app-bg">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
|
||||
<!-- Actions -->
|
||||
{% include "editor_post_actions.html" %}
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
@@ -0,0 +1,36 @@
|
||||
{% if p.editorial_status != 'published' %}
|
||||
<div id="post-actions-{{ p.id }}" class="mt-5 pt-4 border-t border-app-border flex flex-wrap items-center justify-between gap-3">
|
||||
<div class="flex items-center gap-3">
|
||||
{% if action_message %}
|
||||
<div class="flex items-center gap-2 text-sm font-medium {% if action_tone == 'success' %}text-app-success{% elif action_tone == 'warning' %}text-app-warning{% elif action_tone == 'error' %}text-app-error{% else %}text-app-textMuted{% endif %}">
|
||||
<i data-lucide="{% if action_tone == 'success' %}check-circle-2{% elif action_tone == 'warning' %}corner-up-left{% elif action_tone == 'error' %}x-circle{% else %}info{% endif %}" class="w-4 h-4"></i>
|
||||
{{ action_message }}
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if p.editorial_status in ['rejected', 'accepted', 'publish_failed'] %}
|
||||
<button class="btn btn-ghost btn-sm text-app-warning hover:bg-app-warning/10 hover:text-app-warning" type="button"
|
||||
hx-post="/editor/{{ p.id }}/return" hx-target="#post-actions-{{ p.id }}" hx-swap="outerHTML"
|
||||
hx-vals='{"csrf_token": "{{ user.csrf_token }}"}'>
|
||||
<i data-lucide="corner-up-left" class="w-4 h-4"></i> Вернуть на проверку
|
||||
</button>
|
||||
{% else %}
|
||||
<button class="btn btn-ghost btn-sm text-app-error hover:bg-app-errorBg hover:text-app-error" type="button"
|
||||
hx-post="/editor/{{ p.id }}/reject" hx-target="#post-actions-{{ p.id }}" hx-swap="outerHTML"
|
||||
hx-vals='{"csrf_token": "{{ user.csrf_token }}"}'>
|
||||
<i data-lucide="x" class="w-4 h-4"></i> Отклонить
|
||||
</button>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
{% if p.editorial_status not in ['rejected', 'accepted', 'publish_failed'] %}
|
||||
<div class="flex gap-2">
|
||||
<button type="submit" name="action" value="save" class="btn btn-surface btn-sm">
|
||||
<i data-lucide="save" class="w-4 h-4"></i> Сохранить как черновик
|
||||
</button>
|
||||
<button type="submit" name="action" value="accept" class="btn btn-primary btn-sm">
|
||||
<i data-lucide="send" class="w-4 h-4"></i> Сохранить и Опубликовать
|
||||
</button>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endif %}
|
||||
@@ -0,0 +1,11 @@
|
||||
<div id="post-status-{{ p.id }}" {% if status_oob %}hx-swap-oob="outerHTML"{% endif %} class="flex items-center gap-1 text-xs font-medium {% if p.editorial_status == 'accepted' or p.editorial_status == 'published' %}text-app-success bg-app-success/10{% elif p.editorial_status == 'rejected' or p.editorial_status == 'publish_failed' %}text-app-error bg-app-error/10{% elif p.editorial_status == 'regenerating' %}text-app-warning bg-app-warning/10{% else %}text-blue-400 bg-blue-400/10{% endif %} px-2 py-1 rounded-md">
|
||||
{% if p.editorial_status == 'accepted' or p.editorial_status == 'published' %}
|
||||
<i data-lucide="check-circle-2" class="w-3 h-3"></i> Принят
|
||||
{% elif p.editorial_status == 'rejected' or p.editorial_status == 'publish_failed' %}
|
||||
<i data-lucide="x-circle" class="w-3 h-3"></i> Отклонен
|
||||
{% elif p.editorial_status == 'regenerating' %}
|
||||
<i data-lucide="refresh-cw" class="w-3 h-3 animate-spin"></i> В работе
|
||||
{% else %}
|
||||
<i data-lucide="clock" class="w-3 h-3"></i> Ожидает
|
||||
{% endif %}
|
||||
</div>
|
||||
@@ -0,0 +1,3 @@
|
||||
{% include "editor_post_actions.html" %}
|
||||
{% set status_oob = true %}
|
||||
{% include "editor_post_status.html" %}
|
||||
@@ -0,0 +1,30 @@
|
||||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
<div class="flex items-center justify-center min-h-[70vh]">
|
||||
<div class="card w-96 bg-base-100 shadow-xl border border-base-200">
|
||||
<div class="card-body">
|
||||
<h2 class="card-title text-2xl font-bold justify-center mb-4">Вход в систему</h2>
|
||||
|
||||
{% if error %}
|
||||
<div class="alert alert-error text-sm py-2 mb-4">
|
||||
<span>{{ error }}</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<form method="post" action="/login" class="flex flex-col gap-4">
|
||||
<div class="form-control">
|
||||
<label class="label"><span class="label-text font-bold">Логин</span></label>
|
||||
<input name="login" autocomplete="username" class="input input-bordered" required>
|
||||
</div>
|
||||
<div class="form-control">
|
||||
<label class="label"><span class="label-text font-bold">Пароль</span></label>
|
||||
<input name="password" type="password" autocomplete="current-password" class="input input-bordered" required>
|
||||
</div>
|
||||
<div class="form-control mt-6">
|
||||
<button class="btn btn-primary w-full" type="submit">Войти</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,160 @@
|
||||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
<div class="mb-8">
|
||||
<h1 class="text-3xl font-bold tracking-tight text-white flex items-center gap-3 mb-2">
|
||||
<i data-lucide="scroll-text" class="text-app-primary w-8 h-8"></i>
|
||||
Логи
|
||||
</h1>
|
||||
<div class="text-app-textMuted text-sm">Журнал действий и запросов админки. События старше 30 дней удаляются автоматически.</div>
|
||||
</div>
|
||||
|
||||
<div class="card overflow-hidden mb-8">
|
||||
<div class="p-4 bg-app-surface border-b border-app-border">
|
||||
<form method="get" action="/logs" class="flex flex-wrap items-end gap-4">
|
||||
<div class="flex-1 min-w-[200px]">
|
||||
<label class="block text-[10px] uppercase font-bold text-app-textMuted mb-1">Поиск</label>
|
||||
<div class="relative">
|
||||
<i data-lucide="search" class="absolute left-3 top-1/2 -translate-y-1/2 w-4 h-4 text-app-textMuted"></i>
|
||||
<input name="q" value="{{ q }}" placeholder="action, entity, JSON" class="input w-full pl-9">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="w-full sm:w-auto">
|
||||
<label class="block text-[10px] uppercase font-bold text-app-textMuted mb-1">Действие</label>
|
||||
<select name="action" class="select w-full sm:w-40">
|
||||
<option value="">Все</option>
|
||||
{% for item in actions %}
|
||||
<option value="{{ item.action }}" {% if action == item.action %}selected{% endif %}>{{ item.action }} ({{ item.count }})</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="w-full sm:w-auto">
|
||||
<label class="block text-[10px] uppercase font-bold text-app-textMuted mb-1">Пользователь</label>
|
||||
<select name="actor_id" class="select w-full sm:w-40">
|
||||
<option value="0">Все</option>
|
||||
{% for item in actors %}
|
||||
<option value="{{ item.id }}" {% if actor_id == item.id %}selected{% endif %}>{{ item.login }} ({{ item.count }})</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="w-full sm:w-auto">
|
||||
<label class="block text-[10px] uppercase font-bold text-app-textMuted mb-1">Сущность</label>
|
||||
<select name="entity_type" class="select w-full sm:w-40">
|
||||
<option value="">Все</option>
|
||||
{% for item in entity_types %}
|
||||
<option value="{{ item.entity_type }}" {% if entity_type == item.entity_type %}selected{% endif %}>{{ item.entity_type }} ({{ item.count }})</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="w-full sm:w-auto">
|
||||
<label class="block text-[10px] uppercase font-bold text-app-textMuted mb-1">Период</label>
|
||||
<div class="flex gap-2">
|
||||
<input type="date" name="date_from" value="{{ date_from }}" class="input w-[130px]">
|
||||
<input type="date" name="date_to" value="{{ date_to }}" class="input w-[130px]">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="flex gap-2 w-full lg:w-auto">
|
||||
<button class="btn btn-primary flex-1 lg:flex-none" type="submit">Найти</button>
|
||||
<a class="btn btn-surface flex-1 lg:flex-none border-app-border" href="/logs">Сбросить</a>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div class="p-4 border-b border-app-border flex justify-between items-center bg-app-bg/50">
|
||||
<div class="text-sm text-app-textMuted flex items-center gap-2">
|
||||
<i data-lucide="layers" class="w-4 h-4"></i>
|
||||
Всего записей: <span class="font-bold text-white">{{ pagination.total }}</span>
|
||||
</div>
|
||||
<div class="flex items-center gap-2">
|
||||
<span class="text-sm text-app-textMuted">На странице:</span>
|
||||
<form method="get" action="/logs" class="m-0 flex items-center gap-2">
|
||||
{% for item in preserved_query %}
|
||||
{% if item.key != "per_page" %}
|
||||
<input type="hidden" name="{{ item.key }}" value="{{ item.value }}">
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
<select name="per_page" class="select select-sm w-20 py-1 pr-8" onchange="this.form.submit()">
|
||||
{% for n in [25,50,100,200] %}
|
||||
<option value="{{ n }}" {% if pagination.per_page == n %}selected{% endif %}>{{ n }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="overflow-x-auto">
|
||||
<table class="w-full text-left text-sm whitespace-nowrap">
|
||||
<thead class="bg-app-bg border-b border-app-border">
|
||||
<tr class="text-app-textMuted font-semibold tracking-wide uppercase text-xs">
|
||||
<th class="px-4 py-3 w-40">Время</th>
|
||||
<th class="px-4 py-3 w-32">Пользователь</th>
|
||||
<th class="px-4 py-3 w-48">Действие</th>
|
||||
<th class="px-4 py-3 w-48">Сущность</th>
|
||||
<th class="px-4 py-3 min-w-[300px] w-full">Детали</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody class="divide-y divide-app-border text-xs">
|
||||
{% for item in logs %}
|
||||
<tr class="hover:bg-app-surfaceHover transition-colors group">
|
||||
<td class="px-4 py-3 font-mono text-app-textMuted text-[10px]">{{ item.created_at_fmt }}</td>
|
||||
<td class="px-4 py-3 font-bold text-white">{{ item.actor_label }}</td>
|
||||
<td class="px-4 py-3"><span class="badge badge-neutral px-1.5 py-0.5 text-[10px] font-mono border-app-border">{{ item.action }}</span></td>
|
||||
<td class="px-4 py-3 font-mono text-app-textMuted text-[10px]">
|
||||
{{ item.entity_type }}
|
||||
{% if item.entity_id %}
|
||||
<span class="text-white ml-1">#{{ item.entity_id }}</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td class="px-4 py-3 whitespace-normal">
|
||||
{% if item.details_pretty %}
|
||||
<details class="group/json">
|
||||
<summary class="cursor-pointer select-none flex items-center gap-1.5 text-app-primary hover:text-white transition-colors">
|
||||
<i data-lucide="chevron-right" class="w-4 h-4 transition-transform group-open/json:rotate-90"></i>
|
||||
<span class="text-[10px] font-bold uppercase tracking-wider">JSON Payload</span>
|
||||
</summary>
|
||||
<div class="mt-2 pl-5">
|
||||
<div class="bg-[#0f1115] border border-app-border rounded-lg p-3 max-h-64 overflow-auto">
|
||||
<pre class="font-mono text-[10px] text-app-textMuted leading-relaxed">{{ item.details_pretty }}</pre>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
{% else %}
|
||||
<span class="text-app-textMuted/50">—</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% else %}
|
||||
<tr>
|
||||
<td colspan="5" class="px-4 py-12 text-center">
|
||||
<div class="flex flex-col items-center justify-center text-app-textMuted">
|
||||
<i data-lucide="search-x" class="w-12 h-12 mb-3 text-app-borderFocus"></i>
|
||||
<p class="text-sm">Записей по заданным фильтрам не найдено</p>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
{% if pagination.pages > 1 %}
|
||||
<div class="p-4 border-t border-app-border flex justify-center bg-app-bg/50">
|
||||
<div class="tabs p-1">
|
||||
<a class="btn btn-sm btn-ghost w-10 h-10 p-0 rounded-md {% if pagination.page == 1 %}pointer-events-none opacity-50{% endif %}" href="?page=1{% for k,v in preserved_query %}{% if k != 'page' %}&{{ k }}={{ v }}{% endif %}{% endfor %}"><i data-lucide="chevrons-left" class="w-4 h-4"></i></a>
|
||||
<a class="btn btn-sm btn-ghost w-10 h-10 p-0 rounded-md {% if pagination.page == 1 %}pointer-events-none opacity-50{% endif %}" href="?page={{ pagination.page - 1 }}{% for k,v in preserved_query %}{% if k != 'page' %}&{{ k }}={{ v }}{% endif %}{% endfor %}"><i data-lucide="chevron-left" class="w-4 h-4"></i></a>
|
||||
|
||||
<span class="flex items-center justify-center px-4 text-sm font-medium text-app-textMuted bg-app-bg rounded-md border border-app-border mx-1">
|
||||
{{ pagination.page }} из {{ pagination.pages }}
|
||||
</span>
|
||||
|
||||
<a class="btn btn-sm btn-ghost w-10 h-10 p-0 rounded-md {% if pagination.page >= pagination.pages %}pointer-events-none opacity-50{% endif %}" href="?page={{ pagination.page + 1 }}{% for k,v in preserved_query %}{% if k != 'page' %}&{{ k }}={{ v }}{% endif %}{% endfor %}"><i data-lucide="chevron-right" class="w-4 h-4"></i></a>
|
||||
<a class="btn btn-sm btn-ghost w-10 h-10 p-0 rounded-md {% if pagination.page >= pagination.pages %}pointer-events-none opacity-50{% endif %}" href="?page={{ pagination.pages }}{% for k,v in preserved_query %}{% if k != 'page' %}&{{ k }}={{ v }}{% endif %}{% endfor %}"><i data-lucide="chevrons-right" class="w-4 h-4"></i></a>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,37 @@
|
||||
{% if pagination and pagination.pages > 1 %}
|
||||
<div class="pagination">
|
||||
<form method="get" class="pagination-form">
|
||||
{% if preserved_query is defined %}
|
||||
{% for item in preserved_query %}
|
||||
<input type="hidden" name="{{ item.key }}" value="{{ item.value }}">
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{% if q is defined %}<input type="hidden" name="q" value="{{ q }}">{% endif %}
|
||||
{% if status_filter is defined %}<input type="hidden" name="status_filter" value="{{ status_filter }}">{% endif %}
|
||||
{% if date_from is defined %}<input type="hidden" name="date_from" value="{{ date_from }}">{% endif %}
|
||||
{% if date_to is defined %}<input type="hidden" name="date_to" value="{{ date_to }}">{% endif %}
|
||||
{% if category_filter is defined %}<input type="hidden" name="category_filter" value="{{ category_filter }}">{% endif %}
|
||||
{% if sort is defined %}<input type="hidden" name="sort" value="{{ sort }}">{% endif %}
|
||||
{% if table_filters is defined %}
|
||||
{% for item in table_filters %}
|
||||
<input type="hidden" name="f_field" value="{{ item.field }}">
|
||||
<input type="hidden" name="f_op" value="{{ item.op }}">
|
||||
<input type="hidden" name="f_value" value="{{ item.value }}">
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% if table_sorts is defined %}
|
||||
{% for item in table_sorts %}
|
||||
<input type="hidden" name="sort_field" value="{{ item.field }}">
|
||||
<input type="hidden" name="sort_dir" value="{{ item.dir }}">
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
<input type="hidden" name="per_page" value="{{ pagination.per_page }}">
|
||||
<button class="btn" name="page" value="{{ pagination.page - 1 }}" {% if not pagination.has_prev %}disabled{% endif %}>‹</button>
|
||||
<span class="muted">Страница {{ pagination.page }} из {{ pagination.pages }} · всего {{ pagination.total }}</span>
|
||||
<button class="btn" name="page" value="{{ pagination.page + 1 }}" {% if not pagination.has_next %}disabled{% endif %}>›</button>
|
||||
</form>
|
||||
</div>
|
||||
{% else %}
|
||||
{% if pagination %}<div class="pagination muted">Всего {{ pagination.total }}</div>{% endif %}
|
||||
{% endif %}
|
||||
@@ -0,0 +1,163 @@
|
||||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
<div class="mb-8 flex flex-col md:flex-row justify-between items-start md:items-center gap-4">
|
||||
<div>
|
||||
<h1 class="text-3xl font-bold">Тест промпта райтера</h1>
|
||||
<div class="text-base-content/60 mt-1">Песочница для свободной части <span class="font-mono bg-base-200 px-1 rounded text-xs">ai_writer_prompt</span>. Посты и статусы в БД не меняются.</div>
|
||||
</div>
|
||||
<a class="btn btn-outline" href="/workers">Настройки AI</a>
|
||||
</div>
|
||||
|
||||
<div class="collapse collapse-arrow bg-base-100 shadow-sm border border-base-200 mb-8" open>
|
||||
<input type="checkbox" checked />
|
||||
<div class="collapse-title text-lg font-bold">Как пользоваться</div>
|
||||
<div class="collapse-content border-t border-base-200 pt-4 grid grid-cols-1 md:grid-cols-2 gap-6 text-sm">
|
||||
<div>
|
||||
<div class="font-bold uppercase tracking-widest text-xs text-base-content/60 mb-1">Что тестируем</div>
|
||||
<p class="text-base-content/80">Отправляется только текст из поля ниже как system prompt. <span class="font-mono bg-base-200 px-1 rounded text-xs">ai_writer_contract</span> специально не добавляется, чтобы проверять стиль и структуру.</p>
|
||||
</div>
|
||||
<div>
|
||||
<div class="font-bold uppercase tracking-widest text-xs text-base-content/60 mb-1">Что уйдёт в модель</div>
|
||||
<p class="text-base-content/80">Один выбранный raw-пост в формате: categories, producer, ссылка, оценка, медиа и обрезанный исходный текст.</p>
|
||||
</div>
|
||||
<div>
|
||||
<div class="font-bold uppercase tracking-widest text-xs text-base-content/60 mb-1">Что безопасно менять</div>
|
||||
<p class="text-base-content/80">Голос канала, структуру текста, длину, правила про эмоджи, запрет выдумок, примеры формулировок.</p>
|
||||
</div>
|
||||
<div>
|
||||
<div class="font-bold uppercase tracking-widest text-xs text-base-content/60 mb-1">Что не происходит</div>
|
||||
<p class="text-base-content/80">Рерайт не сохраняется, batch не создаётся. Тест только тратит токены текущей модели райтера.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<form method="post" action="/prompt-test" class="flex flex-col lg:flex-row gap-8">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
|
||||
<section class="flex-1 flex flex-col gap-6">
|
||||
<div class="card bg-base-100 shadow-sm border border-base-200">
|
||||
<div class="card-body p-4 flex flex-col gap-4">
|
||||
|
||||
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<div class="form-control">
|
||||
<label class="label"><span class="label-text font-bold">Фильтр постов</span></label>
|
||||
<select name="q_status" data-status-select class="select select-bordered select-sm w-full">
|
||||
{% for item in status_options %}
|
||||
<option value="{{ item.value }}" {% if q_status == item.value %}selected{% endif %}>{{ item.label }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
<div class="form-control">
|
||||
<label class="label"><span class="label-text font-bold">Пост для теста</span></label>
|
||||
<select name="post_id" data-post-select class="select select-bordered select-sm w-full">
|
||||
{% for post in posts %}
|
||||
<option value="{{ post.id }}" {% if selected and post.id == selected.id %}selected{% endif %}>{{ post.label }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="flex items-center gap-2 mt-2">
|
||||
<span class="text-sm text-base-content/60">Модель:</span>
|
||||
<span class="badge badge-primary">{{ provider }}</span>
|
||||
<span class="badge badge-neutral font-mono">{{ model or "не выбрана" }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% if selected %}
|
||||
<div class="card bg-base-200 border border-base-300">
|
||||
<div class="card-body p-4 text-sm flex flex-col gap-2">
|
||||
<div class="flex justify-between items-start">
|
||||
<strong class="text-base">#{{ selected.id }} · {{ selected.source_name }}</strong>
|
||||
<a href="{{ selected.original_url }}" target="_blank" rel="noopener" class="btn btn-xs btn-outline">Оригинал</a>
|
||||
</div>
|
||||
<div class="text-base-content/60">
|
||||
{{ selected.posted_at_fmt or selected.created_at_fmt }} · {{ selected.media_count }} медиа · оценка {{ selected.qualification_score or "—" }}
|
||||
</div>
|
||||
<pre class="whitespace-pre-wrap font-mono text-xs max-h-48 overflow-y-auto mt-2">{{ selected.raw_text }}</pre>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<div class="card bg-base-100 shadow-sm border border-base-200">
|
||||
<div class="card-body p-4">
|
||||
<div class="form-control">
|
||||
<label class="label"><span class="label-text font-bold">ai_writer_prompt для теста</span></label>
|
||||
<textarea name="prompt" rows="18" spellcheck="false" class="textarea textarea-bordered font-mono text-sm leading-relaxed w-full">{{ prompt }}</textarea>
|
||||
</div>
|
||||
<div class="mt-4 flex justify-end">
|
||||
<button class="btn btn-primary" type="submit">Отправить тест</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<aside class="w-full lg:w-1/3 flex flex-col gap-6">
|
||||
<div class="card bg-base-100 shadow-sm border border-base-200">
|
||||
<div class="card-body p-4">
|
||||
<h2 class="card-title text-base">Payload (Входные данные)</h2>
|
||||
<p class="text-xs text-base-content/60 mb-2">Именно это отправится user-сообщением. Текст обрезан по настройке ai_writer_max_text_chars={{ max_text_chars }}.</p>
|
||||
<pre class="bg-base-200 p-2 rounded text-xs font-mono overflow-auto max-h-64">{{ payload_json }}</pre>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card bg-base-100 shadow-sm border border-base-200">
|
||||
<div class="card-body p-4 flex flex-col gap-4">
|
||||
<h2 class="card-title text-base">Ответ модели</h2>
|
||||
|
||||
{% if error %}
|
||||
<div class="alert alert-error text-sm">{{ error }}</div>
|
||||
|
||||
{% elif result %}
|
||||
<div class="grid grid-cols-2 gap-2 text-xs bg-base-200 p-3 rounded">
|
||||
<div class="font-bold text-base-content/60">Модель</div><div class="font-mono text-right truncate" title="{{ result.model }}">{{ result.model }}</div>
|
||||
<div class="font-bold text-base-content/60">Токены</div><div class="text-right">{{ result.usage.total_tokens or "—" }}</div>
|
||||
<div class="font-bold text-base-content/60">Стоимость</div><div class="text-right text-success">{{ result.usage.estimated_cost_usd or "—" }}</div>
|
||||
</div>
|
||||
|
||||
{% if result.preview_text %}
|
||||
<div>
|
||||
<h3 class="font-bold text-sm mb-2">Как будет выглядеть</h3>
|
||||
<div class="prose prose-sm max-w-none text-base-content whitespace-pre-wrap leading-snug bg-base-200 p-3 rounded">{{ result.preview_text }}</div>
|
||||
<div class="grid grid-cols-2 gap-2 text-xs mt-2">
|
||||
<div class="font-bold text-base-content/60">Категория</div><div class="text-right">{{ result.preview_category or "—" }}</div>
|
||||
<div class="font-bold text-base-content/60">Заметка</div><div class="text-right">{{ result.preview_notes or "—" }}</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if result.parsed_json %}
|
||||
<div>
|
||||
<h3 class="font-bold text-sm mb-2">JSON разобран</h3>
|
||||
<pre class="bg-base-200 p-2 rounded text-xs font-mono overflow-auto max-h-48">{{ result.parsed_json }}</pre>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<div>
|
||||
<h3 class="font-bold text-sm mb-2">Сырой ответ</h3>
|
||||
<pre class="bg-base-200 p-2 rounded text-xs font-mono overflow-auto max-h-48">{{ result.content }}</pre>
|
||||
</div>
|
||||
|
||||
{% else %}
|
||||
<div class="text-sm text-base-content/50 italic text-center py-8 border border-dashed border-base-300 rounded">
|
||||
После отправки здесь появится ответ модели.
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</aside>
|
||||
</form>
|
||||
|
||||
<script>
|
||||
const statusSelect = document.querySelector("[data-status-select]");
|
||||
document.querySelector("[data-post-select]")?.addEventListener("change", (event) => {
|
||||
const id = event.target.value;
|
||||
const status = statusSelect?.value || "all";
|
||||
if (id) window.location.href = `/prompt-test?post_id=${encodeURIComponent(id)}&q_status=${encodeURIComponent(status)}`;
|
||||
});
|
||||
statusSelect?.addEventListener("change", (event) => {
|
||||
window.location.href = `/prompt-test?q_status=${encodeURIComponent(event.target.value)}`;
|
||||
});
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,194 @@
|
||||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
<div class="toolbar">
|
||||
<div>
|
||||
<a class="muted" href="/raw">← Raw-посты</a>
|
||||
<h1>Raw #{{ post.id }}</h1>
|
||||
<div class="muted">
|
||||
<a href="{{ post.original_url }}" target="_blank">{{ post.source_name }}</a>
|
||||
· {{ post.source_platform }}
|
||||
· #{{ post.source_tag or "—" }}
|
||||
{% if post.posted_at_fmt %}· пост VK: {{ post.posted_at_fmt }}{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<span class="pill {{ post.stage.class }}" title="{{ post.stage.key }}">{{ post.stage.label }}</span>
|
||||
</div>
|
||||
|
||||
<div class="detail-grid">
|
||||
<section class="panel">
|
||||
<div class="media-strip detail-media">
|
||||
{% for m in post.media_items %}
|
||||
{% if m.type == "photo" and m.url %}
|
||||
<button class="media-thumb detail-thumb" type="button" data-gallery-src="{{ m.url }}" data-gallery-title="#{{ post.id }} · фото {{ loop.index }}" title="{{ m.status }}">
|
||||
<img src="{{ m.url }}" alt="photo">
|
||||
</button>
|
||||
{% elif m.url %}
|
||||
<a class="media-thumb detail-thumb media-video" href="{{ m.url }}" target="_blank" title="{{ m.error or m.status }}">
|
||||
<span>▶</span>
|
||||
</a>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% if post.media_count %}<div class="muted media-count">{{ post.media_count }} медиа</div>{% endif %}
|
||||
<div class="raw-full-text detail-text">{{ post.raw_text }}</div>
|
||||
<div class="post-meta">Загружен парсером: {{ post.created_at_fmt }}</div>
|
||||
</section>
|
||||
|
||||
<aside class="panel side-panel" id="ai">
|
||||
<h2>AI-оценка</h2>
|
||||
<a class="ai-badge detail-ai {{ post.ai_badge.class }}" href="#ai" title="{{ post.ai_badge.reason or post.ai_badge.sublabel }}">
|
||||
<span>{{ post.ai_badge.label }}</span>
|
||||
<small>{{ post.ai_badge.sublabel }}</small>
|
||||
</a>
|
||||
<dl class="kv">
|
||||
<dt>Статус</dt>
|
||||
<dd>{{ post.qualification_status or "pending" }}</dd>
|
||||
<dt>Решение модели</dt>
|
||||
<dd>{{ post.qualification_model_decision or post.qualification_decision or "—" }}</dd>
|
||||
<dt>Итог по порогу</dt>
|
||||
<dd>{{ post.qualification_decision or "—" }}</dd>
|
||||
<dt>Причина</dt>
|
||||
<dd>{{ post.qualification_reason or "—" }}</dd>
|
||||
<dt>Reject tag</dt>
|
||||
<dd>{{ post.qualification_reject_tag or "—" }}</dd>
|
||||
<dt>Модель</dt>
|
||||
<dd>{{ post.qualification_model or post.batch_model or "—" }}</dd>
|
||||
<dt>Проверен</dt>
|
||||
<dd>{{ post.qualified_at_fmt or "—" }}</dd>
|
||||
<dt>Batch</dt>
|
||||
<dd>{% if post.qualification_batch_id %}#{{ post.qualification_batch_id }}{% else %}—{% endif %}</dd>
|
||||
<dt>Токены</dt>
|
||||
<dd>{{ post.batch_total_tokens or "—" }}</dd>
|
||||
</dl>
|
||||
</aside>
|
||||
</div>
|
||||
|
||||
<section class="panel" id="rewrite">
|
||||
<div class="toolbar">
|
||||
<div>
|
||||
<h2>Рерайт</h2>
|
||||
<div class="muted">Готовый текст для будущей редакторской проверки.</div>
|
||||
</div>
|
||||
<span class="pill {% if post.rewrite_status == 'ready' %}ok{% elif post.rewrite_status == 'failed' %}bad{% elif post.rewrite_status == 'processing' %}warn{% endif %}">
|
||||
{{ post.rewrite_status or "pending" }}
|
||||
</span>
|
||||
</div>
|
||||
{% if post.rewritten_text %}
|
||||
<div class="rewrite-text">{{ post.rewritten_text }}</div>
|
||||
{% else %}
|
||||
<div class="muted">Рерайт ещё не готов.</div>
|
||||
{% endif %}
|
||||
<dl class="kv kv-wide rewrite-meta">
|
||||
<dt>Категория</dt><dd>{{ post.rewrite_category or "—" }}</dd>
|
||||
<dt>Хэштеги</dt>
|
||||
<dd>
|
||||
{% if post.rewrite_category_tag or post.rewrite_source_tag %}
|
||||
#{{ post.rewrite_category_tag or "—" }} #{{ post.rewrite_source_tag or "—" }}
|
||||
{% else %}
|
||||
—
|
||||
{% endif %}
|
||||
</dd>
|
||||
<dt>Заметка</dt><dd>{{ post.rewrite_notes or "—" }}</dd>
|
||||
<dt>Модель</dt><dd>{{ post.rewrite_model or post.writer_batch_model or "—" }}</dd>
|
||||
<dt>Готов</dt><dd>{{ post.rewritten_at_fmt or "—" }}</dd>
|
||||
<dt>Batch</dt><dd>{% if post.rewrite_batch_id %}#{{ post.rewrite_batch_id }}{% else %}—{% endif %}</dd>
|
||||
<dt>Токены</dt><dd>{{ post.writer_total_tokens or "—" }}</dd>
|
||||
<dt>Стоимость</dt><dd>{% if post.writer_estimated_cost_usd %}${{ post.writer_estimated_cost_usd }}{% else %}—{% endif %}</dd>
|
||||
</dl>
|
||||
</section>
|
||||
|
||||
{% if post.qualification_batch_id %}
|
||||
<details class="panel">
|
||||
<summary><strong>Технический ответ AI</strong></summary>
|
||||
<dl class="kv kv-wide">
|
||||
<dt>Provider</dt><dd>{{ post.qualification_provider or "—" }}</dd>
|
||||
<dt>Batch status</dt><dd>{{ post.batch_status or "—" }}</dd>
|
||||
<dt>Создан</dt><dd>{{ post.batch_created_at_fmt or "—" }}</dd>
|
||||
<dt>Завершён</dt><dd>{{ post.batch_completed_at_fmt or "—" }}</dd>
|
||||
<dt>Постов в batch</dt><dd>{{ post.batch_posts_count or "—" }}</dd>
|
||||
<dt>Accepted / rejected / maybe</dt>
|
||||
<dd>{{ post.batch_accepted_count or 0 }} / {{ post.batch_rejected_count or 0 }} / {{ post.batch_maybe_count or 0 }}</dd>
|
||||
<dt>Токены</dt>
|
||||
<dd>{{ post.batch_prompt_tokens or 0 }} / {{ post.batch_completion_tokens or 0 }} / {{ post.batch_total_tokens or 0 }}</dd>
|
||||
<dt>Стоимость</dt><dd>{% if post.batch_estimated_cost_usd %}${{ post.batch_estimated_cost_usd }}{% else %}—{% endif %}</dd>
|
||||
<dt>Ошибка</dt><dd>{{ post.batch_error or "—" }}</dd>
|
||||
</dl>
|
||||
{% if post.batch_prompt_text %}
|
||||
<details class="nested-details">
|
||||
<summary>Использованный prompt квалификатора</summary>
|
||||
<pre class="prompt-box">{{ post.batch_prompt_text }}</pre>
|
||||
</details>
|
||||
{% endif %}
|
||||
<pre class="json-box">{{ post.batch_response_pretty or "{}" }}</pre>
|
||||
</details>
|
||||
{% endif %}
|
||||
|
||||
{% if post.rewrite_batch_id %}
|
||||
<details class="panel">
|
||||
<summary><strong>Технический ответ AI-райтера</strong></summary>
|
||||
<dl class="kv kv-wide">
|
||||
<dt>Provider</dt><dd>{{ post.rewrite_provider or "—" }}</dd>
|
||||
<dt>Batch status</dt><dd>{{ post.writer_batch_status or "—" }}</dd>
|
||||
<dt>Создан</dt><dd>{{ post.writer_batch_created_at_fmt or "—" }}</dd>
|
||||
<dt>Завершён</dt><dd>{{ post.writer_batch_completed_at_fmt or "—" }}</dd>
|
||||
<dt>Постов в batch</dt><dd>{{ post.writer_batch_posts_count or "—" }}</dd>
|
||||
<dt>Ready / failed</dt><dd>{{ post.writer_batch_ready_count or 0 }} / {{ post.writer_batch_failed_count or 0 }}</dd>
|
||||
<dt>Токены</dt>
|
||||
<dd>{{ post.writer_prompt_tokens or 0 }} / {{ post.writer_completion_tokens or 0 }} / {{ post.writer_total_tokens or 0 }}</dd>
|
||||
<dt>Стоимость</dt><dd>{% if post.writer_estimated_cost_usd %}${{ post.writer_estimated_cost_usd }}{% else %}—{% endif %}</dd>
|
||||
<dt>Ошибка</dt><dd>{{ post.writer_batch_error or "—" }}</dd>
|
||||
</dl>
|
||||
{% if post.writer_batch_prompt_text %}
|
||||
<details class="nested-details">
|
||||
<summary>Использованный prompt райтера</summary>
|
||||
<pre class="prompt-box">{{ post.writer_batch_prompt_text }}</pre>
|
||||
</details>
|
||||
{% endif %}
|
||||
<pre class="json-box">{{ post.writer_batch_response_pretty or "{}" }}</pre>
|
||||
</details>
|
||||
{% endif %}
|
||||
|
||||
<div class="gallery-modal" id="galleryModal" aria-hidden="true">
|
||||
<button class="gallery-close" type="button" aria-label="Закрыть">×</button>
|
||||
<button class="gallery-nav gallery-prev" type="button" aria-label="Назад">‹</button>
|
||||
<img id="galleryImage" src="" alt="">
|
||||
<button class="gallery-nav gallery-next" type="button" aria-label="Вперёд">›</button>
|
||||
<div class="gallery-title" id="galleryTitle"></div>
|
||||
</div>
|
||||
<script>
|
||||
(() => {
|
||||
const items = Array.from(document.querySelectorAll("[data-gallery-src]"));
|
||||
const modal = document.getElementById("galleryModal");
|
||||
const image = document.getElementById("galleryImage");
|
||||
const title = document.getElementById("galleryTitle");
|
||||
let index = 0;
|
||||
|
||||
function show(nextIndex) {
|
||||
if (!items.length) return;
|
||||
index = (nextIndex + items.length) % items.length;
|
||||
const item = items[index];
|
||||
image.src = item.dataset.gallerySrc;
|
||||
title.textContent = item.dataset.galleryTitle || "";
|
||||
modal.classList.add("open");
|
||||
modal.setAttribute("aria-hidden", "false");
|
||||
}
|
||||
function close() {
|
||||
modal.classList.remove("open");
|
||||
modal.setAttribute("aria-hidden", "true");
|
||||
image.src = "";
|
||||
}
|
||||
|
||||
items.forEach((item, i) => item.addEventListener("click", () => show(i)));
|
||||
modal.querySelector(".gallery-close").addEventListener("click", close);
|
||||
modal.querySelector(".gallery-prev").addEventListener("click", () => show(index - 1));
|
||||
modal.querySelector(".gallery-next").addEventListener("click", () => show(index + 1));
|
||||
modal.addEventListener("click", (event) => { if (event.target === modal) close(); });
|
||||
document.addEventListener("keydown", (event) => {
|
||||
if (!modal.classList.contains("open")) return;
|
||||
if (event.key === "Escape") close();
|
||||
if (event.key === "ArrowLeft") show(index - 1);
|
||||
if (event.key === "ArrowRight") show(index + 1);
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,17 @@
|
||||
{% extends "base.html" %}
|
||||
|
||||
{% block body %}
|
||||
<div class="mb-6 flex flex-col md:flex-row justify-between items-start md:items-center gap-4">
|
||||
<div>
|
||||
<h1 class="text-3xl font-bold tracking-tight text-white flex items-center gap-3">
|
||||
<i data-lucide="inbox" class="text-app-primary w-8 h-8"></i>
|
||||
Сырые посты
|
||||
</h1>
|
||||
<div class="text-app-textMuted mt-1 text-sm">Этапы обработки исходных постов и AI-квалификация.</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="content-area">
|
||||
{% include "raw_posts_content.html" %}
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,223 @@
|
||||
<div class="flex flex-col xl:flex-row gap-6">
|
||||
<!-- Sidebar Filters -->
|
||||
<aside class="w-full xl:w-72 flex-shrink-0">
|
||||
<div class="card p-4 sticky top-6">
|
||||
<form id="filter-form" hx-get="/raw" hx-target="#content-area" hx-push-url="true" hx-trigger="submit">
|
||||
<input type="hidden" name="sort" value="{{ sort }}">
|
||||
|
||||
<div class="flex justify-between items-center mb-6">
|
||||
<h2 class="text-sm font-bold text-app-textMuted uppercase tracking-wider">Фильтры</h2>
|
||||
<a href="/raw" class="text-xs text-app-primary hover:text-app-primaryHover transition-colors" hx-boost="true" hx-target="#content-area">Сбросить</a>
|
||||
</div>
|
||||
|
||||
<div class="space-y-6">
|
||||
<!-- Score -->
|
||||
<div>
|
||||
<label class="block text-xs font-semibold text-app-textMain mb-2">Оценка AI</label>
|
||||
<div class="flex gap-2">
|
||||
<input type="number" name="score_min" placeholder="От" value="{{ score_min }}" class="input w-full" />
|
||||
<input type="number" name="score_max" placeholder="До" value="{{ score_max }}" class="input w-full" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Dates -->
|
||||
<div>
|
||||
<label class="block text-xs font-semibold text-app-textMain mb-2">Дата поста</label>
|
||||
<div class="space-y-2">
|
||||
<input type="date" name="date_from" value="{{ date_from }}" class="input w-full" />
|
||||
<input type="date" name="date_to" value="{{ date_to }}" class="input w-full" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Qualification Status -->
|
||||
<div>
|
||||
<label class="block text-xs font-semibold text-app-textMain mb-2">Квалификация</label>
|
||||
<div class="bg-app-bg border border-app-border rounded-lg p-2 max-h-48 overflow-y-auto space-y-1 custom-scrollbar">
|
||||
{% for item in facets.qualification_statuses %}
|
||||
<label class="flex items-center gap-2 px-2 py-1 hover:bg-app-surfaceHover rounded cursor-pointer transition-colors group">
|
||||
<input type="checkbox" name="qualification_status" value="{{ item.value }}" class="checkbox" {% if item.value in qualification_statuses %}checked{% endif %} />
|
||||
<span class="text-sm text-app-textMain flex-1 group-hover:text-white">{{ item.value }}</span>
|
||||
<span class="badge badge-neutral bg-transparent border-none">{{ item.count }}</span>
|
||||
</label>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Rewrite Status -->
|
||||
<div>
|
||||
<label class="block text-xs font-semibold text-app-textMain mb-2">Рерайт</label>
|
||||
<div class="bg-app-bg border border-app-border rounded-lg p-2 max-h-48 overflow-y-auto space-y-1 custom-scrollbar">
|
||||
{% for item in facets.rewrite_statuses %}
|
||||
<label class="flex items-center gap-2 px-2 py-1 hover:bg-app-surfaceHover rounded cursor-pointer transition-colors group">
|
||||
<input type="checkbox" name="rewrite_status" value="{{ item.value }}" class="checkbox" {% if item.value in rewrite_statuses %}checked{% endif %} />
|
||||
<span class="text-sm text-app-textMain flex-1 group-hover:text-white">{{ item.value }}</span>
|
||||
<span class="badge badge-neutral bg-transparent border-none">{{ item.count }}</span>
|
||||
</label>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Source -->
|
||||
<div>
|
||||
<label class="block text-xs font-semibold text-app-textMain mb-2">Источник</label>
|
||||
<div class="bg-app-bg border border-app-border rounded-lg p-2 max-h-48 overflow-y-auto space-y-1 custom-scrollbar">
|
||||
{% for item in facets.sources %}
|
||||
<label class="flex items-center gap-2 px-2 py-1 hover:bg-app-surfaceHover rounded cursor-pointer transition-colors group">
|
||||
<input type="checkbox" name="source_id" value="{{ item.id }}" class="checkbox" {% if item.id in source_ids %}checked{% endif %} />
|
||||
<span class="text-xs text-app-textMain flex-1 truncate group-hover:text-white" title="{{ item.name }}">{{ item.name }}</span>
|
||||
<span class="badge badge-neutral bg-transparent border-none text-[10px]">{{ item.count }}</span>
|
||||
</label>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<button class="btn btn-primary w-full" type="submit">Применить фильтры</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
<!-- Main Content (Table) -->
|
||||
<div class="flex-1 min-w-0 flex flex-col gap-6">
|
||||
<div class="flex flex-col sm:flex-row justify-between items-start sm:items-center gap-4 bg-app-surface border border-app-border p-3 rounded-xl shadow-sm">
|
||||
<div class="text-sm text-app-textMuted flex items-center gap-2">
|
||||
<i data-lucide="layers" class="w-4 h-4"></i>
|
||||
Всего: <span class="font-bold text-white">{{ pagination.total }}</span>
|
||||
</div>
|
||||
<div class="flex items-center gap-2">
|
||||
<span class="text-sm text-app-textMuted">На странице:</span>
|
||||
<select name="per_page" class="select select-sm w-auto py-1 pr-8" form="filter-form" onchange="document.getElementById('filter-form').dispatchEvent(new Event('submit', {cancelable: true}))">
|
||||
{% for n in [25,50,100,200] %}
|
||||
<option value="{{ n }}" {% if pagination.per_page == n %}selected{% endif %}>{{ n }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card overflow-hidden">
|
||||
<div class="md:hidden divide-y divide-app-border">
|
||||
{% for p in posts %}
|
||||
<a href="/raw/{{ p.id }}" class="block p-4 hover:bg-app-surfaceHover transition-colors">
|
||||
<div class="flex items-start justify-between gap-3">
|
||||
<div class="min-w-0">
|
||||
<div class="font-mono text-xs text-app-primary">#{{ p.id }}</div>
|
||||
<div class="mt-1 font-semibold text-app-textMain truncate">{{ p.source_name }}</div>
|
||||
<div class="mt-1 flex flex-wrap gap-1">
|
||||
<span class="badge badge-neutral text-[9px] uppercase tracking-wider">{{ p.status }}</span>
|
||||
{% if p.qualification_status %}
|
||||
<span class="badge badge-success text-[9px] uppercase tracking-wider bg-app-success/10 text-app-success border-app-success/20">Квалиф: {{ p.qualification_status }}</span>
|
||||
{% endif %}
|
||||
{% if p.rewrite_status %}
|
||||
<span class="badge badge-warning text-[9px] uppercase tracking-wider bg-app-warning/10 text-app-warning border-app-warning/20">Рерайт: {{ p.rewrite_status }}</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex-shrink-0 text-right">
|
||||
{% if p.qualification_score is not none %}
|
||||
<div class="font-mono font-bold text-lg {% if p.qualification_score >= 8 %}text-app-success{% elif p.qualification_score >= 5 %}text-app-warning{% else %}text-app-error{% endif %}">{{ p.qualification_score }}</div>
|
||||
{% else %}
|
||||
<span class="text-app-textMuted">—</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="mt-3 text-xs text-app-textMain/80 leading-relaxed line-clamp-3">{{ p.raw_text }}</div>
|
||||
<div class="mt-2 text-[10px] text-app-textMuted">{{ p.posted_at_fmt or p.created_at_fmt }}</div>
|
||||
</a>
|
||||
{% else %}
|
||||
<div class="px-4 py-12">
|
||||
<div class="flex flex-col items-center justify-center text-app-textMuted">
|
||||
<i data-lucide="inbox" class="w-12 h-12 mb-3 text-app-borderFocus"></i>
|
||||
<p class="text-sm">Нет данных по заданным фильтрам</p>
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
<div class="hidden md:block overflow-x-auto">
|
||||
<table class="w-full text-left text-sm whitespace-nowrap">
|
||||
<thead class="bg-app-bg border-b border-app-border">
|
||||
<tr class="text-app-textMuted font-semibold tracking-wide uppercase text-xs">
|
||||
<th class="px-4 py-3">#</th>
|
||||
<th class="px-4 py-3">Источник</th>
|
||||
<th class="px-4 py-3">Этап</th>
|
||||
<th class="px-4 py-3 min-w-[300px] w-full">Пост</th>
|
||||
<th class="px-4 py-3 text-center">AI Оценка</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody class="divide-y divide-app-border">
|
||||
{% for p in posts %}
|
||||
<tr class="hover:bg-app-surfaceHover transition-colors group">
|
||||
<td class="px-4 py-3 font-mono text-xs">
|
||||
<a href="/raw/{{ p.id }}" class="text-app-primary hover:text-white transition-colors">#{{ p.id }}</a>
|
||||
</td>
|
||||
<td class="px-4 py-3 whitespace-normal min-w-[200px]">
|
||||
<a href="{{ p.original_url }}" target="_blank" class="font-semibold text-app-textMain hover:text-app-primary transition-colors">{{ p.source_name }}</a>
|
||||
<div class="flex items-center gap-2 mt-1">
|
||||
<span class="text-[10px] text-app-textMuted font-mono bg-app-bg px-1 rounded">{{ p.source_platform }}</span>
|
||||
<span class="text-[10px] text-blue-400 font-mono">#{{ p.source_tag or "—" }}</span>
|
||||
</div>
|
||||
</td>
|
||||
<td class="px-4 py-3">
|
||||
<div class="flex flex-col items-start gap-1">
|
||||
<span class="badge badge-neutral text-[9px] uppercase tracking-wider">{{ p.status }}</span>
|
||||
{% if p.qualification_status %}
|
||||
<span class="badge badge-success text-[9px] uppercase tracking-wider bg-app-success/10 text-app-success border-app-success/20">Квалиф: {{ p.qualification_status }}</span>
|
||||
{% endif %}
|
||||
{% if p.rewrite_status %}
|
||||
<span class="badge badge-warning text-[9px] uppercase tracking-wider bg-app-warning/10 text-app-warning border-app-warning/20">Рерайт: {{ p.rewrite_status }}</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% if p.error %}
|
||||
<div class="text-[10px] text-app-error mt-1 max-w-[200px] whitespace-normal leading-tight">{{ p.error }}</div>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td class="px-4 py-3 whitespace-normal w-full max-w-sm lg:max-w-xl xl:max-w-2xl">
|
||||
<div class="text-xs text-app-textMain/80 leading-relaxed font-sans line-clamp-3">
|
||||
{{ p.raw_text }}
|
||||
</div>
|
||||
{% if p.posted_at_fmt %}
|
||||
<div class="text-[10px] text-app-textMuted mt-1">Опубликован: {{ p.posted_at_fmt }}</div>
|
||||
{% else %}
|
||||
<div class="text-[10px] text-app-textMuted mt-1">Создан: {{ p.created_at_fmt }}</div>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td class="px-4 py-3 text-center">
|
||||
{% if p.qualification_score is not none %}
|
||||
<div class="font-mono font-bold text-lg {% if p.qualification_score >= 8 %}text-app-success{% elif p.qualification_score >= 5 %}text-app-warning{% else %}text-app-error{% endif %}">{{ p.qualification_score }}</div>
|
||||
{% else %}
|
||||
<span class="text-app-textMuted">—</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% else %}
|
||||
<tr>
|
||||
<td colspan="5" class="px-4 py-12">
|
||||
<div class="flex flex-col items-center justify-center text-app-textMuted">
|
||||
<i data-lucide="inbox" class="w-12 h-12 mb-3 text-app-borderFocus"></i>
|
||||
<p class="text-sm">Нет данных по заданным фильтрам</p>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Pagination -->
|
||||
{% if pagination.pages > 1 %}
|
||||
<div class="flex justify-center mt-4">
|
||||
<div class="tabs p-1">
|
||||
<button hx-get="?page=1" hx-include="#filter-form" hx-target="#content-area" class="btn btn-sm btn-ghost w-10 h-10 p-0 rounded-md" {% if pagination.page == 1 %}disabled{% endif %}><i data-lucide="chevrons-left" class="w-4 h-4"></i></button>
|
||||
<button hx-get="?page={{ pagination.page - 1 }}" hx-include="#filter-form" hx-target="#content-area" class="btn btn-sm btn-ghost w-10 h-10 p-0 rounded-md" {% if pagination.page == 1 %}disabled{% endif %}><i data-lucide="chevron-left" class="w-4 h-4"></i></button>
|
||||
|
||||
<span class="flex items-center justify-center px-4 text-sm font-medium text-app-textMuted bg-app-bg rounded-md border border-app-border mx-1">
|
||||
{{ pagination.page }} из {{ pagination.pages }}
|
||||
</span>
|
||||
|
||||
<button hx-get="?page={{ pagination.page + 1 }}" hx-include="#filter-form" hx-target="#content-area" class="btn btn-sm btn-ghost w-10 h-10 p-0 rounded-md" {% if pagination.page >= pagination.pages %}disabled{% endif %}><i data-lucide="chevron-right" class="w-4 h-4"></i></button>
|
||||
<button hx-get="?page={{ pagination.pages }}" hx-include="#filter-form" hx-target="#content-area" class="btn btn-sm btn-ghost w-10 h-10 p-0 rounded-md" {% if pagination.page >= pagination.pages %}disabled{% endif %}><i data-lucide="chevrons-right" class="w-4 h-4"></i></button>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
@@ -0,0 +1,57 @@
|
||||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
<div class="mb-6 flex items-center justify-between">
|
||||
<div>
|
||||
<h1 class="text-3xl font-bold">{{ title }}</h1>
|
||||
</div>
|
||||
<a class="btn btn-outline" href="/sources">Назад</a>
|
||||
</div>
|
||||
|
||||
<div class="card bg-base-100 shadow-sm border border-base-200 max-w-3xl">
|
||||
<div class="card-body">
|
||||
<form method="post" action="{{ action }}" class="flex flex-col gap-4">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
|
||||
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<div class="form-control">
|
||||
<label class="label"><span class="label-text font-bold">Площадка</span></label>
|
||||
<select name="platform" class="select select-bordered w-full">
|
||||
<option value="vk" {% if not source or source.platform == "vk" %}selected{% endif %}>VK</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="form-control">
|
||||
<label class="label"><span class="label-text font-bold">Приоритет</span></label>
|
||||
<input name="priority" type="number" value="{{ source.priority if source else 100 }}" class="input input-bordered w-full">
|
||||
</div>
|
||||
|
||||
<div class="form-control md:col-span-2">
|
||||
<label class="label"><span class="label-text font-bold">Название</span></label>
|
||||
<input name="name" value="{{ source.name if source else '' }}" required class="input input-bordered w-full">
|
||||
</div>
|
||||
|
||||
<div class="form-control">
|
||||
<label class="label"><span class="label-text font-bold">Тэг</span></label>
|
||||
<input name="tag" value="{{ source.tag if source else '' }}" placeholder="academy_gear" class="input input-bordered w-full">
|
||||
</div>
|
||||
|
||||
<div class="form-control md:col-span-2">
|
||||
<label class="label"><span class="label-text font-bold">Ссылка</span></label>
|
||||
<input name="url" value="{{ source.url if source else '' }}" required class="input input-bordered w-full text-primary">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-control mt-4">
|
||||
<label class="label cursor-pointer justify-start gap-3">
|
||||
<input name="active" type="checkbox" class="toggle toggle-primary" {% if not source or source.active %}checked{% endif %}>
|
||||
<span class="label-text font-bold">Активен (парсер будет собирать посты)</span>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div class="card-actions justify-end mt-4 pt-4 border-t border-base-200">
|
||||
<button class="btn btn-primary" type="submit">Сохранить</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,63 @@
|
||||
<tr class="hover:bg-app-surfaceHover transition-colors" id="source-row-{{ s.id }}">
|
||||
<td class="hidden sm:table-cell px-4 py-3 font-mono text-[10px] text-app-textMuted">{{ s.id }}</td>
|
||||
<td class="hidden md:table-cell px-4 py-3 uppercase text-[10px] tracking-widest font-bold text-app-textMuted">{{ s.platform }}</td>
|
||||
<td class="px-4 py-3 min-w-0">
|
||||
<div class="flex items-center gap-2 min-w-0">
|
||||
<span class="font-bold text-app-textMain truncate">{{ s.name }}</span>
|
||||
</div>
|
||||
<div class="sm:hidden text-[10px] font-mono text-app-textMuted mt-1">#{{ s.id }}</div>
|
||||
<div class="text-[10px] font-mono text-app-textMuted mt-1 truncate" title="External ID">{{ s.external_id or "" }}</div>
|
||||
<div class="sm:hidden text-[10px] font-mono text-blue-400 mt-1 truncate">#{{ s.tag or "—" }}</div>
|
||||
<a href="{{ s.url }}" target="_blank" class="lg:hidden text-[10px] text-app-textMuted hover:text-white hover:underline truncate block mt-1" title="{{ s.url }}">{{ s.url }}</a>
|
||||
</td>
|
||||
<td class="hidden sm:table-cell px-4 py-3 font-mono text-[10px] text-blue-400 truncate">#{{ s.tag or "—" }}</td>
|
||||
<td class="hidden lg:table-cell px-4 py-3">
|
||||
<a href="{{ s.url }}" target="_blank" class="text-app-textMuted hover:text-white hover:underline truncate inline-block max-w-[12rem]" title="{{ s.url }}">{{ s.url }}</a>
|
||||
</td>
|
||||
<td class="px-3 sm:px-4 py-3">
|
||||
<div class="flex flex-col gap-1 items-start">
|
||||
<div class="flex items-center gap-2">
|
||||
{% if s.active %}
|
||||
<span class="flex items-center gap-1 text-[10px] font-medium text-app-success"><span class="w-1.5 h-1.5 rounded-full bg-app-success"></span> Работает</span>
|
||||
{% else %}
|
||||
<span class="flex items-center gap-1 text-[10px] font-medium text-app-error"><span class="w-1.5 h-1.5 rounded-full bg-app-error"></span> На паузе</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="flex items-center gap-1 mt-0.5">
|
||||
<span class="text-[9px] text-app-textMuted uppercase tracking-wider">Парсер:</span>
|
||||
<span class="badge {% if s.status == 'ok' %}bg-app-success/10 text-app-success border-app-success/20{% elif s.status == 'error' %}bg-app-error/10 text-app-error border-app-error/20{% else %}bg-app-bg text-app-textMuted border-app-border{% endif %} px-1.5 py-0.5 text-[9px] uppercase tracking-wider">{{ s.status }}</span>
|
||||
</div>
|
||||
{% if s.status_msg %}
|
||||
<span class="text-[9px] text-app-textMuted truncate max-w-[10rem] mt-0.5" title="{{ s.status_msg }}">{{ s.status_msg }}</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</td>
|
||||
<td class="hidden sm:table-cell px-4 py-3 text-right">
|
||||
<div class="font-bold text-app-textMain">{{ s.posts_count }}</div>
|
||||
<div class="text-[10px] text-app-success" title="За последние 24 часа">+{{ s.posts_24h }}</div>
|
||||
</td>
|
||||
<td class="hidden xl:table-cell px-4 py-3 text-right text-[10px] text-app-textMuted">{{ s.last_parsed_at_fmt or "Никогда" }}</td>
|
||||
<td class="px-3 sm:px-4 py-3 text-center">
|
||||
<div class="flex justify-center items-center gap-1">
|
||||
<a href="/sources/{{ s.id }}/edit" class="btn btn-ghost btn-icon w-8 h-8 rounded-md hover:bg-app-surface text-app-textMuted hover:text-white" title="Править">
|
||||
<i data-lucide="edit-2" class="w-4 h-4"></i>
|
||||
</a>
|
||||
<form hx-post="/sources/{{ s.id }}/toggle" hx-target="#source-row-{{ s.id }}" hx-swap="outerHTML" class="m-0">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<button class="btn btn-ghost btn-icon w-8 h-8 rounded-md hover:bg-app-surface text-app-textMuted hover:text-white" type="submit" title="{% if s.active %}Поставить на паузу{% else %}Возобновить{% endif %}">
|
||||
{% if s.active %}
|
||||
<i data-lucide="pause-circle" class="w-4 h-4 text-app-warning"></i>
|
||||
{% else %}
|
||||
<i data-lucide="play-circle" class="w-4 h-4 text-app-success"></i>
|
||||
{% endif %}
|
||||
</button>
|
||||
</form>
|
||||
<form hx-post="/sources/{{ s.id }}/delete" hx-target="#source-row-{{ s.id }}" hx-swap="outerHTML swap:1s" hx-confirm="Удалить источник {{ s.name }}?" class="m-0">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<button class="btn btn-ghost btn-icon w-8 h-8 rounded-md hover:bg-app-errorBg text-app-textMuted hover:text-app-error" type="submit" title="Удалить">
|
||||
<i data-lucide="trash-2" class="w-4 h-4"></i>
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
@@ -0,0 +1,146 @@
|
||||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
<div class="mb-8">
|
||||
<h1 class="text-3xl font-bold tracking-tight text-white flex items-center gap-3 mb-2">
|
||||
<i data-lucide="database" class="text-app-primary w-8 h-8"></i>
|
||||
Источники
|
||||
</h1>
|
||||
<div class="text-app-textMuted text-sm">VK-источники для парсинга. Название идёт в prompt, тэг — в будущие хэштеги.</div>
|
||||
</div>
|
||||
|
||||
<details class="card mb-8 group/details" {% if source_preview %}open{% endif %}>
|
||||
<summary class="p-4 flex items-center justify-between cursor-pointer select-none hover:bg-app-surfaceHover transition-colors list-none">
|
||||
<div class="flex items-center gap-3">
|
||||
<div class="w-8 h-8 rounded-lg bg-app-primary/20 text-app-primary flex items-center justify-center">
|
||||
<i data-lucide="plus" class="w-5 h-5"></i>
|
||||
</div>
|
||||
<span class="text-lg font-bold text-white">Добавить источники</span>
|
||||
</div>
|
||||
<i data-lucide="chevron-down" class="w-5 h-5 text-app-textMuted transition-transform group-open/details:rotate-180"></i>
|
||||
</summary>
|
||||
|
||||
<div class="p-6 border-t border-app-border bg-app-bg/30">
|
||||
<div class="max-w-2xl">
|
||||
<form method="post" action="/sources/preview" class="flex flex-col gap-4">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<input type="hidden" name="q" value="{{ q }}">
|
||||
<input type="hidden" name="status_filter" value="{{ status_filter }}">
|
||||
<input type="hidden" name="active_filter" value="{{ active_filter }}">
|
||||
<input type="hidden" name="per_page" value="{{ pagination.per_page }}">
|
||||
|
||||
<div>
|
||||
<label class="block text-xs font-bold text-app-textMuted mb-1 flex justify-between">
|
||||
<span>Список (Название Тэг Ссылка)</span>
|
||||
<span class="text-[10px] text-app-textMuted/60 font-normal">По 1 на строку</span>
|
||||
</label>
|
||||
<textarea name="bulk_text" class="textarea font-mono text-xs h-40 leading-relaxed" placeholder="Academy Gear academy_gear https://vk.com/academy_gear A2 Technologies a2technologies https://vk.com/a2technologies">{{ bulk_text }}</textarea>
|
||||
</div>
|
||||
|
||||
<label class="flex items-center gap-3 mt-2 cursor-pointer group">
|
||||
<input name="bulk_active" type="checkbox" class="checkbox" {% if bulk_active %}checked{% endif %}>
|
||||
<span class="text-sm font-medium text-app-textMain group-hover:text-white transition-colors">Включить после добавления</span>
|
||||
</label>
|
||||
|
||||
<button class="btn btn-surface mt-2 self-start" type="submit">
|
||||
<i data-lucide="search" class="w-4 h-4"></i> Проверить список
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
{% if source_preview %}
|
||||
<div class="mt-8 pt-6 border-t border-app-border">
|
||||
<h3 class="font-bold text-white mb-4 flex items-center gap-2"><i data-lucide="eye" class="w-4 h-4 text-app-primary"></i> Результаты проверки</h3>
|
||||
|
||||
<div class="overflow-x-auto bg-app-bg border border-app-border rounded-xl">
|
||||
<table class="w-full text-left text-sm whitespace-nowrap">
|
||||
<thead class="border-b border-app-border bg-app-surface">
|
||||
<tr class="text-app-textMuted font-semibold tracking-wide uppercase text-[10px]">
|
||||
<th class="px-4 py-3"># Стр.</th>
|
||||
<th class="px-4 py-3">Название</th>
|
||||
<th class="px-4 py-3">Тэг</th>
|
||||
<th class="px-4 py-3">Ссылка</th>
|
||||
<th class="px-4 py-3">VK ID</th>
|
||||
<th class="px-4 py-3">Статус</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody class="divide-y divide-app-border text-xs">
|
||||
{% for item in source_preview %}
|
||||
<tr class="hover:bg-app-surfaceHover transition-colors">
|
||||
<td class="px-4 py-3 text-app-textMuted font-mono">{{ item.line_no }}</td>
|
||||
<td class="px-4 py-3 font-semibold text-app-textMain">{{ item.name or "—" }}</td>
|
||||
<td class="px-4 py-3 font-mono text-blue-400">#{{ item.tag or "—" }}</td>
|
||||
<td class="px-4 py-3"><a href="{{ item.url }}" target="_blank" class="text-app-textMuted hover:text-white hover:underline">{{ item.url }}</a></td>
|
||||
<td class="px-4 py-3 font-mono text-app-textMuted text-[10px]">
|
||||
{{ item.external_id }}<br>
|
||||
{{ item.external_owner_id or "" }}
|
||||
</td>
|
||||
<td class="px-4 py-3">
|
||||
{% if item.ok %}
|
||||
<span class="badge badge-success">Готов</span>
|
||||
{% else %}
|
||||
<span class="badge badge-error" title="{{ item.error }}">Ошибка</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<form method="post" action="/sources/bulk" class="mt-6 flex flex-col sm:flex-row items-center gap-4 bg-app-warningBg border border-app-warning/20 p-4 rounded-xl">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<input type="hidden" name="bulk_payload" value='{{ source_preview|tojson }}'>
|
||||
<button class="btn btn-primary" type="submit">
|
||||
<i data-lucide="check-circle" class="w-4 h-4"></i> Добавить прошедшие проверку
|
||||
</button>
|
||||
<div class="text-sm text-app-warning flex items-center gap-2">
|
||||
<i data-lucide="info" class="w-4 h-4"></i>
|
||||
<span>Строки с ошибками и дубликаты будут автоматически пропущены.</span>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<div class="card mb-6">
|
||||
<div class="p-4 border-b border-app-border bg-app-bg/50">
|
||||
<form id="filter-form" class="flex flex-wrap gap-4 items-end" hx-get="/sources" hx-target="#content-area" hx-push-url="true">
|
||||
<div class="w-full sm:flex-1 min-w-[250px]">
|
||||
<label class="block text-xs font-bold text-app-textMuted mb-1">Поиск</label>
|
||||
<div class="relative">
|
||||
<i data-lucide="search" class="absolute left-3 top-1/2 -translate-y-1/2 w-4 h-4 text-app-textMuted"></i>
|
||||
<input name="q" value="{{ q }}" placeholder="Название, ссылка, id" class="input w-full pl-9">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="w-full sm:w-48">
|
||||
<label class="block text-xs font-bold text-app-textMuted mb-1">Состояние</label>
|
||||
<select name="active_filter" class="select w-full">
|
||||
<option value="">Все состояния</option>
|
||||
<option value="on" {% if active_filter == "on" %}selected{% endif %}>Включен (ON)</option>
|
||||
<option value="off" {% if active_filter == "off" %}selected{% endif %}>Выключен (OFF)</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="w-full sm:w-48">
|
||||
<label class="block text-xs font-bold text-app-textMuted mb-1">Статус Парсера</label>
|
||||
<select name="status_filter" class="select w-full">
|
||||
<option value="">Все статусы</option>
|
||||
{% for st in ["new","ok","paused","error"] %}
|
||||
<option value="{{ st }}" {% if status_filter == st %}selected{% endif %}>{{ st }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="w-full sm:w-auto">
|
||||
<button class="btn btn-surface w-full sm:w-auto" type="submit">Найти</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div id="content-area">
|
||||
{% include "sources_content.html" %}
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,48 @@
|
||||
<div class="overflow-hidden">
|
||||
<table class="w-full table-fixed text-left text-sm">
|
||||
<thead class="bg-app-bg border-b border-app-border">
|
||||
<tr class="text-app-textMuted font-semibold tracking-wide uppercase text-xs">
|
||||
<th class="hidden sm:table-cell px-4 py-3 w-16">#</th>
|
||||
<th class="hidden md:table-cell px-4 py-3 w-24">Платформа</th>
|
||||
<th class="px-4 py-3">Название</th>
|
||||
<th class="hidden sm:table-cell px-4 py-3 w-32">Тэг</th>
|
||||
<th class="hidden lg:table-cell px-4 py-3 w-48">Ссылка</th>
|
||||
<th class="px-3 py-3 w-28 sm:w-32">Статус</th>
|
||||
<th class="hidden sm:table-cell px-4 py-3 w-24 text-right">Посты</th>
|
||||
<th class="hidden xl:table-cell px-4 py-3 w-40 text-right">Последний парсинг</th>
|
||||
<th class="px-3 py-3 w-24 sm:w-32 text-center">Действия</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody class="divide-y divide-app-border text-xs">
|
||||
{% for s in sources %}
|
||||
{% include "source_row.html" %}
|
||||
{% else %}
|
||||
<tr>
|
||||
<td colspan="9" class="px-4 py-12 text-center">
|
||||
<div class="flex flex-col items-center justify-center text-app-textMuted">
|
||||
<i data-lucide="database" class="w-12 h-12 mb-3 text-app-borderFocus"></i>
|
||||
<p class="text-sm">Нет источников, удовлетворяющих фильтрам.</p>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<!-- Pagination bottom -->
|
||||
{% if pagination.pages > 1 %}
|
||||
<div class="flex justify-center p-4 border-t border-app-border">
|
||||
<div class="tabs p-1">
|
||||
<button hx-get="?page=1" hx-include="#filter-form" hx-target="#content-area" class="btn btn-sm btn-ghost w-10 h-10 p-0 rounded-md" {% if pagination.page == 1 %}disabled{% endif %}><i data-lucide="chevrons-left" class="w-4 h-4"></i></button>
|
||||
<button hx-get="?page={{ pagination.page - 1 }}" hx-include="#filter-form" hx-target="#content-area" class="btn btn-sm btn-ghost w-10 h-10 p-0 rounded-md" {% if pagination.page == 1 %}disabled{% endif %}><i data-lucide="chevron-left" class="w-4 h-4"></i></button>
|
||||
|
||||
<span class="flex items-center justify-center px-4 text-sm font-medium text-app-textMuted bg-app-bg rounded-md border border-app-border mx-1">
|
||||
{{ pagination.page }} из {{ pagination.pages }}
|
||||
</span>
|
||||
|
||||
<button hx-get="?page={{ pagination.page + 1 }}" hx-include="#filter-form" hx-target="#content-area" class="btn btn-sm btn-ghost w-10 h-10 p-0 rounded-md" {% if pagination.page >= pagination.pages %}disabled{% endif %}><i data-lucide="chevron-right" class="w-4 h-4"></i></button>
|
||||
<button hx-get="?page={{ pagination.pages }}" hx-include="#filter-form" hx-target="#content-area" class="btn btn-sm btn-ghost w-10 h-10 p-0 rounded-md" {% if pagination.page >= pagination.pages %}disabled{% endif %}><i data-lucide="chevrons-right" class="w-4 h-4"></i></button>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
@@ -0,0 +1,49 @@
|
||||
<div class="collapse collapse-arrow bg-base-200 border border-base-300 rounded-lg mt-4" {% if table_filters|selectattr("field")|list or table_sorts|selectattr("field")|list %}open{% endif %}>
|
||||
<input type="checkbox" {% if table_filters|selectattr("field")|list or table_sorts|selectattr("field")|list %}checked{% endif %} />
|
||||
<div class="collapse-title text-sm font-medium">Расширенные фильтры и сортировка</div>
|
||||
<div class="collapse-content border-t border-base-300 pt-4 flex flex-col lg:flex-row gap-6">
|
||||
<div class="flex-1">
|
||||
<div class="font-bold text-xs uppercase tracking-widest text-base-content/60 mb-2">Фильтры</div>
|
||||
<div class="flex flex-col gap-2">
|
||||
{% for item in table_filters %}
|
||||
<div class="flex flex-col sm:flex-row gap-2">
|
||||
<select name="f_field" class="select select-bordered select-sm w-full sm:w-1/3">
|
||||
<option value="">Поле</option>
|
||||
{% for field in field_options %}
|
||||
<option value="{{ field.value }}" {% if item.field == field.value %}selected{% endif %}>{{ field.label }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
<select name="f_op" class="select select-bordered select-sm w-full sm:w-1/4">
|
||||
{% for op in filter_op_options %}
|
||||
<option value="{{ op.value }}" {% if item.op == op.value %}selected{% endif %}>{{ op.label }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
<input name="f_value" value="{{ item.value }}" placeholder="Значение" class="input input-bordered input-sm w-full sm:w-auto flex-1">
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
<div class="text-xs text-base-content/50 mt-2">Пустые строки игнорируются. Для дат используй YYYY-MM-DD, для булевых значений — true/false.</div>
|
||||
</div>
|
||||
|
||||
<div class="flex-1">
|
||||
<div class="font-bold text-xs uppercase tracking-widest text-base-content/60 mb-2">Сортировка</div>
|
||||
<div class="flex flex-col gap-2">
|
||||
{% for item in table_sorts %}
|
||||
<div class="flex flex-col sm:flex-row gap-2">
|
||||
<select name="sort_field" class="select select-bordered select-sm w-full sm:w-1/2">
|
||||
<option value="">Поле</option>
|
||||
{% for field in field_options %}
|
||||
<option value="{{ field.value }}" {% if item.field == field.value %}selected{% endif %}>{{ field.label }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
<select name="sort_dir" class="select select-bordered select-sm w-full sm:w-1/2">
|
||||
<option value="desc" {% if item.dir == "desc" %}selected{% endif %}>По убыванию</option>
|
||||
<option value="asc" {% if item.dir == "asc" %}selected{% endif %}>По возрастанию</option>
|
||||
</select>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
<div class="text-xs text-base-content/50 mt-2">Сортировки применяются сверху вниз. Заменяют базовую сортировку страницы.</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -0,0 +1,126 @@
|
||||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
<div class="mb-8">
|
||||
<h1 class="text-3xl font-bold tracking-tight text-white flex items-center gap-3 mb-2">
|
||||
<i data-lucide="users" class="text-app-primary w-8 h-8"></i>
|
||||
Пользователи
|
||||
</h1>
|
||||
<div class="text-app-textMuted text-sm">Управление доступом к админ-панели.</div>
|
||||
</div>
|
||||
|
||||
<div class="card overflow-hidden mb-8">
|
||||
<div class="overflow-x-auto">
|
||||
<table class="w-full text-left text-sm whitespace-nowrap">
|
||||
<thead class="bg-app-bg border-b border-app-border">
|
||||
<tr class="text-app-textMuted font-semibold tracking-wide uppercase text-xs">
|
||||
<th class="px-4 py-3 w-16">ID</th>
|
||||
<th class="px-4 py-3">Логин</th>
|
||||
<th class="px-4 py-3">Роль</th>
|
||||
<th class="px-4 py-3">Создан</th>
|
||||
<th class="px-4 py-3">Статус</th>
|
||||
<th class="px-4 py-3 w-32 text-right">Действия</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody class="divide-y divide-app-border text-xs">
|
||||
{% for u in users %}
|
||||
<tr class="hover:bg-app-surfaceHover transition-colors">
|
||||
<td class="px-4 py-3 font-mono text-app-textMuted">{{ u.id }}</td>
|
||||
<td class="px-4 py-3 font-bold text-app-textMain">{{ u.login }}</td>
|
||||
<td class="px-4 py-3">
|
||||
<span class="badge badge-neutral px-2 py-0.5">{{ u.role }}</span>
|
||||
</td>
|
||||
<td class="px-4 py-3 text-app-textMuted">{{ u.created_at_fmt }}</td>
|
||||
<td class="px-4 py-3">
|
||||
{% if u.is_active %}
|
||||
<span class="badge badge-success px-2 py-0.5 flex items-center gap-1.5 w-max">
|
||||
<span class="w-1.5 h-1.5 rounded-full bg-app-success"></span> Активен
|
||||
</span>
|
||||
{% else %}
|
||||
<span class="badge badge-error px-2 py-0.5 flex items-center gap-1.5 w-max">
|
||||
<span class="w-1.5 h-1.5 rounded-full bg-app-error"></span> Заблокирован
|
||||
</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td class="px-4 py-3 text-right">
|
||||
{% if user.role == "admin" and user.id != u.id %}
|
||||
<div class="flex items-center justify-end gap-2">
|
||||
<form method="post" action="/users/{{ u.id }}/toggle" class="m-0 inline-block">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<button class="btn btn-xs {% if u.is_active %}btn-surface{% else %}btn-primary-outline{% endif %}" type="submit">
|
||||
{% if u.is_active %}Блокировать{% else %}Разблокировать{% endif %}
|
||||
</button>
|
||||
</form>
|
||||
<form method="post" action="/users/{{ u.id }}/delete" class="m-0 inline-block" onsubmit="return confirm('Точно удалить пользователя?')">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<button class="btn btn-xs text-app-error hover:bg-app-errorBg hover:text-app-error" type="submit">
|
||||
Удалить
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% if pagination.pages > 1 %}
|
||||
<div class="p-4 border-t border-app-border flex justify-center bg-app-bg/50">
|
||||
<div class="tabs p-1">
|
||||
<a class="btn btn-sm btn-ghost w-10 h-10 p-0 rounded-md {% if pagination.page == 1 %}pointer-events-none opacity-50{% endif %}" href="?page=1"><i data-lucide="chevrons-left" class="w-4 h-4"></i></a>
|
||||
<a class="btn btn-sm btn-ghost w-10 h-10 p-0 rounded-md {% if pagination.page == 1 %}pointer-events-none opacity-50{% endif %}" href="?page={{ pagination.page - 1 }}"><i data-lucide="chevron-left" class="w-4 h-4"></i></a>
|
||||
|
||||
<span class="flex items-center justify-center px-4 text-sm font-medium text-app-textMuted bg-app-bg rounded-md border border-app-border mx-1">
|
||||
{{ pagination.page }} из {{ pagination.pages }}
|
||||
</span>
|
||||
|
||||
<a class="btn btn-sm btn-ghost w-10 h-10 p-0 rounded-md {% if pagination.page >= pagination.pages %}pointer-events-none opacity-50{% endif %}" href="?page={{ pagination.page + 1 }}"><i data-lucide="chevron-right" class="w-4 h-4"></i></a>
|
||||
<a class="btn btn-sm btn-ghost w-10 h-10 p-0 rounded-md {% if pagination.page >= pagination.pages %}pointer-events-none opacity-50{% endif %}" href="?page={{ pagination.pages }}"><i data-lucide="chevrons-right" class="w-4 h-4"></i></a>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
{% if user.role == "admin" %}
|
||||
<details class="card group/details">
|
||||
<summary class="p-4 flex items-center justify-between cursor-pointer select-none hover:bg-app-surfaceHover transition-colors list-none">
|
||||
<div class="flex items-center gap-3">
|
||||
<div class="w-8 h-8 rounded-lg bg-app-primary/20 text-app-primary flex items-center justify-center">
|
||||
<i data-lucide="user-plus" class="w-5 h-5"></i>
|
||||
</div>
|
||||
<span class="text-lg font-bold text-white">Добавить пользователя</span>
|
||||
</div>
|
||||
<i data-lucide="chevron-down" class="w-5 h-5 text-app-textMuted transition-transform group-open/details:rotate-180"></i>
|
||||
</summary>
|
||||
|
||||
<div class="p-6 border-t border-app-border bg-app-bg/30">
|
||||
<form method="post" action="/users/create" class="flex flex-col md:flex-row gap-6 items-end">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
|
||||
<div class="w-full">
|
||||
<label class="block text-xs font-bold text-app-textMuted mb-1">Логин</label>
|
||||
<input name="login" class="input w-full" required>
|
||||
</div>
|
||||
|
||||
<div class="w-full">
|
||||
<label class="block text-xs font-bold text-app-textMuted mb-1">Пароль</label>
|
||||
<input name="password" type="password" class="input w-full" required minlength="10" placeholder="Минимум 10 символов">
|
||||
</div>
|
||||
|
||||
<div class="w-full md:max-w-xs">
|
||||
<label class="block text-xs font-bold text-app-textMuted mb-1">Роль</label>
|
||||
<select name="role" class="select w-full">
|
||||
<option value="editor">Редактор (editor)</option>
|
||||
<option value="viewer">Читатель (viewer)</option>
|
||||
<option value="admin">Администратор (admin)</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<button class="btn btn-primary" type="submit">
|
||||
<i data-lucide="plus" class="w-4 h-4"></i> Создать
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
</details>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,45 @@
|
||||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
<div class="panel" style="max-width:760px;margin:8vh auto 0;">
|
||||
<h1>VK OAuth</h1>
|
||||
{% if result %}
|
||||
{% if result.ok %}
|
||||
<p class="pill ok">Токен VK-постера сохранён</p>
|
||||
<dl class="kv compact-kv">
|
||||
<dt>Сообщество</dt><dd>{{ result.owner_id }}</dd>
|
||||
<dt>User ID</dt><dd>{{ result.user_id or "—" }}</dd>
|
||||
<dt>Живёт, сек</dt><dd>{{ result.expires_in or "—" }}</dd>
|
||||
<dt>Истекает</dt><dd>{{ result.expires_at or "—" }}</dd>
|
||||
<dt>Refresh token</dt><dd>{% if result.has_refresh_token %}OK{% else %}Не получен{% endif %}</dd>
|
||||
<dt>wall.get</dt><dd>{% if result.wall_get_ok %}OK{% else %}Ошибка: {{ result.wall_get_error }}{% endif %}</dd>
|
||||
<dt>Фото upload</dt>
|
||||
<dd>
|
||||
{% if result.photo_upload_server_ok %}
|
||||
OK
|
||||
{% else %}
|
||||
Недоступно: {{ result.photo_upload_server_error }}
|
||||
<div class="muted">VK выдаёт право photos для upload API отдельно; постер всё равно опубликует текст и существующие VK-вложения.</div>
|
||||
{% endif %}
|
||||
</dd>
|
||||
</dl>
|
||||
<p><a class="btn primary" href="/workers">Вернуться к воркерам</a></p>
|
||||
{% else %}
|
||||
<p class="pill bad">VK OAuth не завершился</p>
|
||||
<p>{{ result.error_description or result.error or "Неизвестная ошибка" }}</p>
|
||||
<p><a class="btn" href="/workers">Вернуться к воркерам</a> <a class="btn primary" href="/vk-oauth/start">Повторить</a></p>
|
||||
{% endif %}
|
||||
{% elif code %}
|
||||
<p class="muted">VK вернул code. Новый обработчик должен обменивать его автоматически через <span class="mono">/vk-oauth/callback</span>.</p>
|
||||
<label>code</label>
|
||||
<textarea readonly style="min-height:110px;" onclick="this.select()">{{ code }}</textarea>
|
||||
{% if state and expected_state and state != expected_state %}
|
||||
<p class="pill bad">state не совпал, такой code лучше не использовать.</p>
|
||||
{% endif %}
|
||||
{% elif error %}
|
||||
<p class="pill bad">{{ error }}</p>
|
||||
<p>{{ error_description }}</p>
|
||||
{% else %}
|
||||
<p class="muted">VK вернул callback без данных.</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,396 @@
|
||||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
<div class="mb-8">
|
||||
<h1 class="text-3xl font-bold tracking-tight text-white flex items-center gap-3 mb-2">
|
||||
<i data-lucide="cpu" class="text-app-primary w-8 h-8"></i>
|
||||
Система и Воркеры
|
||||
</h1>
|
||||
<div class="text-app-textMuted text-sm">Управление фоновыми процессами, категориями, расписанием и настройками AI.</div>
|
||||
</div>
|
||||
|
||||
<!-- Workers Panel -->
|
||||
<div class="card overflow-hidden mb-8">
|
||||
<div class="overflow-x-auto">
|
||||
<table class="w-full text-left text-sm whitespace-nowrap">
|
||||
<thead class="bg-app-bg border-b border-app-border">
|
||||
<tr class="text-app-textMuted font-semibold tracking-wide uppercase text-xs">
|
||||
<th class="px-4 py-3">Воркер</th>
|
||||
<th class="px-4 py-3">Состояние</th>
|
||||
<th class="px-4 py-3">Heartbeat</th>
|
||||
<th class="px-4 py-3">Статус</th>
|
||||
<th class="px-4 py-3">Текущая задача</th>
|
||||
<th class="px-4 py-3">Действия</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody class="divide-y divide-app-border text-xs">
|
||||
{% for w in workers %}
|
||||
<tr class="hover:bg-app-surfaceHover transition-colors">
|
||||
<td class="px-4 py-3 font-mono font-bold text-white">{{ w.name }}</td>
|
||||
<td class="px-4 py-3">
|
||||
{% if w.enabled %}
|
||||
<span class="badge badge-success px-2 py-0.5 flex items-center gap-1.5 w-max">
|
||||
<span class="w-1.5 h-1.5 rounded-full bg-app-success"></span> Включен
|
||||
</span>
|
||||
{% else %}
|
||||
<span class="badge badge-error px-2 py-0.5 flex items-center gap-1.5 w-max">
|
||||
<span class="w-1.5 h-1.5 rounded-full bg-app-error"></span> Выключен
|
||||
</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td class="px-4 py-3 text-app-textMuted">{{ w.heartbeat_at or "—" }}</td>
|
||||
<td class="px-4 py-3 max-w-xs text-app-textMain">
|
||||
<div class="truncate">{{ w.status or "—" }}</div>
|
||||
{% if w.meta_json and w.meta_json.error %}
|
||||
<div class="mt-1 text-[10px] text-app-error whitespace-normal break-words">{{ w.meta_json.error }}</div>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td class="px-4 py-3 font-mono text-app-textMuted">{{ w.current_job_id or "—" }}</td>
|
||||
<td class="px-4 py-3">
|
||||
<form method="post" action="/workers/{{ w.name }}/toggle" class="m-0">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<button class="btn btn-xs btn-surface" type="submit">Переключить</button>
|
||||
</form>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="grid grid-cols-1 xl:grid-cols-2 gap-8 mb-8">
|
||||
<!-- Categories -->
|
||||
<details class="card group/details" open>
|
||||
<summary class="p-4 flex items-center justify-between cursor-pointer select-none hover:bg-app-surfaceHover transition-colors border-b border-app-border list-none">
|
||||
<div class="flex items-center gap-2 font-bold text-lg text-white">
|
||||
<i data-lucide="tags" class="w-5 h-5 text-app-primary"></i> Категории публикаций
|
||||
</div>
|
||||
<i data-lucide="chevron-down" class="w-5 h-5 text-app-textMuted transition-transform group-open/details:rotate-180"></i>
|
||||
</summary>
|
||||
<div class="p-4 flex flex-col gap-6 bg-app-bg/30">
|
||||
<div class="text-sm text-app-textMuted">
|
||||
Название описывает категорию для AI, тэг используется в соцсетях.
|
||||
</div>
|
||||
|
||||
<form method="post" action="/categories/create" class="flex flex-col gap-4 bg-app-surface p-4 rounded-xl border border-app-border">
|
||||
<div class="font-bold text-sm text-white">Добавить категорию</div>
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<div>
|
||||
<label class="block text-[10px] uppercase font-bold text-app-textMuted mb-1">Название (для AI)</label>
|
||||
<input name="name" class="input w-full" required>
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-[10px] uppercase font-bold text-app-textMuted mb-1">Тэг (для хэштегов)</label>
|
||||
<input name="tag" class="input w-full">
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-[10px] uppercase font-bold text-app-textMuted mb-1">На сайте</label>
|
||||
<input name="site_name" class="input w-full" required>
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-[10px] uppercase font-bold text-app-textMuted mb-1">URL-slug</label>
|
||||
<input name="site_slug" class="input w-full" pattern="[a-z0-9-]+" required>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex items-center justify-between mt-2">
|
||||
<label class="cursor-pointer flex items-center gap-2 group">
|
||||
<input type="checkbox" name="site_enabled" class="checkbox" checked>
|
||||
<span class="text-sm font-medium text-app-textMain group-hover:text-white transition-colors">Публиковать</span>
|
||||
</label>
|
||||
<button class="btn btn-primary btn-sm" type="submit">Добавить</button>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
<div class="overflow-x-auto border border-app-border rounded-xl">
|
||||
<table class="w-full text-left text-xs whitespace-nowrap bg-app-surface">
|
||||
<thead class="bg-app-bg border-b border-app-border text-app-textMuted uppercase tracking-wider text-[10px]">
|
||||
<tr>
|
||||
<th class="px-3 py-2">Название / Тэг</th>
|
||||
<th class="px-3 py-2">Сайт / Slug</th>
|
||||
<th class="px-3 py-2 text-center">ID</th>
|
||||
<th class="px-3 py-2">Статус</th>
|
||||
<th class="px-3 py-2 w-16"></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody class="divide-y divide-app-border">
|
||||
{% for c in categories %}
|
||||
<tr class="hover:bg-app-surfaceHover transition-colors">
|
||||
<td class="px-3 py-2">
|
||||
<form id="category-update-{{ c.id }}" method="post" action="/categories/{{ c.id }}/update" class="m-0">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
</form>
|
||||
<div class="flex flex-col gap-1.5">
|
||||
<input form="category-update-{{ c.id }}" name="name" value="{{ c.name }}" class="input input-sm w-full bg-app-bg text-white h-7">
|
||||
<input form="category-update-{{ c.id }}" name="tag" value="{{ c.tag }}" class="input input-sm w-full bg-app-bg text-blue-400 font-mono h-7">
|
||||
</div>
|
||||
</td>
|
||||
<td class="px-3 py-2">
|
||||
<div class="flex flex-col gap-1.5">
|
||||
<input form="category-update-{{ c.id }}" name="site_name" value="{{ c.site_name }}" class="input input-sm w-full bg-app-bg text-white h-7" required>
|
||||
<input form="category-update-{{ c.id }}" name="site_slug" value="{{ c.site_slug }}" pattern="[a-z0-9-]+" class="input input-sm w-full bg-app-bg text-app-textMuted font-mono h-7" required>
|
||||
</div>
|
||||
</td>
|
||||
<td class="px-3 py-2 text-center font-mono text-app-textMuted text-[10px]">{{ c.sort_order }}</td>
|
||||
<td class="px-3 py-2">
|
||||
<div class="flex flex-col gap-2">
|
||||
<span class="badge {% if c.is_active %}badge-success{% else %}badge-neutral{% endif %} w-full py-0.5 text-[10px]">{% if c.is_active %}Активна{% else %}Выкл{% endif %}</span>
|
||||
<label class="cursor-pointer flex items-center gap-2 bg-app-bg px-2 py-1 rounded border border-app-border">
|
||||
<input form="category-update-{{ c.id }}" type="checkbox" name="site_enabled" class="checkbox w-3.5 h-3.5 rounded-sm" {% if c.site_enabled %}checked{% endif %}>
|
||||
<span class="text-[10px] font-medium text-app-textMuted">Сайт</span>
|
||||
</label>
|
||||
</div>
|
||||
</td>
|
||||
<td class="px-3 py-2">
|
||||
<div class="flex flex-col gap-1 items-center">
|
||||
<button form="category-update-{{ c.id }}" type="submit" class="btn btn-ghost btn-icon w-7 h-7 hover:text-app-success" title="Сохранить"><i data-lucide="check" class="w-4 h-4"></i></button>
|
||||
<form method="post" action="/categories/{{ c.id }}/toggle" class="m-0 w-full flex justify-center">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<button type="submit" class="btn btn-ghost btn-icon w-7 h-7" title="Вкл/Выкл"><i data-lucide="{% if c.is_active %}pause{% else %}play{% endif %}" class="w-4 h-4"></i></button>
|
||||
</form>
|
||||
<form method="post" action="/categories/{{ c.id }}/delete" class="m-0 w-full flex justify-center" onsubmit="return confirm('Удалить категорию?');">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<button type="submit" class="btn btn-ghost btn-icon w-7 h-7 text-app-textMuted hover:text-app-error hover:bg-app-errorBg" title="Удалить"><i data-lucide="trash-2" class="w-4 h-4"></i></button>
|
||||
</form>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<div class="flex flex-col gap-8">
|
||||
<!-- TG Schedule -->
|
||||
<details class="card group/details" open>
|
||||
<summary class="p-4 flex items-center justify-between cursor-pointer select-none hover:bg-app-surfaceHover transition-colors border-b border-app-border list-none">
|
||||
<div class="flex items-center gap-2 font-bold text-lg text-white">
|
||||
<i data-lucide="send" class="w-5 h-5 text-[#0088cc]"></i> Расписание TG-постера
|
||||
</div>
|
||||
<i data-lucide="chevron-down" class="w-5 h-5 text-app-textMuted transition-transform group-open/details:rotate-180"></i>
|
||||
</summary>
|
||||
<div class="p-4 flex flex-col gap-6 bg-app-bg/30">
|
||||
<div class="text-xs text-app-textMuted leading-relaxed">
|
||||
Время по Екб. Нужны: воркер <span class="font-mono bg-app-surface border border-app-border px-1.5 py-0.5 rounded text-[10px] text-white">tg-poster</span> и настройка <span class="font-mono bg-app-surface border border-app-border px-1.5 py-0.5 rounded text-[10px] text-white">tg_poster_enabled</span>.
|
||||
</div>
|
||||
|
||||
<!-- Active Schedule Chips -->
|
||||
<div class="flex flex-wrap gap-2">
|
||||
{% for row in tg_schedule %}
|
||||
{% if row.enabled %}
|
||||
<div class="inline-flex items-center gap-2 px-3 py-1.5 bg-app-surface border border-app-primary/30 rounded-lg shadow-sm group">
|
||||
<span class="font-mono text-sm font-bold text-white">{{ row.time[:2] }}:00</span>
|
||||
<span class="text-xs text-app-textMuted border-l border-app-border pl-2">{{ row.count }} шт.</span>
|
||||
<form method="post" action="/tg-poster-schedule/{{ row.id }}/delete" class="m-0 ml-1 flex items-center">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<button type="submit" class="text-app-textMuted hover:text-app-error transition-colors" title="Удалить">
|
||||
<i data-lucide="x" class="w-3.5 h-3.5"></i>
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% else %}
|
||||
<div class="text-sm text-app-textMuted italic py-2">Расписание пусто</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
|
||||
<form method="post" action="/tg-poster-schedule/create" class="flex flex-wrap sm:flex-nowrap items-end gap-3 bg-app-surface p-4 rounded-xl border border-app-border">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<input type="hidden" name="enabled" value="on">
|
||||
<div class="w-full sm:w-32">
|
||||
<label class="block text-[10px] uppercase font-bold text-app-textMuted mb-1">Время (час)</label>
|
||||
<select name="time" class="select select-sm w-full font-mono">
|
||||
{% for h in range(24) %}
|
||||
{% set hh = "%02d" % h %}
|
||||
<option value="{{ hh }}:00">{{ hh }}:00</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
<div class="w-full sm:w-24">
|
||||
<label class="block text-[10px] uppercase font-bold text-app-textMuted mb-1">Кол-во</label>
|
||||
<input type="number" name="count" value="1" min="1" max="20" class="input input-sm w-full text-center" required>
|
||||
</div>
|
||||
<button class="btn btn-primary btn-sm h-8 w-full sm:w-auto" type="submit">
|
||||
<i data-lucide="plus" class="w-4 h-4"></i> Добавить
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<!-- VK Schedule -->
|
||||
<details class="card group/details" open>
|
||||
<summary class="p-4 flex items-center justify-between cursor-pointer select-none hover:bg-app-surfaceHover transition-colors border-b border-app-border list-none">
|
||||
<div class="flex items-center gap-2 font-bold text-lg text-white">
|
||||
<i data-lucide="layout-template" class="w-5 h-5 text-[#4680C2]"></i> Расписание VK-постера
|
||||
</div>
|
||||
<i data-lucide="chevron-down" class="w-5 h-5 text-app-textMuted transition-transform group-open/details:rotate-180"></i>
|
||||
</summary>
|
||||
<div class="p-4 flex flex-col gap-6 bg-app-bg/30">
|
||||
<div class="text-xs text-app-textMuted leading-relaxed">
|
||||
Время по Екб. Нужны: воркер <span class="font-mono bg-app-surface border border-app-border px-1.5 py-0.5 rounded text-[10px] text-white">vk-poster</span> и <span class="font-mono bg-app-surface border border-app-border px-1.5 py-0.5 rounded text-[10px] text-white">vk_poster_enabled</span>.
|
||||
</div>
|
||||
<div><a class="btn btn-surface btn-sm border-app-border" href="/vk-oauth/start"><i data-lucide="key" class="w-3.5 h-3.5"></i> Получить VK user token</a></div>
|
||||
|
||||
<!-- Active Schedule Chips -->
|
||||
<div class="flex flex-wrap gap-2">
|
||||
{% for row in vk_schedule %}
|
||||
{% if row.enabled %}
|
||||
<div class="inline-flex items-center gap-2 px-3 py-1.5 bg-app-surface border border-app-primary/30 rounded-lg shadow-sm group">
|
||||
<span class="font-mono text-sm font-bold text-white">{{ row.time[:2] }}:00</span>
|
||||
<span class="text-xs text-app-textMuted border-l border-app-border pl-2">{{ row.count }} шт.</span>
|
||||
<form method="post" action="/vk-poster-schedule/{{ row.id }}/delete" class="m-0 ml-1 flex items-center">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<button type="submit" class="text-app-textMuted hover:text-app-error transition-colors" title="Удалить">
|
||||
<i data-lucide="x" class="w-3.5 h-3.5"></i>
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% else %}
|
||||
<div class="text-sm text-app-textMuted italic py-2">Расписание пусто</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
|
||||
<form method="post" action="/vk-poster-schedule/create" class="flex flex-wrap sm:flex-nowrap items-end gap-3 bg-app-surface p-4 rounded-xl border border-app-border">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<input type="hidden" name="enabled" value="on">
|
||||
<div class="w-full sm:w-32">
|
||||
<label class="block text-[10px] uppercase font-bold text-app-textMuted mb-1">Время (час)</label>
|
||||
<select name="time" class="select select-sm w-full font-mono">
|
||||
{% for h in range(24) %}
|
||||
{% set hh = "%02d" % h %}
|
||||
<option value="{{ hh }}:00">{{ hh }}:00</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
<div class="w-full sm:w-24">
|
||||
<label class="block text-[10px] uppercase font-bold text-app-textMuted mb-1">Кол-во</label>
|
||||
<input type="number" name="count" value="1" min="1" max="20" class="input input-sm w-full text-center" required>
|
||||
</div>
|
||||
<button class="btn btn-primary btn-sm h-8 w-full sm:w-auto" type="submit">
|
||||
<i data-lucide="plus" class="w-4 h-4"></i> Добавить
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="flex items-center gap-4 mb-6 mt-12">
|
||||
<div class="h-px bg-app-border flex-1"></div>
|
||||
<h2 class="text-2xl font-bold text-white flex items-center gap-3">
|
||||
<i data-lucide="settings" class="w-6 h-6 text-app-primary"></i> Глобальные настройки
|
||||
</h2>
|
||||
<div class="h-px bg-app-border flex-1"></div>
|
||||
</div>
|
||||
|
||||
<div class="flex flex-col gap-6">
|
||||
{% for category, rows in settings|groupby("category") %}
|
||||
<details class="card group/details" {% if category == "AI Qualifier" or loop.first %}open{% endif %}>
|
||||
<summary class="p-4 flex items-center justify-between cursor-pointer select-none hover:bg-app-surfaceHover transition-colors border-b border-app-border list-none">
|
||||
<div class="text-lg font-bold text-white">{{ category_titles.get(category, category) }}</div>
|
||||
<i data-lucide="chevron-down" class="w-5 h-5 text-app-textMuted transition-transform group-open/details:rotate-180"></i>
|
||||
</summary>
|
||||
<div class="p-6 bg-app-bg/30">
|
||||
|
||||
<div class="flex flex-col gap-8">
|
||||
{% for s in rows %}
|
||||
<div class="flex flex-col md:flex-row gap-6 p-5 bg-app-surface border border-app-border rounded-xl shadow-sm">
|
||||
<!-- Description Sidebar -->
|
||||
<div class="w-full md:w-1/3 flex flex-col gap-1 border-b md:border-b-0 md:border-r border-app-border pb-4 md:pb-0 pr-0 md:pr-4">
|
||||
<div class="font-mono font-bold text-sm text-app-primary">{{ s.key }}</div>
|
||||
<div class="text-sm font-semibold text-white mt-1">{{ s.title }}</div>
|
||||
<div class="text-xs text-app-textMuted mt-2 leading-relaxed">{{ s.description }}</div>
|
||||
</div>
|
||||
|
||||
<!-- Control Panel -->
|
||||
<div class="flex-1">
|
||||
<form method="post" action="/settings/save" class="flex flex-col items-start gap-3 w-full h-full justify-center">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<input type="hidden" name="key" value="{{ s.key }}">
|
||||
|
||||
<div class="flex gap-3 w-full items-start">
|
||||
<div class="flex-1">
|
||||
{% if s.value_type == "bool" %}
|
||||
<select name="value" class="select w-32">
|
||||
<option value="true" {% if s.value_json == true %}selected{% endif %}>true</option>
|
||||
<option value="false" {% if s.value_json == false %}selected{% endif %}>false</option>
|
||||
</select>
|
||||
|
||||
{% elif s.key in ["ai_qualifier_provider", "ai_writer_provider"] %}
|
||||
<select name="value" class="select max-w-sm w-full">
|
||||
{% for p in provider_options %}
|
||||
<option value="{{ p.value }}" {% if s.value_json == p.value %}selected{% endif %}>{{ p.label }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
|
||||
{% elif s.key == "site_poster_provider" %}
|
||||
<select name="value" class="select max-w-sm w-full">
|
||||
{% for p in site_poster_provider_options %}
|
||||
<option value="{{ p.value }}" {% if s.value_json == p.value %}selected{% endif %}>{{ p.label }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
|
||||
{% elif s.key == "ai_qualifier_model" %}
|
||||
<select name="value" class="select max-w-sm w-full">
|
||||
{% for m in ai_model_options %}
|
||||
<option value="{{ m.value }}" {% if s.value_json == m.value %}selected{% endif %}>{{ m.label }}</option>
|
||||
{% endfor %}
|
||||
{% if s.value_json and s.value_json not in ai_model_options|map(attribute="value")|list %}
|
||||
<option value="{{ s.value_json }}">{{ s.value_json }} · текущее значение</option>
|
||||
{% endif %}
|
||||
</select>
|
||||
|
||||
{% elif s.key == "ai_writer_model" %}
|
||||
<select name="value" class="select max-w-sm w-full">
|
||||
{% for m in writer_model_options %}
|
||||
<option value="{{ m.value }}" {% if s.value_json == m.value %}selected{% endif %}>{{ m.label }}</option>
|
||||
{% endfor %}
|
||||
{% if s.value_json and s.value_json not in writer_model_options|map(attribute="value")|list %}
|
||||
<option value="{{ s.value_json }}">{{ s.value_json }} · текущее значение</option>
|
||||
{% endif %}
|
||||
</select>
|
||||
|
||||
{% elif s.value_type == "secret" %}
|
||||
<input type="password" name="value" value="{{ s.value_json }}" class="input w-full max-w-md" autocomplete="off" spellcheck="false" placeholder="Секретный ключ">
|
||||
|
||||
{% elif s.value_type == "text" %}
|
||||
<div class="w-full flex flex-col gap-3">
|
||||
{% if prompt_hints and s.key in prompt_hints %}
|
||||
<div class="bg-app-primary/10 border border-app-primary/30 text-app-textMain text-xs p-4 rounded-lg flex flex-col gap-2">
|
||||
<strong class="text-app-primary font-bold flex items-center gap-2"><i data-lucide="info" class="w-4 h-4"></i> {{ prompt_hints[s.key].title }}</strong>
|
||||
<div class="leading-relaxed">{{ prompt_hints[s.key].body }}</div>
|
||||
<div class="grid grid-cols-1 sm:grid-cols-2 gap-4 mt-2 pt-2 border-t border-app-primary/20">
|
||||
<div><span class="font-bold text-app-textMain uppercase tracking-wider text-[10px]">Как безопасно менять:</span><br>{{ prompt_hints[s.key].safe }}</div>
|
||||
<div><span class="font-bold text-app-textMain uppercase tracking-wider text-[10px]">На вход:</span><br>{{ prompt_hints[s.key].input }}</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
<textarea name="value" class="textarea font-mono text-xs w-full leading-relaxed p-4 bg-app-bg" rows="12">{{ s.value_json }}</textarea>
|
||||
</div>
|
||||
|
||||
{% elif s.value_type == "json" %}
|
||||
<textarea name="value" class="textarea font-mono text-xs w-full max-w-2xl leading-relaxed p-4 bg-app-bg" rows="5" spellcheck="false">{{ s.value_json | tojson }}</textarea>
|
||||
|
||||
{% else %}
|
||||
<input name="value" value="{{ s.value_json }}" class="input w-full max-w-md">
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<button class="btn btn-primary" type="submit">
|
||||
<i data-lucide="save" class="w-4 h-4"></i> Сохранить
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</details>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,107 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def normalize_hash_tag(value: str, fallback: str = "source") -> str:
|
||||
tag = (value or "").strip().lstrip("#").lower()
|
||||
tag = re.sub(r"\s+", "_", tag)
|
||||
tag = re.sub(r"[^\wа-яё_]+", "_", tag, flags=re.IGNORECASE)
|
||||
tag = re.sub(r"_+", "_", tag).strip("_")
|
||||
return tag or fallback
|
||||
|
||||
|
||||
def strip_trailing_hashtag_line(text: str) -> str:
|
||||
lines = str(text or "").rstrip().splitlines()
|
||||
while lines and not lines[-1].strip():
|
||||
lines.pop()
|
||||
if not lines:
|
||||
return ""
|
||||
parts = lines[-1].split()
|
||||
if parts and all(part.startswith("#") and normalize_hash_tag(part, "") for part in parts):
|
||||
lines.pop()
|
||||
return "\n".join(lines).rstrip()
|
||||
|
||||
|
||||
def publication_hashtags(category_tag: str, source_tag: str) -> str:
|
||||
tags = [
|
||||
normalize_hash_tag(category_tag, "category"),
|
||||
normalize_hash_tag(source_tag, "source"),
|
||||
]
|
||||
return " ".join(f"#{tag}" for tag in tags if tag)
|
||||
|
||||
|
||||
import html
|
||||
|
||||
|
||||
def build_publication_text(
|
||||
text: str,
|
||||
category_tag: str,
|
||||
source_tag: str,
|
||||
format_title: bool = False,
|
||||
parse_mode: str | None = None,
|
||||
) -> str:
|
||||
base = strip_trailing_hashtag_line(text)
|
||||
lines = base.splitlines()
|
||||
sep_pattern = re.compile(r"^\s*[━—─\-=\*\#_]{2,}\s*$")
|
||||
|
||||
cleaned_lines = []
|
||||
for line in lines:
|
||||
if sep_pattern.match(line):
|
||||
cleaned_lines.append("")
|
||||
else:
|
||||
cleaned_lines.append(line)
|
||||
|
||||
title_idx = None
|
||||
for idx, line in enumerate(cleaned_lines):
|
||||
if line.strip():
|
||||
title_idx = idx
|
||||
break
|
||||
|
||||
if title_idx is None:
|
||||
return publication_hashtags(category_tag, source_tag)
|
||||
|
||||
formatted_lines = []
|
||||
for idx, line in enumerate(cleaned_lines):
|
||||
if idx == title_idx and format_title:
|
||||
if parse_mode == "html":
|
||||
escaped_title = html.escape(line.strip())
|
||||
formatted_lines.append(f"<b>{escaped_title}</b>")
|
||||
elif parse_mode == "markdown":
|
||||
formatted_lines.append(f"**{line.strip()}**")
|
||||
else:
|
||||
formatted_lines.append(line.strip())
|
||||
|
||||
# Ensure a newline break (blank line) after the bold title
|
||||
if idx + 1 < len(cleaned_lines) and cleaned_lines[idx + 1].strip() != "":
|
||||
formatted_lines.append("")
|
||||
else:
|
||||
if parse_mode == "html":
|
||||
formatted_lines.append(html.escape(line))
|
||||
else:
|
||||
formatted_lines.append(line)
|
||||
|
||||
raw_body = "\n".join(formatted_lines)
|
||||
# Collapse multiple consecutive blank lines to at most two newlines (\n\n)
|
||||
normalized_body = re.sub(r"\n{3,}", "\n\n", raw_body).strip()
|
||||
hashtags = publication_hashtags(category_tag, source_tag)
|
||||
if hashtags:
|
||||
return f"{normalized_body}\n\n{hashtags}" if normalized_body else hashtags
|
||||
return normalized_body
|
||||
|
||||
|
||||
def parse_categories(value: object) -> list[str]:
|
||||
if isinstance(value, list):
|
||||
raw_items = [str(item) for item in value]
|
||||
else:
|
||||
text = str(value or "")
|
||||
raw_items = re.split(r"[\n,|]+", text)
|
||||
categories: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for item in raw_items:
|
||||
category = item.strip().strip("#")
|
||||
key = category.lower()
|
||||
if category and key not in seen:
|
||||
categories.append(category)
|
||||
seen.add(key)
|
||||
return categories
|
||||
@@ -0,0 +1,304 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
from loguru import logger
|
||||
|
||||
from .config import settings
|
||||
|
||||
|
||||
class VKAPIError(RuntimeError):
|
||||
def __init__(self, code: int | None, message: str) -> None:
|
||||
self.code = code
|
||||
super().__init__(f"VK API error {code}: {message}")
|
||||
|
||||
|
||||
class VKRateLimiter:
|
||||
def __init__(self, rps: int = 3) -> None:
|
||||
self.rps = max(1, int(rps))
|
||||
self.interval = 1.0 / self.rps
|
||||
self._last = 0.0
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def acquire(self) -> None:
|
||||
async with self._lock:
|
||||
now = time.monotonic()
|
||||
wait = self.interval - (now - self._last)
|
||||
if wait > 0:
|
||||
await asyncio.sleep(wait)
|
||||
self._last = time.monotonic()
|
||||
|
||||
|
||||
class VKAPIClient:
|
||||
base_url = "https://api.vk.com/method"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
token: str | None = None,
|
||||
version: str | None = None,
|
||||
rps: int = 3,
|
||||
timeout_total_sec: int = 60,
|
||||
timeout_connect_sec: int = 10,
|
||||
rate_limit_sleep_sec: float = 1.0,
|
||||
retry_attempts: int = 3,
|
||||
retry_min_delay_sec: float = 2.0,
|
||||
retry_max_delay_sec: float = 10.0,
|
||||
) -> None:
|
||||
self.token = token or settings.vk_access_token
|
||||
self.version = version or settings.vk_api_version
|
||||
self.limiter = VKRateLimiter(rps)
|
||||
self.timeout_total_sec = max(1, int(timeout_total_sec))
|
||||
self.timeout_connect_sec = max(1, int(timeout_connect_sec))
|
||||
self.rate_limit_sleep_sec = max(0.1, float(rate_limit_sleep_sec))
|
||||
self.retry_attempts = max(1, int(retry_attempts))
|
||||
self.retry_min_delay_sec = max(0.1, float(retry_min_delay_sec))
|
||||
self.retry_max_delay_sec = max(self.retry_min_delay_sec, float(retry_max_delay_sec))
|
||||
self.session: aiohttp.ClientSession | None = None
|
||||
|
||||
async def __aenter__(self) -> "VKAPIClient":
|
||||
self.session = aiohttp.ClientSession(
|
||||
timeout=aiohttp.ClientTimeout(total=self.timeout_total_sec, connect=self.timeout_connect_sec)
|
||||
)
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args) -> None:
|
||||
if self.session:
|
||||
await self.session.close()
|
||||
|
||||
async def call(self, method: str, **params: Any) -> dict | list:
|
||||
if not self.session:
|
||||
raise RuntimeError("VKAPIClient is not initialized")
|
||||
if not self.token:
|
||||
raise RuntimeError("VK_ACCESS_TOKEN is empty")
|
||||
|
||||
payload = dict(params)
|
||||
payload["access_token"] = self.token
|
||||
payload["v"] = self.version
|
||||
|
||||
for attempt in range(1, self.retry_attempts + 1):
|
||||
try:
|
||||
await self.limiter.acquire()
|
||||
async with self.session.post(f"{self.base_url}/{method}", data=payload) as resp:
|
||||
resp.raise_for_status()
|
||||
data = await resp.json(content_type=None)
|
||||
if "error" not in data:
|
||||
return data.get("response", {})
|
||||
|
||||
err = data["error"]
|
||||
code = err.get("error_code")
|
||||
message = err.get("error_msg", "unknown")
|
||||
if code == 6 and attempt < self.retry_attempts:
|
||||
logger.warning("VK rate limit hit, sleeping {}s", self.rate_limit_sleep_sec)
|
||||
await asyncio.sleep(self.rate_limit_sleep_sec)
|
||||
continue
|
||||
raise VKAPIError(code, message)
|
||||
except VKAPIError:
|
||||
raise
|
||||
except Exception:
|
||||
if attempt >= self.retry_attempts:
|
||||
raise
|
||||
delay = min(self.retry_min_delay_sec * (2 ** (attempt - 1)), self.retry_max_delay_sec)
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
raise VKAPIError(None, "retry exhausted")
|
||||
|
||||
async def resolve_group(self, input_value: str) -> tuple[str, int, str]:
|
||||
external_id = normalize_vk_source(input_value)
|
||||
if external_id.lstrip("-").isdigit():
|
||||
group_id = abs(int(external_id))
|
||||
response = await self.call("groups.getById", group_id=str(group_id))
|
||||
else:
|
||||
response = await self.call("groups.getById", group_id=external_id)
|
||||
items = response if isinstance(response, list) else response.get("groups", [])
|
||||
if not items:
|
||||
raise VKAPIError(None, f"cannot resolve VK group: {input_value}")
|
||||
group = items[0]
|
||||
group_id = int(group["id"])
|
||||
screen_name = str(group.get("screen_name") or external_id)
|
||||
name = str(group.get("name") or screen_name)
|
||||
return screen_name, -group_id, name
|
||||
|
||||
async def get_wall_posts(self, owner_id: int, count: int, offset: int = 0) -> dict:
|
||||
response = await self.call("wall.get", owner_id=owner_id, count=count, offset=offset, filter="owner")
|
||||
if not isinstance(response, dict):
|
||||
raise VKAPIError(None, "wall.get returned non-object response")
|
||||
return response
|
||||
|
||||
async def get_wall_upload_server(self, group_id: int) -> str:
|
||||
response = await self.call("photos.getWallUploadServer", group_id=abs(int(group_id)))
|
||||
if not isinstance(response, dict) or not response.get("upload_url"):
|
||||
raise VKAPIError(None, "photos.getWallUploadServer returned no upload_url")
|
||||
return str(response["upload_url"])
|
||||
|
||||
async def upload_wall_photo_bytes(self, group_id: int, data: bytes, filename: str = "photo.jpg") -> dict:
|
||||
if not self.session:
|
||||
raise RuntimeError("VKAPIClient is not initialized")
|
||||
upload_url = await self.get_wall_upload_server(group_id)
|
||||
form = aiohttp.FormData()
|
||||
form.add_field("photo", data, filename=filename, content_type="image/jpeg")
|
||||
async with self.session.post(upload_url, data=form) as resp:
|
||||
uploaded = await resp.json(content_type=None)
|
||||
saved = await self.call(
|
||||
"photos.saveWallPhoto",
|
||||
group_id=abs(int(group_id)),
|
||||
photo=uploaded.get("photo"),
|
||||
server=uploaded.get("server"),
|
||||
hash=uploaded.get("hash"),
|
||||
)
|
||||
if not isinstance(saved, list) or not saved:
|
||||
raise VKAPIError(None, f"photos.saveWallPhoto returned invalid response: {json.dumps(saved)[:300]}")
|
||||
return saved[0]
|
||||
|
||||
async def upload_wall_photo_url(self, group_id: int, url: str) -> dict:
|
||||
if not self.session:
|
||||
raise RuntimeError("VKAPIClient is not initialized")
|
||||
async with self.session.get(url) as resp:
|
||||
resp.raise_for_status()
|
||||
content_type = resp.headers.get("content-type") or "image/jpeg"
|
||||
data = await resp.read()
|
||||
suffix = ".jpg"
|
||||
if "png" in content_type:
|
||||
suffix = ".png"
|
||||
elif "webp" in content_type:
|
||||
suffix = ".webp"
|
||||
return await self.upload_wall_photo_bytes(group_id, data, filename=f"photo{suffix}")
|
||||
|
||||
async def create_wall_post(
|
||||
self,
|
||||
owner_id: int,
|
||||
message: str,
|
||||
attachments: list[str],
|
||||
from_group: bool = True,
|
||||
) -> int:
|
||||
response = await self.call(
|
||||
"wall.post",
|
||||
owner_id=int(owner_id),
|
||||
from_group=1 if from_group else 0,
|
||||
message=message or "",
|
||||
attachments=",".join(attachments) if attachments else "",
|
||||
)
|
||||
if not isinstance(response, dict) or "post_id" not in response:
|
||||
raise VKAPIError(None, f"wall.post returned invalid response: {response}")
|
||||
return int(response["post_id"])
|
||||
|
||||
|
||||
def normalize_vk_source(value: str) -> str:
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
return ""
|
||||
if "vk.com" in raw:
|
||||
raw = raw.split("vk.com", 1)[1]
|
||||
raw = raw.lstrip("/")
|
||||
raw = raw.split("?", 1)[0].split("#", 1)[0].strip()
|
||||
raw = re.sub(r"^(club|public)", "", raw, flags=re.IGNORECASE)
|
||||
return raw.lower()
|
||||
|
||||
|
||||
def is_repost(post: dict) -> bool:
|
||||
return bool(post.get("copy_history"))
|
||||
|
||||
|
||||
def is_deleted_or_invalid(post: dict) -> bool:
|
||||
if post.get("is_deleted") or post.get("deleted"):
|
||||
return True
|
||||
if not post.get("id") or not post.get("date"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_fatal_source_error(error: Exception | str) -> bool:
|
||||
message = str(error or "").lower()
|
||||
fatal_patterns = (
|
||||
"vk api error 15:",
|
||||
"vk api error 18:",
|
||||
"vk api error 19:",
|
||||
"vk api error 100:",
|
||||
"vk api error 113:",
|
||||
"vk api error 1051:",
|
||||
"vk api error 200:",
|
||||
"vk api error 201:",
|
||||
"vk api error 203:",
|
||||
"[15]",
|
||||
"[18]",
|
||||
"[19]",
|
||||
"[100]",
|
||||
"[113]",
|
||||
"[1051]",
|
||||
"[200]",
|
||||
"[201]",
|
||||
"[203]",
|
||||
"access denied",
|
||||
"private profile",
|
||||
"cannot resolve vk group",
|
||||
"cannot resolve screen_name",
|
||||
"method is unavailable with current profile type",
|
||||
"wall is disabled",
|
||||
)
|
||||
return any(pattern in message for pattern in fatal_patterns)
|
||||
|
||||
|
||||
def post_vk_url(owner_id: int, post_id: int | str) -> str:
|
||||
return f"https://vk.com/wall{int(owner_id)}_{post_id}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExtractedMedia:
|
||||
media_type: str
|
||||
original_url: str | None
|
||||
attachment_id: str
|
||||
width: int | None = None
|
||||
height: int | None = None
|
||||
duration_sec: int | None = None
|
||||
sort_order: int = 0
|
||||
|
||||
|
||||
def extract_media(post: dict) -> list[ExtractedMedia]:
|
||||
out: list[ExtractedMedia] = []
|
||||
for idx, att in enumerate(post.get("attachments", []) or []):
|
||||
att_type = att.get("type")
|
||||
if att_type == "photo":
|
||||
photo = att.get("photo") or {}
|
||||
sizes = sorted(
|
||||
photo.get("sizes", []) or [],
|
||||
key=lambda s: int(s.get("width") or 0) * int(s.get("height") or 0),
|
||||
reverse=True,
|
||||
)
|
||||
best = sizes[0] if sizes else {}
|
||||
owner_id = photo.get("owner_id")
|
||||
media_id = photo.get("id")
|
||||
if owner_id is None or media_id is None:
|
||||
continue
|
||||
out.append(
|
||||
ExtractedMedia(
|
||||
media_type="photo",
|
||||
original_url=best.get("url"),
|
||||
attachment_id=f"photo{owner_id}_{media_id}",
|
||||
width=best.get("width"),
|
||||
height=best.get("height"),
|
||||
sort_order=idx,
|
||||
)
|
||||
)
|
||||
elif att_type == "video":
|
||||
video = att.get("video") or {}
|
||||
owner_id = video.get("owner_id")
|
||||
media_id = video.get("id")
|
||||
if owner_id is None or media_id is None:
|
||||
continue
|
||||
out.append(
|
||||
ExtractedMedia(
|
||||
media_type="video",
|
||||
original_url=f"https://vk.com/video{owner_id}_{media_id}",
|
||||
attachment_id=f"video{owner_id}_{media_id}",
|
||||
width=video.get("width"),
|
||||
height=video.get("height"),
|
||||
duration_sec=video.get("duration"),
|
||||
sort_order=idx,
|
||||
)
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from aiogram import Bot
|
||||
from aiogram.exceptions import TelegramRetryAfter
|
||||
from loguru import logger
|
||||
|
||||
from ..config import settings
|
||||
from ..db import fetch_setting
|
||||
|
||||
|
||||
def parse_recipients(value: Any) -> list[int]:
|
||||
if isinstance(value, list):
|
||||
raw_items = value
|
||||
elif isinstance(value, str):
|
||||
raw = value.strip()
|
||||
if not raw:
|
||||
return []
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
raw_items = parsed if isinstance(parsed, list) else [parsed]
|
||||
except json.JSONDecodeError:
|
||||
raw_items = re.split(r"[\s,;]+", raw)
|
||||
else:
|
||||
raw_items = [value]
|
||||
|
||||
recipients: list[int] = []
|
||||
for item in raw_items:
|
||||
try:
|
||||
recipient_id = int(str(item).strip())
|
||||
except Exception:
|
||||
continue
|
||||
if recipient_id not in recipients:
|
||||
recipients.append(recipient_id)
|
||||
return recipients
|
||||
|
||||
|
||||
async def send_ai_worker_error_alert(worker_name: str, model: str, post_ids: list[int], error: str) -> None:
|
||||
token = (
|
||||
str(await fetch_setting("daily_report_bot_token", "") or "").strip()
|
||||
or str(await fetch_setting("tg_poster_bot_token", "") or "").strip()
|
||||
or settings.tg_bot_token
|
||||
)
|
||||
recipients = parse_recipients(await fetch_setting("daily_report_recipient_ids", [442509142]))
|
||||
if not token or not recipients:
|
||||
logger.warning("AI worker alert skipped: token or recipients are empty")
|
||||
return
|
||||
|
||||
post_part = ", ".join(str(post_id) for post_id in post_ids) if post_ids else "-"
|
||||
text = (
|
||||
"AI worker batch failed\n"
|
||||
f"worker: {worker_name}\n"
|
||||
f"model: {model or '-'}\n"
|
||||
f"posts: {post_part}\n"
|
||||
f"error: {error[:1000]}"
|
||||
)
|
||||
bot = Bot(token=token)
|
||||
try:
|
||||
for recipient_id in recipients:
|
||||
while True:
|
||||
try:
|
||||
await bot.send_message(recipient_id, text, disable_web_page_preview=True)
|
||||
break
|
||||
except TelegramRetryAfter as exc:
|
||||
await asyncio.sleep(float(exc.retry_after) + 1)
|
||||
finally:
|
||||
await bot.session.close()
|
||||
@@ -0,0 +1,479 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
import litellm
|
||||
from loguru import logger
|
||||
|
||||
from ..config import settings
|
||||
from ..constants import WORKER_AI_QUALIFIER
|
||||
from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool
|
||||
from ..heartbeat import HeartbeatReporter
|
||||
from ..jobs import is_worker_enabled
|
||||
from .ai_alerts import send_ai_worker_error_alert
|
||||
|
||||
|
||||
def now_utc() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def prompt_hash(prompt: str) -> str:
|
||||
return hashlib.sha256((prompt or "").encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def normalize_prompt(prompt: str) -> str:
|
||||
return (prompt or "").replace("\\r\\n", "\n").replace("\\n", "\n").strip()
|
||||
|
||||
|
||||
DEFAULT_CONTRACT_PROMPT = """
|
||||
OUTPUT SCHEMA (return array matching input order):
|
||||
{"results":[{"id":123,"score":8,"decision":"accepted","reason":"до 10 слов на русском","reject_tag":null}]}
|
||||
|
||||
Rules:
|
||||
- Input is a JSON array of posts.
|
||||
- Return JSON only. No markdown. No text outside JSON.
|
||||
- score must be integer 1..10.
|
||||
- decision must be exactly one of: "accepted", "rejected", "maybe".
|
||||
- reject_tag (rejected/maybe only) must be one of: "meme", "no_product", "politics", "discount_only", "off_topic", "vacancy", "low_content", "wrong_language", "injection", "weapon", null.
|
||||
- Return one result for every input post id.
|
||||
"""
|
||||
|
||||
|
||||
def build_system_prompt(user_prompt: str, contract_prompt: str) -> str:
|
||||
parts = [normalize_prompt(user_prompt), normalize_prompt(contract_prompt or DEFAULT_CONTRACT_PROMPT)]
|
||||
return "\n\n".join(part for part in parts if part).strip()
|
||||
|
||||
|
||||
def normalize_model(provider: str, model: str) -> str:
|
||||
provider = (provider or "").strip().lower()
|
||||
model = (model or "").strip()
|
||||
if not model:
|
||||
return model
|
||||
if "/" in model:
|
||||
return model
|
||||
if provider in {"openrouter", "anthropic", "gemini", "vertex_ai", "bedrock"}:
|
||||
return f"{provider}/{model}"
|
||||
return model
|
||||
|
||||
|
||||
def parse_ai_json(content: str) -> dict:
|
||||
raw = (content or "").strip()
|
||||
if raw.startswith("```"):
|
||||
raw = raw.strip("`")
|
||||
if raw.lower().startswith("json"):
|
||||
raw = raw[4:].strip()
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
def response_usage(response: Any) -> dict[str, Any]:
|
||||
usage = getattr(response, "usage", None)
|
||||
if usage is None and isinstance(response, dict):
|
||||
usage = response.get("usage")
|
||||
|
||||
def get(name: str) -> int | None:
|
||||
if usage is None:
|
||||
return None
|
||||
value = getattr(usage, name, None)
|
||||
if value is None and isinstance(usage, dict):
|
||||
value = usage.get(name)
|
||||
try:
|
||||
return int(value) if value is not None else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
cost = getattr(response, "_hidden_params", None)
|
||||
if isinstance(cost, dict):
|
||||
cost = cost.get("response_cost")
|
||||
else:
|
||||
cost = None
|
||||
try:
|
||||
cost_value = Decimal(str(cost)) if cost is not None else None
|
||||
except Exception:
|
||||
cost_value = None
|
||||
|
||||
return {
|
||||
"prompt_tokens": get("prompt_tokens"),
|
||||
"completion_tokens": get("completion_tokens"),
|
||||
"total_tokens": get("total_tokens"),
|
||||
"estimated_cost_usd": cost_value,
|
||||
}
|
||||
|
||||
|
||||
def validate_results(data: dict, expected_ids: set[int], min_score: int) -> list[dict]:
|
||||
results = data.get("results")
|
||||
if not isinstance(results, list) and isinstance(data.get("data"), dict):
|
||||
results = data["data"].get("results")
|
||||
if not isinstance(results, list):
|
||||
raise ValueError("AI response has no results list")
|
||||
|
||||
out: list[dict] = []
|
||||
seen: set[int] = set()
|
||||
for item in results:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
post_id = int(item.get("id"))
|
||||
if post_id not in expected_ids:
|
||||
raise ValueError(f"AI returned unexpected id={post_id}")
|
||||
score = max(1, min(10, int(item.get("score"))))
|
||||
decision = str(item.get("decision") or "").strip().lower()
|
||||
if decision not in {"accepted", "rejected", "maybe"}:
|
||||
decision = "accepted" if score >= min_score else "rejected"
|
||||
reason = str(item.get("reason") or "").strip()[:1000]
|
||||
reject_tag = item.get("reject_tag")
|
||||
reject_tag = str(reject_tag).strip().lower()[:80] if reject_tag not in {None, ""} else None
|
||||
seen.add(post_id)
|
||||
out.append({"id": post_id, "score": score, "decision": decision, "reason": reason, "reject_tag": reject_tag})
|
||||
|
||||
missing = expected_ids - seen
|
||||
if missing:
|
||||
raise ValueError(f"AI response missing ids: {sorted(missing)[:10]}")
|
||||
return out
|
||||
|
||||
|
||||
class AIQualifierWorker:
|
||||
def __init__(self) -> None:
|
||||
self.pool = None
|
||||
self.heartbeat = HeartbeatReporter(WORKER_AI_QUALIFIER, 30)
|
||||
|
||||
async def init(self) -> None:
|
||||
self.pool = await get_pool()
|
||||
|
||||
async def claim_posts(self, batch_size: int) -> list[dict]:
|
||||
async with self.pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
count = await conn.fetchval(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM raw_posts
|
||||
WHERE status='storage_ready'
|
||||
AND COALESCE(qualification_status, 'pending') = 'pending'
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media m
|
||||
WHERE m.raw_post_id=raw_posts.id
|
||||
AND COALESCE(m.editor_hidden, FALSE)=FALSE
|
||||
AND m.media_type IN ('photo', 'video')
|
||||
AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') <> ''
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media m
|
||||
WHERE m.raw_post_id=raw_posts.id
|
||||
AND COALESCE(m.editor_hidden, FALSE)=FALSE
|
||||
AND m.media_type IN ('photo', 'video')
|
||||
AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = ''
|
||||
)
|
||||
"""
|
||||
)
|
||||
if int(count or 0) < batch_size:
|
||||
return []
|
||||
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
WITH cte AS (
|
||||
SELECT id
|
||||
FROM raw_posts
|
||||
WHERE status='storage_ready'
|
||||
AND COALESCE(qualification_status, 'pending') = 'pending'
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media m
|
||||
WHERE m.raw_post_id=raw_posts.id
|
||||
AND COALESCE(m.editor_hidden, FALSE)=FALSE
|
||||
AND m.media_type IN ('photo', 'video')
|
||||
AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') <> ''
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media m
|
||||
WHERE m.raw_post_id=raw_posts.id
|
||||
AND COALESCE(m.editor_hidden, FALSE)=FALSE
|
||||
AND m.media_type IN ('photo', 'video')
|
||||
AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = ''
|
||||
)
|
||||
ORDER BY created_at ASC, id ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT $1
|
||||
)
|
||||
UPDATE raw_posts rp
|
||||
SET qualification_status='processing',
|
||||
updated_at=NOW()
|
||||
FROM cte
|
||||
WHERE rp.id=cte.id
|
||||
RETURNING rp.id, rp.raw_text, rp.original_url, rp.storage_post_url, rp.created_at,
|
||||
rp.source_id,
|
||||
(SELECT s.name FROM sources s WHERE s.id=rp.source_id) AS source_name,
|
||||
(SELECT COUNT(*) FROM raw_post_media m WHERE m.raw_post_id=rp.id) AS media_count,
|
||||
(SELECT ARRAY_AGG(DISTINCT m.media_type ORDER BY m.media_type)
|
||||
FROM raw_post_media m WHERE m.raw_post_id=rp.id) AS media_types
|
||||
""",
|
||||
batch_size,
|
||||
)
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
async def create_batch(self, provider: str, model: str, prompt: str, payload: dict) -> int:
|
||||
return int(
|
||||
await self.pool.fetchval(
|
||||
"""
|
||||
INSERT INTO ai_qualification_batches(provider, model, prompt_hash, prompt_text, posts_count, request_json)
|
||||
VALUES($1, $2, $3, $4, $5, $6::jsonb)
|
||||
RETURNING id
|
||||
""",
|
||||
provider,
|
||||
model,
|
||||
prompt_hash(prompt),
|
||||
prompt,
|
||||
len(payload),
|
||||
json.dumps(payload, ensure_ascii=False),
|
||||
)
|
||||
)
|
||||
|
||||
async def finish_batch(
|
||||
self,
|
||||
batch_id: int,
|
||||
status: str,
|
||||
response: Any = None,
|
||||
error: str | None = None,
|
||||
usage: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
counts = {"accepted": 0, "rejected": 0, "maybe": 0}
|
||||
if isinstance(response, dict) and isinstance(response.get("results"), list):
|
||||
for item in response["results"]:
|
||||
decision = str(item.get("decision") or "")
|
||||
if decision in counts:
|
||||
counts[decision] += 1
|
||||
usage = usage or {}
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE ai_qualification_batches
|
||||
SET status=$2,
|
||||
response_json=$3::jsonb,
|
||||
error=$4,
|
||||
accepted_count=$5,
|
||||
rejected_count=$6,
|
||||
maybe_count=$7,
|
||||
prompt_tokens=$8,
|
||||
completion_tokens=$9,
|
||||
total_tokens=$10,
|
||||
estimated_cost_usd=$11,
|
||||
completed_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
batch_id,
|
||||
status,
|
||||
json.dumps(response, ensure_ascii=False) if response is not None else None,
|
||||
error[:2000] if error else None,
|
||||
counts["accepted"],
|
||||
counts["rejected"],
|
||||
counts["maybe"],
|
||||
usage.get("prompt_tokens"),
|
||||
usage.get("completion_tokens"),
|
||||
usage.get("total_tokens"),
|
||||
usage.get("estimated_cost_usd"),
|
||||
)
|
||||
|
||||
async def mark_posts_after_failed_batch(self, post_ids: list[int], error: str, max_attempts: int = 2) -> tuple[list[int], list[int]]:
|
||||
if not post_ids:
|
||||
return [], []
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
WITH target AS (
|
||||
SELECT unnest($1::bigint[]) AS id
|
||||
),
|
||||
attempts AS (
|
||||
SELECT t.id, COUNT(b.id) AS failed_batches
|
||||
FROM target t
|
||||
LEFT JOIN ai_qualification_batches b
|
||||
ON b.status='failed'
|
||||
AND b.request_json @> jsonb_build_array(jsonb_build_object('id', t.id))
|
||||
GROUP BY t.id
|
||||
)
|
||||
UPDATE raw_posts
|
||||
SET qualification_status=CASE WHEN attempts.failed_batches < $3 THEN 'pending' ELSE 'failed' END,
|
||||
qualification_reason=CASE
|
||||
WHEN attempts.failed_batches < $3 THEN $2 || ' (retry once)'
|
||||
ELSE $2
|
||||
END,
|
||||
updated_at=NOW()
|
||||
FROM attempts
|
||||
WHERE raw_posts.id=attempts.id
|
||||
RETURNING raw_posts.id, raw_posts.qualification_status
|
||||
""",
|
||||
post_ids,
|
||||
error[:900],
|
||||
max_attempts,
|
||||
)
|
||||
retry_ids = [int(row["id"]) for row in rows if row["qualification_status"] == "pending"]
|
||||
failed_ids = [int(row["id"]) for row in rows if row["qualification_status"] == "failed"]
|
||||
return retry_ids, failed_ids
|
||||
|
||||
async def report_failed_batch(self, model: str, post_ids: list[int], error: str) -> None:
|
||||
meta = {"error": error[:300], "post_ids": post_ids, "model": model}
|
||||
await self.heartbeat.beat(self.pool, status="error", meta=meta, force=True)
|
||||
try:
|
||||
await send_ai_worker_error_alert(WORKER_AI_QUALIFIER, model, post_ids, error)
|
||||
except Exception as alert_exc:
|
||||
logger.warning("AI qualifier alert failed: {}", alert_exc)
|
||||
|
||||
async def apply_results(self, batch_id: int, results: list[dict], model: str, prompt: str, min_score: int) -> None:
|
||||
for item in results:
|
||||
score = int(item["score"])
|
||||
decision = str(item["decision"])
|
||||
model_decision = decision
|
||||
if score >= min_score and decision == "maybe":
|
||||
decision = "accepted"
|
||||
if score < min_score and decision == "accepted":
|
||||
decision = "rejected"
|
||||
status = "accepted" if score >= min_score and decision == "accepted" else "rejected"
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE raw_posts
|
||||
SET qualification_status=$2,
|
||||
qualification_score=$3,
|
||||
qualification_decision=$4,
|
||||
qualification_reason=$5,
|
||||
qualification_model=$6,
|
||||
qualification_prompt_hash=$7,
|
||||
qualification_batch_id=$8,
|
||||
qualification_reject_tag=$9,
|
||||
qualification_model_decision=$10,
|
||||
qualified_at=NOW(),
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
int(item["id"]),
|
||||
status,
|
||||
score,
|
||||
decision,
|
||||
item["reason"],
|
||||
model,
|
||||
prompt_hash(prompt),
|
||||
batch_id,
|
||||
item.get("reject_tag"),
|
||||
model_decision,
|
||||
)
|
||||
|
||||
async def call_ai(
|
||||
self,
|
||||
provider: str,
|
||||
model: str,
|
||||
api_key: str,
|
||||
api_base: str,
|
||||
prompt: str,
|
||||
payload: dict,
|
||||
temperature: float,
|
||||
timeout: int,
|
||||
) -> tuple[dict, dict[str, Any]]:
|
||||
messages = [
|
||||
{"role": "system", "content": prompt},
|
||||
{"role": "user", "content": json.dumps(payload, ensure_ascii=False)},
|
||||
]
|
||||
kwargs: dict[str, Any] = {
|
||||
"model": normalize_model(provider, model),
|
||||
"messages": messages,
|
||||
"temperature": temperature,
|
||||
"timeout": timeout,
|
||||
"response_format": {"type": "json_object"},
|
||||
}
|
||||
if api_key:
|
||||
kwargs["api_key"] = api_key
|
||||
if api_base:
|
||||
kwargs["api_base"] = api_base
|
||||
response = await asyncio.to_thread(litellm.completion, **kwargs)
|
||||
content = response.choices[0].message.content
|
||||
return parse_ai_json(content), response_usage(response)
|
||||
|
||||
async def run_once(self) -> bool:
|
||||
enabled = await is_worker_enabled(self.pool, WORKER_AI_QUALIFIER)
|
||||
setting_enabled = await fetch_bool_setting("ai_qualifier_enabled", False)
|
||||
if not enabled or not setting_enabled:
|
||||
await self.heartbeat.beat(self.pool, status="disabled", force=True)
|
||||
return False
|
||||
|
||||
provider = str(await fetch_setting("ai_qualifier_provider", "openrouter") or "openrouter")
|
||||
model = str(await fetch_setting("ai_qualifier_model", "") or "").strip()
|
||||
api_key = str(await fetch_setting("ai_qualifier_api_key", "") or "").strip()
|
||||
api_base = str(await fetch_setting("ai_qualifier_api_base", "") or "").strip()
|
||||
batch_size = max(1, await fetch_int_setting("ai_qualifier_batch_size", 30))
|
||||
min_score = max(1, min(10, await fetch_int_setting("ai_qualifier_min_score", 7)))
|
||||
max_text_chars = max(100, await fetch_int_setting("ai_qualifier_max_text_chars", 2000))
|
||||
temperature = max(0.0, await fetch_float_setting("ai_qualifier_temperature", 0.0))
|
||||
timeout = max(10, await fetch_int_setting("ai_qualifier_timeout_sec", 120))
|
||||
prompt = build_system_prompt(
|
||||
str(await fetch_setting("ai_qualifier_prompt", "") or ""),
|
||||
str(await fetch_setting("ai_qualifier_contract", DEFAULT_CONTRACT_PROMPT) or DEFAULT_CONTRACT_PROMPT),
|
||||
)
|
||||
|
||||
if not model or not api_key or not prompt:
|
||||
await self.heartbeat.beat(self.pool, status="not_configured", force=True)
|
||||
return False
|
||||
|
||||
posts = await self.claim_posts(batch_size)
|
||||
await self.heartbeat.beat(self.pool, meta={"claimed": len(posts), "batch_size": batch_size})
|
||||
if not posts:
|
||||
return False
|
||||
|
||||
payload = [
|
||||
{
|
||||
"id": int(p["id"]),
|
||||
"source": p.get("source_name") or "",
|
||||
"original_url": p.get("original_url") or "",
|
||||
"media_count": int(p.get("media_count") or 0),
|
||||
"media_types": list(p.get("media_types") or []),
|
||||
"text": str(p.get("raw_text") or "")[:max_text_chars],
|
||||
}
|
||||
for p in posts
|
||||
]
|
||||
post_ids = [int(p["id"]) for p in posts]
|
||||
batch_id = await self.create_batch(provider, model, prompt, payload)
|
||||
try:
|
||||
response, usage = await self.call_ai(provider, model, api_key, api_base, prompt, payload, temperature, timeout)
|
||||
results = validate_results(response, set(post_ids), min_score)
|
||||
await self.apply_results(batch_id, results, normalize_model(provider, model), prompt, min_score)
|
||||
await self.finish_batch(batch_id, "done", {"results": results}, usage=usage)
|
||||
logger.info("AI qualifier batch done: id={} posts={} usage={}", batch_id, len(results), usage)
|
||||
return True
|
||||
except Exception as exc:
|
||||
await self.finish_batch(batch_id, "failed", error=str(exc))
|
||||
retry_ids, failed_ids = await self.mark_posts_after_failed_batch(post_ids, str(exc))
|
||||
if failed_ids:
|
||||
await self.report_failed_batch(normalize_model(provider, model), failed_ids, str(exc))
|
||||
else:
|
||||
await self.heartbeat.beat(
|
||||
self.pool,
|
||||
status="retrying_after_error",
|
||||
meta={"error": str(exc)[:300], "post_ids": retry_ids, "model": normalize_model(provider, model)},
|
||||
force=True,
|
||||
)
|
||||
logger.exception("AI qualifier batch failed: id={} error={}", batch_id, exc)
|
||||
return True
|
||||
|
||||
async def run_loop(self) -> None:
|
||||
await self.init()
|
||||
worker_id = f"{WORKER_AI_QUALIFIER}"
|
||||
logger.info("{} started", worker_id)
|
||||
while True:
|
||||
try:
|
||||
had_work = await self.run_once()
|
||||
except Exception as exc:
|
||||
logger.exception("AI qualifier loop error: {}", exc)
|
||||
had_work = False
|
||||
interval = max(10, await fetch_int_setting("ai_qualifier_interval_sec", 60))
|
||||
if not had_work:
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
logger.remove()
|
||||
logger.add(lambda msg: print(msg, end=""), level=settings.log_level)
|
||||
worker = AIQualifierWorker()
|
||||
await worker.run_loop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,556 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
import litellm
|
||||
from loguru import logger
|
||||
|
||||
from ..config import settings
|
||||
from ..constants import WORKER_AI_WRITER
|
||||
from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool
|
||||
from ..heartbeat import HeartbeatReporter
|
||||
from ..jobs import is_worker_enabled
|
||||
from ..text_utils import build_publication_text, normalize_hash_tag, parse_categories
|
||||
from .ai_alerts import send_ai_worker_error_alert
|
||||
|
||||
|
||||
NON_TARGET_CATEGORY_ID = 18
|
||||
NON_TARGET_CATEGORY_TAGS = {"нцк", "nck"}
|
||||
EMOJI_LEADING_MARKER_RE = re.compile(
|
||||
r"^[ \t]*(?:🧵|🎯|🛡|🧤|📦|💵|⚙\ufe0f?|🎨|🎒|🔧|👕|💪)\s+"
|
||||
)
|
||||
|
||||
|
||||
def now_utc() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def prompt_hash(prompt: str) -> str:
|
||||
return hashlib.sha256((prompt or "").encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def normalize_prompt(prompt: str) -> str:
|
||||
return (prompt or "").replace("\\r\\n", "\n").replace("\\n", "\n").strip()
|
||||
|
||||
|
||||
def strip_leading_emoji_markers(text: str) -> str:
|
||||
lines = str(text or "").splitlines()
|
||||
cleaned: list[str] = []
|
||||
for line in lines:
|
||||
cleaned.append(EMOJI_LEADING_MARKER_RE.sub("", line))
|
||||
return "\n".join(cleaned).strip()
|
||||
|
||||
|
||||
DEFAULT_CONTRACT_PROMPT = """
|
||||
OUTPUT SCHEMA (return array matching input order):
|
||||
{"rewrites":[{"id":123,"category_id":2,"text":"готовый текст без хэштегов","notes":"короткая заметка для редактора или null"}]}
|
||||
|
||||
Rules:
|
||||
- Input is a JSON object with key "posts" containing accepted posts.
|
||||
- Input has key "categories" with objects: id, name, tag.
|
||||
- Each post includes producer_name and producer_tag. producer_name can be a manufacturer, shop, or publishing source.
|
||||
- The "text" field for every rewrite must explicitly mention producer_name at least once. Do not satisfy this only through producer_tag, hashtags, notes, or metadata.
|
||||
- Pick exactly one category from the categories list in input and return its numeric id as category_id.
|
||||
- If the post is non-target content, choose category_id 18 ("Не целевой контент", tag "НЦК"); the app will reject it automatically.
|
||||
- Do not add links or hashtags to rewritten text.
|
||||
- Return JSON only. No markdown. No text outside JSON.
|
||||
- Never return an empty object. Never omit the "rewrites" key.
|
||||
- Return one rewrite for every input post id.
|
||||
- The "text" field must be a ready-to-publish Russian post with normal paragraph line breaks encoded as JSON string newlines.
|
||||
- The "notes" field must be a short editor note in Russian or null.
|
||||
"""
|
||||
|
||||
|
||||
def build_system_prompt(user_prompt: str, contract_prompt: str) -> str:
|
||||
parts = [normalize_prompt(user_prompt), normalize_prompt(contract_prompt or DEFAULT_CONTRACT_PROMPT)]
|
||||
return "\n\n".join(part for part in parts if part).strip()
|
||||
|
||||
|
||||
def normalize_model(provider: str, model: str) -> str:
|
||||
provider = (provider or "").strip().lower()
|
||||
model = (model or "").strip()
|
||||
if not model:
|
||||
return model
|
||||
if "/" in model:
|
||||
return model
|
||||
if provider in {"openrouter", "anthropic", "gemini", "vertex_ai", "bedrock"}:
|
||||
return f"{provider}/{model}"
|
||||
return model
|
||||
|
||||
|
||||
def parse_ai_json(content: str) -> dict:
|
||||
raw = (content or "").strip()
|
||||
if raw.startswith("```"):
|
||||
raw = raw.strip("`")
|
||||
if raw.lower().startswith("json"):
|
||||
raw = raw[4:].strip()
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
def response_usage(response: Any) -> dict[str, Any]:
|
||||
usage = getattr(response, "usage", None)
|
||||
if usage is None and isinstance(response, dict):
|
||||
usage = response.get("usage")
|
||||
|
||||
def get(name: str) -> int | None:
|
||||
if usage is None:
|
||||
return None
|
||||
value = getattr(usage, name, None)
|
||||
if value is None and isinstance(usage, dict):
|
||||
value = usage.get(name)
|
||||
try:
|
||||
return int(value) if value is not None else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
cost = getattr(response, "_hidden_params", None)
|
||||
if isinstance(cost, dict):
|
||||
cost = cost.get("response_cost")
|
||||
else:
|
||||
cost = None
|
||||
try:
|
||||
cost_value = Decimal(str(cost)) if cost is not None else None
|
||||
except Exception:
|
||||
cost_value = None
|
||||
|
||||
return {
|
||||
"prompt_tokens": get("prompt_tokens"),
|
||||
"completion_tokens": get("completion_tokens"),
|
||||
"total_tokens": get("total_tokens"),
|
||||
"estimated_cost_usd": cost_value,
|
||||
}
|
||||
|
||||
|
||||
def validate_rewrites(
|
||||
data: dict,
|
||||
expected_ids: set[int],
|
||||
categories: list[dict[str, Any]],
|
||||
source_tags_by_id: dict[int, str],
|
||||
) -> list[dict]:
|
||||
rewrites = data.get("rewrites")
|
||||
if not isinstance(rewrites, list) and isinstance(data.get("data"), dict):
|
||||
rewrites = data["data"].get("rewrites")
|
||||
if isinstance(rewrites, str):
|
||||
try:
|
||||
rewrites = json.loads(rewrites)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
if not isinstance(rewrites, list):
|
||||
raise ValueError("AI response has no rewrites list")
|
||||
out: list[dict] = []
|
||||
seen: set[int] = set()
|
||||
for item in rewrites:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
post_id = int(item.get("id"))
|
||||
if post_id not in expected_ids:
|
||||
raise ValueError(f"AI returned unexpected id={post_id}")
|
||||
text = strip_leading_emoji_markers(str(item.get("text") or "").strip())
|
||||
by_id = {int(c["id"]): c for c in categories if c.get("id") is not None}
|
||||
by_name = {str(c["name"]).lower(): c for c in categories if c.get("name")}
|
||||
if item.get("category_id") is not None:
|
||||
category_id = int(item["category_id"])
|
||||
if category_id not in by_id:
|
||||
raise ValueError(f"AI returned unknown category_id={category_id!r} for id={post_id}")
|
||||
category_row = by_id[category_id]
|
||||
else:
|
||||
category_name = str(item.get("category") or "").strip().strip("#")
|
||||
category_row = by_name.get(category_name.lower())
|
||||
if not category_row:
|
||||
raise ValueError(f"AI returned unknown category={category_name!r} for id={post_id}")
|
||||
notes = str(item.get("notes") or "").strip()[:1000]
|
||||
seen.add(post_id)
|
||||
category_tag = normalize_hash_tag(category_row.get("tag") or category_row.get("name") or "", "category")
|
||||
reject_by_category = int(category_row["id"]) == NON_TARGET_CATEGORY_ID or category_tag.lower() in NON_TARGET_CATEGORY_TAGS
|
||||
if not reject_by_category and len(text) < 40:
|
||||
raise ValueError(f"AI returned too short rewrite for id={post_id}")
|
||||
out.append(
|
||||
{
|
||||
"id": post_id,
|
||||
"category_id": int(category_row["id"]),
|
||||
"category": str(category_row["name"]),
|
||||
"category_tag": category_tag,
|
||||
"source_tag": normalize_hash_tag(source_tags_by_id.get(post_id) or "", "source"),
|
||||
"text": text,
|
||||
"notes": notes,
|
||||
"reject_by_category": reject_by_category,
|
||||
}
|
||||
)
|
||||
missing = expected_ids - seen
|
||||
if missing:
|
||||
raise ValueError(f"AI response missing ids: {sorted(missing)[:10]}")
|
||||
return out
|
||||
|
||||
|
||||
async def load_writer_categories(pool) -> list[dict[str, Any]]:
|
||||
try:
|
||||
rows = await pool.fetch(
|
||||
"""
|
||||
SELECT sort_order AS id, name, tag
|
||||
FROM content_categories
|
||||
WHERE is_active=TRUE
|
||||
ORDER BY sort_order, name
|
||||
"""
|
||||
)
|
||||
except Exception:
|
||||
rows = []
|
||||
categories = [{"id": int(row["id"]), "name": str(row["name"]), "tag": str(row["tag"])} for row in rows]
|
||||
if categories:
|
||||
return categories
|
||||
legacy = parse_categories(await fetch_setting("ai_writer_categories", []))
|
||||
names = legacy or [
|
||||
"защита",
|
||||
"одежда",
|
||||
"разгрузка",
|
||||
"рюкзаки",
|
||||
"airsoft",
|
||||
"патчи",
|
||||
"электроника",
|
||||
"аксессуары",
|
||||
"производство",
|
||||
]
|
||||
return [{"id": idx + 1, "name": name, "tag": normalize_hash_tag(name, "category")} for idx, name in enumerate(names)]
|
||||
|
||||
|
||||
class AIWriterWorker:
|
||||
def __init__(self) -> None:
|
||||
self.pool = None
|
||||
self.heartbeat = HeartbeatReporter(WORKER_AI_WRITER, 30)
|
||||
|
||||
async def init(self) -> None:
|
||||
self.pool = await get_pool()
|
||||
|
||||
async def claim_posts(self, batch_size: int) -> list[dict]:
|
||||
async with self.pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
count = await conn.fetchval(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM raw_posts
|
||||
WHERE status='storage_ready'
|
||||
AND qualification_status='accepted'
|
||||
AND COALESCE(rewrite_status, 'pending') = 'pending'
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media m
|
||||
WHERE m.raw_post_id=raw_posts.id
|
||||
AND COALESCE(m.editor_hidden, FALSE)=FALSE
|
||||
AND m.media_type IN ('photo', 'video')
|
||||
AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') <> ''
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media m
|
||||
WHERE m.raw_post_id=raw_posts.id
|
||||
AND COALESCE(m.editor_hidden, FALSE)=FALSE
|
||||
AND m.media_type IN ('photo', 'video')
|
||||
AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = ''
|
||||
)
|
||||
"""
|
||||
)
|
||||
if int(count or 0) < batch_size:
|
||||
return []
|
||||
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
WITH cte AS (
|
||||
SELECT id
|
||||
FROM raw_posts
|
||||
WHERE status='storage_ready'
|
||||
AND qualification_status='accepted'
|
||||
AND COALESCE(rewrite_status, 'pending') = 'pending'
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media m
|
||||
WHERE m.raw_post_id=raw_posts.id
|
||||
AND COALESCE(m.editor_hidden, FALSE)=FALSE
|
||||
AND m.media_type IN ('photo', 'video')
|
||||
AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') <> ''
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media m
|
||||
WHERE m.raw_post_id=raw_posts.id
|
||||
AND COALESCE(m.editor_hidden, FALSE)=FALSE
|
||||
AND m.media_type IN ('photo', 'video')
|
||||
AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = ''
|
||||
)
|
||||
ORDER BY qualified_at ASC NULLS LAST, id ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT $1
|
||||
)
|
||||
UPDATE raw_posts rp
|
||||
SET rewrite_status='processing',
|
||||
updated_at=NOW()
|
||||
FROM cte
|
||||
WHERE rp.id=cte.id
|
||||
RETURNING rp.id, rp.raw_text, rp.original_url, rp.created_at, rp.qualification_score,
|
||||
rp.qualification_reason,
|
||||
(SELECT s.name FROM sources s WHERE s.id=rp.source_id) AS source_name,
|
||||
(SELECT s.tag FROM sources s WHERE s.id=rp.source_id) AS source_tag,
|
||||
(SELECT COUNT(*) FROM raw_post_media m WHERE m.raw_post_id=rp.id) AS media_count,
|
||||
(SELECT ARRAY_AGG(DISTINCT m.media_type ORDER BY m.media_type)
|
||||
FROM raw_post_media m WHERE m.raw_post_id=rp.id) AS media_types
|
||||
""",
|
||||
batch_size,
|
||||
)
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
async def create_batch(self, provider: str, model: str, prompt: str, payload: dict) -> int:
|
||||
return int(
|
||||
await self.pool.fetchval(
|
||||
"""
|
||||
INSERT INTO ai_writer_batches(provider, model, prompt_hash, prompt_text, posts_count, request_json)
|
||||
VALUES($1, $2, $3, $4, $5, $6::jsonb)
|
||||
RETURNING id
|
||||
""",
|
||||
provider,
|
||||
model,
|
||||
prompt_hash(prompt),
|
||||
prompt,
|
||||
len(payload["posts"]),
|
||||
json.dumps(payload, ensure_ascii=False),
|
||||
)
|
||||
)
|
||||
|
||||
async def finish_batch(
|
||||
self,
|
||||
batch_id: int,
|
||||
status: str,
|
||||
response: Any = None,
|
||||
error: str | None = None,
|
||||
usage: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
ready_count = 0
|
||||
if isinstance(response, dict) and isinstance(response.get("rewrites"), list):
|
||||
ready_count = len(response["rewrites"])
|
||||
usage = usage or {}
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE ai_writer_batches
|
||||
SET status=$2,
|
||||
response_json=$3::jsonb,
|
||||
error=$4,
|
||||
ready_count=$5,
|
||||
failed_count=$6,
|
||||
prompt_tokens=$7,
|
||||
completion_tokens=$8,
|
||||
total_tokens=$9,
|
||||
estimated_cost_usd=$10,
|
||||
completed_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
batch_id,
|
||||
status,
|
||||
json.dumps(response, ensure_ascii=False) if response is not None else None,
|
||||
error[:2000] if error else None,
|
||||
ready_count if status == "done" else 0,
|
||||
0 if status == "done" else 1,
|
||||
usage.get("prompt_tokens"),
|
||||
usage.get("completion_tokens"),
|
||||
usage.get("total_tokens"),
|
||||
usage.get("estimated_cost_usd"),
|
||||
)
|
||||
|
||||
async def mark_posts_failed(self, post_ids: list[int], error: str) -> None:
|
||||
if not post_ids:
|
||||
return
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE raw_posts
|
||||
SET rewrite_status='failed',
|
||||
rewrite_notes=$2,
|
||||
updated_at=NOW()
|
||||
WHERE id=ANY($1::bigint[])
|
||||
""",
|
||||
post_ids,
|
||||
error[:1000],
|
||||
)
|
||||
|
||||
async def report_failed_batch(self, model: str, post_ids: list[int], error: str) -> None:
|
||||
meta = {"error": error[:300], "post_ids": post_ids, "model": model}
|
||||
await self.heartbeat.beat(self.pool, status="error", meta=meta, force=True)
|
||||
try:
|
||||
await send_ai_worker_error_alert(WORKER_AI_WRITER, model, post_ids, error)
|
||||
except Exception as alert_exc:
|
||||
logger.warning("AI writer alert failed: {}", alert_exc)
|
||||
|
||||
async def apply_rewrites(self, batch_id: int, rewrites: list[dict], model: str, prompt: str) -> None:
|
||||
for item in rewrites:
|
||||
final_text = build_publication_text(item["text"], item["category_tag"], item["source_tag"])
|
||||
editorial_status = "rejected" if item.get("reject_by_category") else "review"
|
||||
editor_notes = "Отклонено AI: Не целевой контент" if item.get("reject_by_category") else None
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE raw_posts
|
||||
SET rewrite_status='ready',
|
||||
rewritten_text=$2,
|
||||
rewrite_notes=$3,
|
||||
rewrite_model=$4,
|
||||
rewrite_prompt_hash=$5,
|
||||
rewrite_batch_id=$6,
|
||||
rewrite_category=$7,
|
||||
rewrite_category_tag=$8,
|
||||
rewrite_source_tag=$9,
|
||||
rewrite_category_id=$10,
|
||||
editorial_status=$11,
|
||||
final_text=$12,
|
||||
final_category=$7,
|
||||
final_category_tag=$8,
|
||||
final_source_tag=$9,
|
||||
final_category_id=$10,
|
||||
editor_notes=$13,
|
||||
qualification_status=CASE WHEN $11='rejected' THEN 'rejected' ELSE qualification_status END,
|
||||
qualification_decision=CASE WHEN $11='rejected' THEN 'rejected' ELSE qualification_decision END,
|
||||
qualification_model_decision=CASE WHEN $11='rejected' THEN 'rejected' ELSE qualification_model_decision END,
|
||||
qualification_reason=CASE WHEN $11='rejected' THEN 'AI writer selected non-target category' ELSE qualification_reason END,
|
||||
qualification_reject_tag=CASE WHEN $11='rejected' THEN 'non_target_content' ELSE qualification_reject_tag END,
|
||||
reviewed_at=CASE WHEN $11='rejected' THEN NOW() ELSE reviewed_at END,
|
||||
rewritten_at=NOW(),
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
int(item["id"]),
|
||||
item["text"],
|
||||
item["notes"],
|
||||
model,
|
||||
prompt_hash(prompt),
|
||||
batch_id,
|
||||
item["category"],
|
||||
item["category_tag"],
|
||||
item["source_tag"],
|
||||
item["category_id"],
|
||||
editorial_status,
|
||||
final_text,
|
||||
editor_notes,
|
||||
)
|
||||
|
||||
async def call_ai(
|
||||
self,
|
||||
provider: str,
|
||||
model: str,
|
||||
api_key: str,
|
||||
api_base: str,
|
||||
prompt: str,
|
||||
payload: dict,
|
||||
temperature: float,
|
||||
timeout: int,
|
||||
) -> tuple[dict, dict[str, Any]]:
|
||||
messages = [
|
||||
{"role": "system", "content": prompt},
|
||||
{"role": "user", "content": json.dumps(payload, ensure_ascii=False)},
|
||||
]
|
||||
kwargs: dict[str, Any] = {
|
||||
"model": normalize_model(provider, model),
|
||||
"messages": messages,
|
||||
"temperature": temperature,
|
||||
"timeout": timeout,
|
||||
"response_format": {"type": "json_object"},
|
||||
}
|
||||
if api_key:
|
||||
kwargs["api_key"] = api_key
|
||||
if api_base:
|
||||
kwargs["api_base"] = api_base
|
||||
response = await asyncio.to_thread(litellm.completion, **kwargs)
|
||||
content = response.choices[0].message.content
|
||||
return parse_ai_json(content), response_usage(response)
|
||||
|
||||
async def run_once(self) -> bool:
|
||||
enabled = await is_worker_enabled(self.pool, WORKER_AI_WRITER)
|
||||
setting_enabled = await fetch_bool_setting("ai_writer_enabled", False)
|
||||
if not enabled or not setting_enabled:
|
||||
await self.heartbeat.beat(self.pool, status="disabled", force=True)
|
||||
return False
|
||||
|
||||
provider = str(await fetch_setting("ai_writer_provider", "anthropic") or "anthropic")
|
||||
model = str(await fetch_setting("ai_writer_model", "") or "").strip()
|
||||
api_key = str(await fetch_setting("ai_writer_api_key", "") or "").strip()
|
||||
api_base = str(await fetch_setting("ai_writer_api_base", "") or "").strip()
|
||||
batch_size = max(1, await fetch_int_setting("ai_writer_batch_size", 1))
|
||||
max_text_chars = max(200, await fetch_int_setting("ai_writer_max_text_chars", 3500))
|
||||
temperature = max(0.0, await fetch_float_setting("ai_writer_temperature", 0.4))
|
||||
timeout = max(10, await fetch_int_setting("ai_writer_timeout_sec", 180))
|
||||
prompt = build_system_prompt(
|
||||
str(await fetch_setting("ai_writer_prompt", "") or ""),
|
||||
str(await fetch_setting("ai_writer_contract", DEFAULT_CONTRACT_PROMPT) or DEFAULT_CONTRACT_PROMPT),
|
||||
)
|
||||
categories = await load_writer_categories(self.pool)
|
||||
|
||||
if not model or not api_key or not prompt or not categories:
|
||||
await self.heartbeat.beat(self.pool, status="not_configured", force=True)
|
||||
return False
|
||||
|
||||
posts = await self.claim_posts(batch_size)
|
||||
await self.heartbeat.beat(self.pool, meta={"claimed": len(posts), "batch_size": batch_size})
|
||||
if not posts:
|
||||
return False
|
||||
|
||||
payload = {
|
||||
"task": "rewrite_accepted_posts",
|
||||
"categories": categories,
|
||||
"posts": [
|
||||
{
|
||||
"id": int(p["id"]),
|
||||
"producer_name": p.get("source_name") or "",
|
||||
"producer_tag": normalize_hash_tag(p.get("source_tag") or p.get("source_name") or "", "source"),
|
||||
"original_url": p.get("original_url") or "",
|
||||
"qualification_score": p.get("qualification_score"),
|
||||
"qualification_reason": p.get("qualification_reason") or "",
|
||||
"media_count": int(p.get("media_count") or 0),
|
||||
"media_types": list(p.get("media_types") or []),
|
||||
"text": str(p.get("raw_text") or "")[:max_text_chars],
|
||||
}
|
||||
for p in posts
|
||||
],
|
||||
}
|
||||
post_ids = [int(p["id"]) for p in posts]
|
||||
source_tags_by_id = {
|
||||
int(p["id"]): normalize_hash_tag(p.get("source_tag") or p.get("source_name") or "", "source")
|
||||
for p in posts
|
||||
}
|
||||
batch_id = await self.create_batch(provider, model, prompt, payload)
|
||||
response = None
|
||||
usage = None
|
||||
try:
|
||||
response, usage = await self.call_ai(provider, model, api_key, api_base, prompt, payload, temperature, timeout)
|
||||
rewrites = validate_rewrites(response, set(post_ids), categories, source_tags_by_id)
|
||||
await self.apply_rewrites(batch_id, rewrites, normalize_model(provider, model), prompt)
|
||||
await self.finish_batch(batch_id, "done", {"rewrites": rewrites}, usage=usage)
|
||||
logger.info("AI writer batch done: id={} posts={} usage={}", batch_id, len(rewrites), usage)
|
||||
return True
|
||||
except Exception as exc:
|
||||
await self.mark_posts_failed(post_ids, str(exc))
|
||||
await self.finish_batch(batch_id, "failed", response=response, error=str(exc), usage=usage)
|
||||
await self.report_failed_batch(normalize_model(provider, model), post_ids, str(exc))
|
||||
logger.exception("AI writer batch failed: id={} error={}", batch_id, exc)
|
||||
return True
|
||||
|
||||
async def run_loop(self) -> None:
|
||||
await self.init()
|
||||
logger.info("{} started", WORKER_AI_WRITER)
|
||||
while True:
|
||||
try:
|
||||
had_work = await self.run_once()
|
||||
except Exception as exc:
|
||||
logger.exception("AI writer loop error: {}", exc)
|
||||
had_work = False
|
||||
interval = max(10, await fetch_int_setting("ai_writer_interval_sec", 60))
|
||||
if not had_work:
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
logger.remove()
|
||||
logger.add(lambda msg: print(msg, end=""), level=settings.log_level)
|
||||
worker = AIWriterWorker()
|
||||
await worker.run_loop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,457 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import date, datetime, time, timedelta, timezone
|
||||
from typing import Any
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from aiogram import Bot
|
||||
from aiogram.exceptions import TelegramRetryAfter
|
||||
from loguru import logger
|
||||
|
||||
from ..config import settings
|
||||
from ..constants import WORKER_DAILY_REPORT
|
||||
from ..db import fetch_bool_setting, fetch_int_setting, fetch_setting, get_pool
|
||||
from ..heartbeat import HeartbeatReporter
|
||||
from ..jobs import is_worker_enabled
|
||||
|
||||
LOCAL_TZ = ZoneInfo("Asia/Yekaterinburg")
|
||||
MESSAGE_LIMIT = 4096
|
||||
|
||||
|
||||
def valid_time(value: str) -> bool:
|
||||
return bool(re.fullmatch(r"(?:[01]\d|2[0-3]):[0-5]\d", str(value or "").strip()))
|
||||
|
||||
|
||||
def parse_recipients(value: Any) -> list[int]:
|
||||
raw_items: list[Any]
|
||||
if isinstance(value, list):
|
||||
raw_items = value
|
||||
elif isinstance(value, str):
|
||||
raw = value.strip()
|
||||
if not raw:
|
||||
return []
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
raw_items = parsed if isinstance(parsed, list) else [parsed]
|
||||
except json.JSONDecodeError:
|
||||
raw_items = re.split(r"[\s,;]+", raw)
|
||||
else:
|
||||
raw_items = [value]
|
||||
out: list[int] = []
|
||||
for item in raw_items:
|
||||
try:
|
||||
recipient_id = int(str(item).strip())
|
||||
except Exception:
|
||||
continue
|
||||
if recipient_id not in out:
|
||||
out.append(recipient_id)
|
||||
return out
|
||||
|
||||
|
||||
def day_bounds(report_date: date) -> tuple[datetime, datetime]:
|
||||
start_local = datetime.combine(report_date, time.min, tzinfo=LOCAL_TZ)
|
||||
end_local = start_local + timedelta(days=1)
|
||||
return start_local.astimezone(timezone.utc), end_local.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def fmt_int(value: Any) -> str:
|
||||
try:
|
||||
return f"{int(value):,}".replace(",", " ")
|
||||
except Exception:
|
||||
return "0"
|
||||
|
||||
|
||||
def split_chunks(text: str, limit: int = MESSAGE_LIMIT) -> list[str]:
|
||||
text = str(text or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
chunks: list[str] = []
|
||||
while len(text) > limit:
|
||||
split_at = text.rfind("\n", 0, limit)
|
||||
if split_at < limit // 2:
|
||||
split_at = limit
|
||||
chunks.append(text[:split_at].strip())
|
||||
text = text[split_at:].strip()
|
||||
if text:
|
||||
chunks.append(text)
|
||||
return chunks
|
||||
|
||||
|
||||
class DailyReportWorker:
|
||||
def __init__(self) -> None:
|
||||
self.pool = None
|
||||
self.bot: Bot | None = None
|
||||
self.bot_token = ""
|
||||
self.recipients: list[int] = []
|
||||
self.report_time = "00:04"
|
||||
self.interval_sec = 60
|
||||
self.heartbeat = HeartbeatReporter(WORKER_DAILY_REPORT, 30)
|
||||
|
||||
async def init(self) -> None:
|
||||
self.pool = await get_pool()
|
||||
await self.reload_settings()
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.bot:
|
||||
await self.bot.session.close()
|
||||
|
||||
async def reload_settings(self) -> None:
|
||||
token = (
|
||||
str(await fetch_setting("daily_report_bot_token", "") or "").strip()
|
||||
or str(await fetch_setting("tg_poster_bot_token", "") or "").strip()
|
||||
or settings.tg_bot_token
|
||||
)
|
||||
if not token:
|
||||
raise RuntimeError("daily_report_bot_token, tg_poster_bot_token and TG_BOT_TOKEN are empty")
|
||||
if token != self.bot_token:
|
||||
if self.bot:
|
||||
await self.bot.session.close()
|
||||
self.bot = Bot(token=token)
|
||||
self.bot_token = token
|
||||
self.recipients = parse_recipients(await fetch_setting("daily_report_recipient_ids", [442509142]))
|
||||
if not self.recipients:
|
||||
raise RuntimeError("daily_report_recipient_ids is empty")
|
||||
report_time = str(await fetch_setting("daily_report_time", "00:04") or "00:04").strip()
|
||||
self.report_time = report_time if valid_time(report_time) else "00:04"
|
||||
self.interval_sec = max(10, await fetch_int_setting("daily_report_interval_sec", 60))
|
||||
logger.info("Daily report config: time={} recipients={}", self.report_time, self.recipients)
|
||||
|
||||
async def save_last_sent_date(self, report_date: date) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE app_settings
|
||||
SET value_json=$2::jsonb,
|
||||
updated_at=NOW()
|
||||
WHERE key=$1
|
||||
""",
|
||||
"daily_report_last_sent_date",
|
||||
json.dumps(report_date.isoformat()),
|
||||
)
|
||||
|
||||
async def countval(self, sql: str, *args: Any) -> int:
|
||||
return int(await self.pool.fetchval(sql, *args) or 0)
|
||||
|
||||
async def collect(self, report_date: date) -> dict[str, Any]:
|
||||
start_utc, end_utc = day_bounds(report_date)
|
||||
now_utc = datetime.now(timezone.utc)
|
||||
async with self.pool.acquire() as conn:
|
||||
raw_collected = await conn.fetchval(
|
||||
"SELECT COUNT(*) FROM raw_posts WHERE created_at >= $1 AND created_at < $2",
|
||||
start_utc,
|
||||
end_utc,
|
||||
)
|
||||
storage_ready = await conn.fetchval(
|
||||
"SELECT COUNT(*) FROM raw_posts WHERE status='storage_ready' AND updated_at >= $1 AND updated_at < $2",
|
||||
start_utc,
|
||||
end_utc,
|
||||
)
|
||||
media_failed = await conn.fetchval(
|
||||
"SELECT COUNT(*) FROM raw_post_media WHERE status IN ('failed','link_only') AND updated_at >= $1 AND updated_at < $2",
|
||||
start_utc,
|
||||
end_utc,
|
||||
)
|
||||
qualification_rows = await conn.fetch(
|
||||
"""
|
||||
SELECT COALESCE(qualification_status, 'pending') AS status, COUNT(*) AS count
|
||||
FROM raw_posts
|
||||
WHERE qualified_at >= $1 AND qualified_at < $2
|
||||
GROUP BY COALESCE(qualification_status, 'pending')
|
||||
""",
|
||||
start_utc,
|
||||
end_utc,
|
||||
)
|
||||
writer_ready = await conn.fetchval(
|
||||
"SELECT COUNT(*) FROM raw_posts WHERE rewrite_status='ready' AND rewritten_at >= $1 AND rewritten_at < $2",
|
||||
start_utc,
|
||||
end_utc,
|
||||
)
|
||||
review_now = await conn.fetchval(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM raw_posts
|
||||
WHERE rewrite_status='ready'
|
||||
AND COALESCE(editorial_status, 'review')='review'
|
||||
"""
|
||||
)
|
||||
accepted_today = await conn.fetchval(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM raw_posts
|
||||
WHERE reviewed_at >= $1 AND reviewed_at < $2
|
||||
AND editorial_status='accepted'
|
||||
""",
|
||||
start_utc,
|
||||
end_utc,
|
||||
)
|
||||
rejected_today = await conn.fetchval(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM raw_posts
|
||||
WHERE reviewed_at >= $1 AND reviewed_at < $2
|
||||
AND editorial_status='rejected'
|
||||
""",
|
||||
start_utc,
|
||||
end_utc,
|
||||
)
|
||||
publication_rows = await conn.fetch(
|
||||
"""
|
||||
SELECT platform, status, COUNT(*) AS count
|
||||
FROM post_publications
|
||||
WHERE updated_at >= $1 AND updated_at < $2
|
||||
GROUP BY platform, status
|
||||
ORDER BY platform, status
|
||||
""",
|
||||
start_utc,
|
||||
end_utc,
|
||||
)
|
||||
tg_queue = await conn.fetchval(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM raw_posts rp
|
||||
WHERE rp.status='storage_ready'
|
||||
AND rp.rewrite_status='ready'
|
||||
AND rp.editorial_status='accepted'
|
||||
AND COALESCE(rp.publication_status, 'pending')='pending'
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM raw_post_media m
|
||||
WHERE m.raw_post_id=rp.id
|
||||
AND COALESCE(m.editor_hidden,FALSE)=FALSE
|
||||
AND m.media_type IN ('photo','video')
|
||||
AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') <> ''
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM raw_post_media m
|
||||
WHERE m.raw_post_id=rp.id
|
||||
AND COALESCE(m.editor_hidden,FALSE)=FALSE
|
||||
AND m.media_type IN ('photo','video')
|
||||
AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = ''
|
||||
)
|
||||
"""
|
||||
)
|
||||
vk_queue = await conn.fetchval(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM raw_posts rp
|
||||
LEFT JOIN post_publications pp ON pp.raw_post_id=rp.id AND pp.platform='vk'
|
||||
WHERE rp.status='storage_ready'
|
||||
AND rp.rewrite_status='ready'
|
||||
AND COALESCE(rp.editorial_status, 'review') IN ('accepted', 'published', 'publish_failed')
|
||||
AND COALESCE(pp.status, 'pending') IN ('pending', 'publish_failed')
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM raw_post_media rpm
|
||||
WHERE rpm.raw_post_id=rp.id
|
||||
AND COALESCE(rpm.editor_hidden, FALSE)=FALSE
|
||||
AND COALESCE(rpm.editor_added, FALSE)=FALSE
|
||||
AND (
|
||||
(rpm.media_type='photo' AND rpm.original_attachment_id LIKE 'photo%')
|
||||
OR (rpm.media_type='video' AND rpm.original_attachment_id LIKE 'video%')
|
||||
)
|
||||
)
|
||||
"""
|
||||
)
|
||||
publish_failed_now = await conn.fetchval(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM raw_posts
|
||||
WHERE rewrite_status='ready'
|
||||
AND COALESCE(editorial_status, '')='publish_failed'
|
||||
"""
|
||||
)
|
||||
workers = await conn.fetch(
|
||||
"""
|
||||
SELECT wc.name, wc.enabled, wh.heartbeat_at, wh.status
|
||||
FROM worker_controls wc
|
||||
LEFT JOIN worker_heartbeats wh ON wh.name=wc.name
|
||||
ORDER BY wc.name
|
||||
"""
|
||||
)
|
||||
top_errors = await conn.fetch(
|
||||
"""
|
||||
SELECT platform, error, COUNT(*) AS count
|
||||
FROM post_publications
|
||||
WHERE status='publish_failed'
|
||||
AND updated_at >= $1 AND updated_at < $2
|
||||
AND COALESCE(error, '') <> ''
|
||||
GROUP BY platform, error
|
||||
ORDER BY COUNT(*) DESC
|
||||
LIMIT 5
|
||||
""",
|
||||
start_utc,
|
||||
end_utc,
|
||||
)
|
||||
return {
|
||||
"date": report_date,
|
||||
"raw_collected": raw_collected,
|
||||
"storage_ready": storage_ready,
|
||||
"media_failed": media_failed,
|
||||
"qualification": {row["status"]: int(row["count"]) for row in qualification_rows},
|
||||
"writer_ready": writer_ready,
|
||||
"review_now": review_now,
|
||||
"accepted_today": accepted_today,
|
||||
"rejected_today": rejected_today,
|
||||
"publications": {(row["platform"], row["status"]): int(row["count"]) for row in publication_rows},
|
||||
"tg_queue": tg_queue,
|
||||
"vk_queue": vk_queue,
|
||||
"publish_failed_now": publish_failed_now,
|
||||
"workers": [dict(row) for row in workers],
|
||||
"top_errors": [dict(row) for row in top_errors],
|
||||
"now_utc": now_utc,
|
||||
}
|
||||
|
||||
def render(self, data: dict[str, Any]) -> str:
|
||||
q = data["qualification"]
|
||||
pubs = data["publications"]
|
||||
workers_lines: list[str] = []
|
||||
for row in data["workers"]:
|
||||
heartbeat_at = row.get("heartbeat_at")
|
||||
is_stale = True
|
||||
if heartbeat_at:
|
||||
is_stale = (data["now_utc"] - heartbeat_at).total_seconds() > 600
|
||||
marker = "OK" if row.get("enabled") and heartbeat_at and not is_stale else ("OFF" if not row.get("enabled") else "STALE")
|
||||
workers_lines.append(f"{marker} {row['name']}: {row.get('status') or '-'}")
|
||||
errors_lines = [
|
||||
f"- {row['platform']}: {row['count']} x {str(row['error'])[:120]}"
|
||||
for row in data["top_errors"]
|
||||
]
|
||||
if not errors_lines:
|
||||
errors_lines = ["- нет"]
|
||||
report_date = data["date"].strftime("%d.%m.%Y")
|
||||
return "\n".join(
|
||||
[
|
||||
f"Ежедневный отчет {settings.display_site_title} за {report_date}",
|
||||
"",
|
||||
"Сбор:",
|
||||
f"raw собрано: {fmt_int(data['raw_collected'])}",
|
||||
f"storage ready за день: {fmt_int(data['storage_ready'])}",
|
||||
f"media failed/link_only: {fmt_int(data['media_failed'])}",
|
||||
"",
|
||||
"AI:",
|
||||
f"принято категоризатором: {fmt_int(q.get('accepted', 0))}",
|
||||
f"отклонено категоризатором: {fmt_int(q.get('rejected', 0))}",
|
||||
f"ошибки категоризатора: {fmt_int(q.get('failed', 0))}",
|
||||
f"рерайтов готово: {fmt_int(data['writer_ready'])}",
|
||||
"",
|
||||
"Редакторская:",
|
||||
f"принято редактором за день: {fmt_int(data['accepted_today'])}",
|
||||
f"отклонено редактором за день: {fmt_int(data['rejected_today'])}",
|
||||
f"на проверке сейчас: {fmt_int(data['review_now'])}",
|
||||
f"ошибка публикации сейчас: {fmt_int(data['publish_failed_now'])}",
|
||||
"",
|
||||
"Публикации за день:",
|
||||
f"TG опубликовано: {fmt_int(pubs.get(('tg', 'published'), 0))}",
|
||||
f"VK опубликовано: {fmt_int(pubs.get(('vk', 'published'), 0))}",
|
||||
f"TG ошибки: {fmt_int(pubs.get(('tg', 'publish_failed'), 0))}",
|
||||
f"VK ошибки: {fmt_int(pubs.get(('vk', 'publish_failed'), 0))}",
|
||||
"",
|
||||
"Очередь сейчас:",
|
||||
f"TG кандидатов: {fmt_int(data['tg_queue'])}",
|
||||
f"VK кандидатов: {fmt_int(data['vk_queue'])}",
|
||||
"",
|
||||
"Ошибки дня:",
|
||||
*errors_lines,
|
||||
"",
|
||||
"Воркеры:",
|
||||
*workers_lines,
|
||||
]
|
||||
)
|
||||
|
||||
async def send_text(self, text: str) -> None:
|
||||
if not self.bot:
|
||||
raise RuntimeError("bot is not initialized")
|
||||
for recipient_id in self.recipients:
|
||||
for chunk in split_chunks(text):
|
||||
while True:
|
||||
try:
|
||||
await self.bot.send_message(recipient_id, chunk, disable_web_page_preview=True)
|
||||
break
|
||||
except TelegramRetryAfter as exc:
|
||||
await asyncio.sleep(float(exc.retry_after) + 1)
|
||||
|
||||
async def send_report(self, report_date: date, save_last_sent: bool = False) -> str:
|
||||
await self.reload_settings()
|
||||
data = await self.collect(report_date)
|
||||
text = self.render(data)
|
||||
await self.send_text(text)
|
||||
if save_last_sent:
|
||||
await self.save_last_sent_date(report_date)
|
||||
await self.heartbeat.beat(
|
||||
self.pool,
|
||||
status="sent",
|
||||
meta={"date": report_date.isoformat(), "recipients": self.recipients},
|
||||
force=True,
|
||||
)
|
||||
return text
|
||||
|
||||
async def due_report_date(self) -> date | None:
|
||||
now = datetime.now(LOCAL_TZ)
|
||||
hour, minute = [int(part) for part in self.report_time.split(":", 1)]
|
||||
slot_start = datetime.combine(now.date(), time(hour=hour, minute=minute), tzinfo=LOCAL_TZ)
|
||||
slot_end = slot_start + timedelta(minutes=10)
|
||||
if not (slot_start <= now < slot_end):
|
||||
return None
|
||||
report_date = now.date() - timedelta(days=1)
|
||||
last_sent = str(await fetch_setting("daily_report_last_sent_date", "") or "").strip()
|
||||
if last_sent == report_date.isoformat():
|
||||
return None
|
||||
return report_date
|
||||
|
||||
async def run_once(self) -> bool:
|
||||
await self.reload_settings()
|
||||
enabled = await is_worker_enabled(self.pool, WORKER_DAILY_REPORT)
|
||||
app_enabled = await fetch_bool_setting("daily_report_enabled", True)
|
||||
if not enabled or not app_enabled:
|
||||
await self.heartbeat.beat(self.pool, status="disabled", force=True)
|
||||
return False
|
||||
report_date = await self.due_report_date()
|
||||
if not report_date:
|
||||
await self.heartbeat.beat(self.pool, status="idle")
|
||||
return False
|
||||
await self.send_report(report_date, save_last_sent=True)
|
||||
return True
|
||||
|
||||
async def run_loop(self) -> None:
|
||||
await self.init()
|
||||
logger.info("{} started pid={}", WORKER_DAILY_REPORT, os.getpid())
|
||||
while True:
|
||||
try:
|
||||
await self.run_once()
|
||||
except Exception as exc:
|
||||
logger.exception("Daily report loop error: {}", exc)
|
||||
if self.pool:
|
||||
await self.heartbeat.beat(self.pool, status="error", meta={"error": str(exc)}, force=True)
|
||||
await asyncio.sleep(self.interval_sec)
|
||||
|
||||
|
||||
async def run_send_now(report_date: date | None = None) -> str:
|
||||
worker = DailyReportWorker()
|
||||
await worker.init()
|
||||
try:
|
||||
return await worker.send_report(report_date or datetime.now(LOCAL_TZ).date(), save_last_sent=False)
|
||||
finally:
|
||||
await worker.close()
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--send-now", action="store_true")
|
||||
parser.add_argument("--date", help="Report date YYYY-MM-DD, default today in EKB for --send-now")
|
||||
args = parser.parse_args()
|
||||
logger.remove()
|
||||
logger.add(sys.stdout, level=settings.log_level)
|
||||
if args.send_now:
|
||||
report_date = date.fromisoformat(args.date) if args.date else None
|
||||
text = await run_send_now(report_date)
|
||||
print(text)
|
||||
return
|
||||
worker = DailyReportWorker()
|
||||
await worker.run_loop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,728 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from datetime import datetime, time, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import aiohttp
|
||||
from loguru import logger
|
||||
|
||||
from ..config import settings
|
||||
from ..constants import (
|
||||
PUBLICATION_STATUS_FAILED,
|
||||
PUBLICATION_STATUS_PENDING,
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
WORKER_MAX_POSTER,
|
||||
)
|
||||
from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool
|
||||
from ..heartbeat import HeartbeatReporter
|
||||
from ..jobs import is_worker_enabled
|
||||
from ..text_utils import build_publication_text
|
||||
|
||||
LOCAL_TZ = ZoneInfo("Asia/Yekaterinburg")
|
||||
MAX_MESSAGE_LIMIT = 4000
|
||||
MAX_MEDIA_ITEMS = 10
|
||||
MAX_VIDEO_BYTES = 250 * 1024 * 1024
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
UPLOAD_ROOT = PROJECT_ROOT / "uploads"
|
||||
|
||||
|
||||
class MAXAPIError(RuntimeError):
|
||||
def __init__(self, message: str, status: int | None = None, code: str = "") -> None:
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
|
||||
|
||||
def valid_time(value: str) -> bool:
|
||||
return bool(re.fullmatch(r"(?:[01]\d|2[0-3]):[0-5]\d", str(value or "").strip()))
|
||||
|
||||
|
||||
def normalize_schedule(value: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(value, list):
|
||||
value = []
|
||||
rows: list[dict[str, Any]] = []
|
||||
for idx, item in enumerate(value):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
time_value = str(item.get("time") or "").strip()
|
||||
if not valid_time(time_value):
|
||||
continue
|
||||
count = max(0, min(20, int(item.get("count") or 0)))
|
||||
if count < 1:
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"id": str(item.get("id") or f"slot-{idx + 1}").strip()[:80],
|
||||
"time": time_value,
|
||||
"count": count,
|
||||
"enabled": bool(item.get("enabled", True)),
|
||||
}
|
||||
)
|
||||
return sorted(rows, key=lambda row: row["time"])
|
||||
|
||||
|
||||
def split_message_chunks(text: str, limit: int) -> list[str]:
|
||||
text = str(text or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
chunks: list[str] = []
|
||||
while len(text) > limit:
|
||||
split_at = text.rfind("\n", 0, limit)
|
||||
if split_at < limit // 2:
|
||||
split_at = text.rfind(" ", 0, limit)
|
||||
if split_at < limit // 2:
|
||||
split_at = limit
|
||||
chunks.append(text[:split_at].strip())
|
||||
text = text[split_at:].strip()
|
||||
if text:
|
||||
chunks.append(text)
|
||||
return chunks
|
||||
|
||||
|
||||
def local_upload_path(url: str) -> Path | None:
|
||||
raw = str(url or "").strip()
|
||||
prefix = "/uploads/"
|
||||
if not raw.startswith(prefix):
|
||||
return None
|
||||
path = (UPLOAD_ROOT / raw.removeprefix(prefix)).resolve()
|
||||
try:
|
||||
path.relative_to(UPLOAD_ROOT.resolve())
|
||||
except ValueError:
|
||||
return None
|
||||
return path if path.is_file() else None
|
||||
|
||||
|
||||
class MAXAPIClient:
|
||||
def __init__(self, token: str, base_url: str, timeout_sec: int, max_attempts: int, retry_backoff_max_sec: int) -> None:
|
||||
self.token = token
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.timeout_sec = timeout_sec
|
||||
self.max_attempts = max_attempts
|
||||
self.retry_backoff_max_sec = retry_backoff_max_sec
|
||||
self.session: aiohttp.ClientSession | None = None
|
||||
|
||||
async def __aenter__(self) -> "MAXAPIClient":
|
||||
timeout = aiohttp.ClientTimeout(total=self.timeout_sec)
|
||||
self.session = aiohttp.ClientSession(timeout=timeout, headers={"Authorization": self.token})
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb) -> None:
|
||||
if self.session:
|
||||
await self.session.close()
|
||||
|
||||
async def request(self, method: str, path: str, **kwargs) -> dict[str, Any]:
|
||||
if not self.session:
|
||||
raise RuntimeError("MAX session is not initialized")
|
||||
url = f"{self.base_url}{path}"
|
||||
last_error = ""
|
||||
for attempt in range(1, self.max_attempts + 1):
|
||||
try:
|
||||
async with self.session.request(method, url, **kwargs) as resp:
|
||||
text = await resp.text()
|
||||
try:
|
||||
data = json.loads(text) if text else {}
|
||||
except json.JSONDecodeError:
|
||||
data = {"raw": text}
|
||||
if 200 <= resp.status < 300:
|
||||
return data
|
||||
retry_after = resp.headers.get("Retry-After")
|
||||
message = data.get("message") if isinstance(data, dict) else text
|
||||
code = data.get("code") if isinstance(data, dict) else ""
|
||||
last_error = f"MAX API {resp.status} {code}: {message or text}"
|
||||
if resp.status == 429 and retry_after:
|
||||
await asyncio.sleep(float(retry_after) + 0.5)
|
||||
continue
|
||||
if code == "attachment.not.ready" and attempt < self.max_attempts:
|
||||
await asyncio.sleep(min(2 ** attempt, self.retry_backoff_max_sec))
|
||||
continue
|
||||
if resp.status < 500:
|
||||
raise MAXAPIError(last_error, resp.status, str(code or ""))
|
||||
except Exception as exc:
|
||||
last_error = str(exc)
|
||||
if attempt >= self.max_attempts:
|
||||
raise
|
||||
await asyncio.sleep(min(2 ** attempt, self.retry_backoff_max_sec))
|
||||
raise RuntimeError(last_error or "MAX API request failed")
|
||||
|
||||
async def send_message(self, chat_id: int, text: str, attachments: list[dict[str, Any]] | None = None) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {"text": text, "notify": True}
|
||||
if attachments:
|
||||
payload["attachments"] = attachments
|
||||
return await self.request("POST", f"/messages?chat_id={chat_id}", json=payload)
|
||||
|
||||
async def get_message(self, message_id: str) -> dict[str, Any]:
|
||||
return await self.request("GET", f"/messages/{quote(message_id, safe='')}")
|
||||
|
||||
async def get_video_info(self, token: str) -> dict[str, Any]:
|
||||
return await self.request("GET", f"/videos/{quote(token, safe='')}")
|
||||
|
||||
async def get_upload_url(self, media_type: str) -> dict[str, Any]:
|
||||
upload_type = "video" if media_type == "video" else "image"
|
||||
return await self.request("POST", f"/uploads?type={upload_type}")
|
||||
|
||||
async def upload_file(self, upload_url: str, path: Path) -> dict[str, Any]:
|
||||
if not self.session:
|
||||
raise RuntimeError("MAX session is not initialized")
|
||||
form = aiohttp.FormData()
|
||||
content_type = mimetypes.guess_type(str(path))[0] or "application/octet-stream"
|
||||
with path.open("rb") as fh:
|
||||
form.add_field("data", fh, filename=path.name, content_type=content_type)
|
||||
async with self.session.post(upload_url, data=form) as resp:
|
||||
text = await resp.text()
|
||||
try:
|
||||
data = json.loads(text) if text else {}
|
||||
except json.JSONDecodeError:
|
||||
data = {"raw": text}
|
||||
if resp.status < 200 or resp.status >= 300:
|
||||
raise RuntimeError(f"MAX upload {resp.status}: {data}")
|
||||
return data
|
||||
|
||||
async def react(self, path_template: str, message_id: str, reaction: str) -> None:
|
||||
path = path_template.format(message_id=message_id)
|
||||
payload = {"reaction": reaction}
|
||||
await self.request("POST", path, json=payload)
|
||||
|
||||
|
||||
class MAXPoster:
|
||||
def __init__(self) -> None:
|
||||
self.pool = None
|
||||
self.heartbeat = HeartbeatReporter(WORKER_MAX_POSTER, 30)
|
||||
self.token = ""
|
||||
self.chat_id = 0
|
||||
self.api_base_url = "https://platform-api2.max.ru"
|
||||
self.schedule: list[dict[str, Any]] = []
|
||||
self.interval_sec = 60
|
||||
self.message_limit = MAX_MESSAGE_LIMIT
|
||||
self.media_max_items = MAX_MEDIA_ITEMS
|
||||
self.media_process_delay_sec = 3.0
|
||||
self.video_ready_attempts = 5
|
||||
self.video_ready_delay_sec = 10.0
|
||||
self.max_attempts = 3
|
||||
self.retry_backoff_max_sec = 30
|
||||
self.send_delay_sec = 1.0
|
||||
self.recent_window = 20
|
||||
self.category_repeat_penalty = 3.0
|
||||
self.source_repeat_penalty = 4.0
|
||||
self.auto_reaction_enabled = True
|
||||
self.auto_reaction = "👍"
|
||||
self.reaction_path_template = "/messages/{message_id}/reactions"
|
||||
self.dry_run = False
|
||||
|
||||
async def init(self) -> None:
|
||||
self.pool = await get_pool()
|
||||
|
||||
async def reload_settings(self) -> None:
|
||||
self.token = str(await fetch_setting("max_poster_bot_token", "") or "").strip()
|
||||
self.chat_id = int(str(await fetch_setting("max_poster_chat_id", "0") or "0").strip())
|
||||
self.api_base_url = str(await fetch_setting("max_poster_api_base_url", self.api_base_url) or self.api_base_url).strip()
|
||||
self.schedule = normalize_schedule(await fetch_setting("max_poster_schedule_json", []))
|
||||
self.interval_sec = max(10, await fetch_int_setting("max_poster_interval_sec", 60))
|
||||
self.message_limit = max(512, min(MAX_MESSAGE_LIMIT, await fetch_int_setting("max_poster_message_limit", MAX_MESSAGE_LIMIT)))
|
||||
self.media_max_items = max(0, min(MAX_MEDIA_ITEMS, await fetch_int_setting("max_poster_media_max_items", MAX_MEDIA_ITEMS)))
|
||||
self.media_process_delay_sec = max(0.0, await fetch_float_setting("max_poster_media_process_delay_sec", 3.0))
|
||||
self.video_ready_attempts = max(1, await fetch_int_setting("max_poster_video_ready_attempts", 5))
|
||||
self.video_ready_delay_sec = max(1.0, await fetch_float_setting("max_poster_video_ready_delay_sec", 10.0))
|
||||
self.max_attempts = max(1, await fetch_int_setting("max_poster_max_attempts", 3))
|
||||
self.retry_backoff_max_sec = max(1, await fetch_int_setting("max_poster_retry_backoff_max_sec", 30))
|
||||
self.send_delay_sec = max(0.0, await fetch_float_setting("max_poster_send_delay_sec", 1.0))
|
||||
self.recent_window = max(1, await fetch_int_setting("max_poster_recent_window", 20))
|
||||
self.category_repeat_penalty = max(0.0, await fetch_float_setting("max_poster_category_repeat_penalty", 3.0))
|
||||
self.source_repeat_penalty = max(0.0, await fetch_float_setting("max_poster_source_repeat_penalty", 4.0))
|
||||
self.auto_reaction_enabled = await fetch_bool_setting("max_poster_auto_reaction_enabled", True)
|
||||
self.auto_reaction = str(await fetch_setting("max_poster_auto_reaction", "👍") or "👍").strip() or "👍"
|
||||
self.reaction_path_template = str(await fetch_setting("max_poster_reaction_path_template", self.reaction_path_template) or "").strip()
|
||||
self.dry_run = await fetch_bool_setting("max_poster_dry_run", False)
|
||||
if not self.token:
|
||||
raise RuntimeError("max_poster_bot_token is empty")
|
||||
if not self.chat_id:
|
||||
raise RuntimeError("max_poster_chat_id is empty")
|
||||
logger.info("MAX poster config: chat={} schedule={} dry_run={}", self.chat_id, self.schedule, self.dry_run)
|
||||
|
||||
async def due_slots(self) -> list[dict[str, Any]]:
|
||||
now = datetime.now(LOCAL_TZ)
|
||||
today = now.date()
|
||||
due: list[dict[str, Any]] = []
|
||||
for row in self.schedule:
|
||||
if not row.get("enabled"):
|
||||
continue
|
||||
hour, minute = [int(part) for part in str(row["time"]).split(":", 1)]
|
||||
slot_start = datetime.combine(today, time(hour=hour, minute=minute), tzinfo=LOCAL_TZ)
|
||||
slot_end = slot_start + timedelta(minutes=10)
|
||||
if not (slot_start <= now < slot_end):
|
||||
continue
|
||||
inserted = await self.pool.fetchrow(
|
||||
"""
|
||||
INSERT INTO publication_runs(poster, schedule_id, scheduled_for, scheduled_time, planned_count, status)
|
||||
VALUES($1, $2, $3, $4, $5, 'started')
|
||||
ON CONFLICT (poster, schedule_id, scheduled_for) DO NOTHING
|
||||
RETURNING id
|
||||
""",
|
||||
"max",
|
||||
row["id"],
|
||||
today,
|
||||
row["time"],
|
||||
int(row["count"]),
|
||||
)
|
||||
if inserted:
|
||||
due.append({**row, "run_id": int(inserted["id"]), "date": today})
|
||||
return due
|
||||
|
||||
async def load_recent(self) -> list[dict[str, Any]]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT rp.final_category_tag, rp.rewrite_category_tag, rp.final_source_tag, rp.rewrite_source_tag
|
||||
FROM post_publications pp
|
||||
JOIN raw_posts rp ON rp.id=pp.raw_post_id
|
||||
WHERE pp.platform='max'
|
||||
AND pp.status=$1
|
||||
AND pp.target_id=$2
|
||||
ORDER BY pp.published_at DESC NULLS LAST, pp.id DESC
|
||||
LIMIT $3::int
|
||||
""",
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
str(self.chat_id),
|
||||
self.recent_window,
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
async def load_candidates(self, limit: int) -> list[dict[str, Any]]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT rp.*, s.name AS source_name, s.tag AS source_tag
|
||||
FROM raw_posts rp
|
||||
JOIN sources s ON s.id=rp.source_id
|
||||
LEFT JOIN post_publications pp ON pp.raw_post_id=rp.id AND pp.platform='max'
|
||||
WHERE rp.status='storage_ready'
|
||||
AND rp.rewrite_status='ready'
|
||||
AND COALESCE(rp.editorial_status, 'review') IN ('accepted', 'published', 'publish_failed')
|
||||
AND COALESCE(pp.status, $1) = $1
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media rpm
|
||||
WHERE rpm.raw_post_id=rp.id
|
||||
AND COALESCE(rpm.editor_hidden, FALSE)=FALSE
|
||||
AND rpm.media_type IN ('photo', 'video')
|
||||
AND COALESCE(rpm.original_url, '') <> ''
|
||||
)
|
||||
ORDER BY COALESCE(rp.reviewed_at, rp.edited_at, rp.rewritten_at, rp.created_at) ASC,
|
||||
rp.id ASC
|
||||
LIMIT $2::int
|
||||
""",
|
||||
PUBLICATION_STATUS_PENDING,
|
||||
max(limit, 50),
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def choose_posts(self, candidates: list[dict[str, Any]], recent: list[dict[str, Any]], count: int) -> list[dict[str, Any]]:
|
||||
selected: list[dict[str, Any]] = []
|
||||
recent_categories = [str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "") for row in recent]
|
||||
recent_sources = [str(row.get("final_source_tag") or row.get("rewrite_source_tag") or "") for row in recent]
|
||||
pool = list(candidates)
|
||||
while pool and len(selected) < count:
|
||||
best_idx = 0
|
||||
selected_categories = [str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "") for row in selected]
|
||||
selected_sources = [str(row.get("final_source_tag") or row.get("rewrite_source_tag") or row.get("source_tag") or "") for row in selected]
|
||||
previous_category = selected_categories[-1] if selected_categories else (recent_categories[0] if recent_categories else "")
|
||||
previous_source = selected_sources[-1] if selected_sources else (recent_sources[0] if recent_sources else "")
|
||||
best_rank: tuple[int, float, int] | None = None
|
||||
for idx, post in enumerate(pool):
|
||||
category = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or "")
|
||||
source = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "")
|
||||
immediate_repeats = int(bool(category) and category == previous_category) + int(bool(source) and source == previous_source)
|
||||
repeat_penalty = (
|
||||
(recent_categories.count(category) + selected_categories.count(category)) * self.category_repeat_penalty
|
||||
+ (recent_sources.count(source) + selected_sources.count(source)) * self.source_repeat_penalty
|
||||
)
|
||||
rank = (immediate_repeats, repeat_penalty, idx)
|
||||
if best_rank is None or rank < best_rank:
|
||||
best_idx = idx
|
||||
best_rank = rank
|
||||
selected.append(pool.pop(best_idx))
|
||||
return selected
|
||||
|
||||
async def load_media(self, raw_post_id: int) -> list[dict[str, Any]]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT *
|
||||
FROM raw_post_media
|
||||
WHERE raw_post_id=$1
|
||||
AND COALESCE(editor_hidden, FALSE)=FALSE
|
||||
AND media_type IN ('photo', 'video')
|
||||
AND COALESCE(original_url, '') <> ''
|
||||
ORDER BY sort_order ASC, id ASC
|
||||
LIMIT $2::int
|
||||
""",
|
||||
raw_post_id,
|
||||
self.media_max_items,
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def build_text(self, post: dict[str, Any]) -> str:
|
||||
category_tag = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or post.get("final_category") or post.get("rewrite_category") or "")
|
||||
source_tag = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "")
|
||||
return build_publication_text(post.get("final_text") or post.get("rewritten_text") or "", category_tag, source_tag)
|
||||
|
||||
async def download_media_to_temp(self, session: aiohttp.ClientSession, item: dict[str, Any]) -> Path | None:
|
||||
media_type = str(item.get("media_type") or "")
|
||||
file_id = str(item.get("tg_file_id") or item.get("storage_attachment_id") or "").strip()
|
||||
if media_type == "video" and file_id:
|
||||
return await self.download_telegram_file_to_temp(session, file_id, ".mp4")
|
||||
url = str(item.get("original_url") or "").strip()
|
||||
local_path = local_upload_path(url)
|
||||
if local_path:
|
||||
return local_path
|
||||
if not url.startswith(("http://", "https://")):
|
||||
return None
|
||||
suffix = Path(url.split("?", 1)[0]).suffix[:12] or (".mp4" if item.get("media_type") == "video" else ".jpg")
|
||||
tmp = tempfile.NamedTemporaryFile(prefix="max-poster-", suffix=suffix, delete=False)
|
||||
path = Path(tmp.name)
|
||||
tmp.close()
|
||||
total = 0
|
||||
try:
|
||||
async with session.get(url) as resp:
|
||||
resp.raise_for_status()
|
||||
with path.open("wb") as fh:
|
||||
async for chunk in resp.content.iter_chunked(1024 * 512):
|
||||
total += len(chunk)
|
||||
if total > MAX_VIDEO_BYTES:
|
||||
raise RuntimeError("media file is larger than MAX video limit")
|
||||
fh.write(chunk)
|
||||
return path
|
||||
except Exception:
|
||||
path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
async def download_telegram_file_to_temp(self, session: aiohttp.ClientSession, file_id: str, suffix: str) -> Path:
|
||||
token = settings.tg_bot_token
|
||||
if not token:
|
||||
raise RuntimeError("TG_BOT_TOKEN is empty; cannot download stored video")
|
||||
api_base = str(await fetch_setting("local_bot_api_url", settings.local_bot_api_url) or "").strip().rstrip("/")
|
||||
if not api_base:
|
||||
api_base = "https://api.telegram.org"
|
||||
get_file_url = f"{api_base}/bot{token}/getFile"
|
||||
async with session.post(get_file_url, json={"file_id": file_id}) as resp:
|
||||
data = await resp.json(content_type=None)
|
||||
if not data.get("ok") or not isinstance(data.get("result"), dict):
|
||||
raise RuntimeError(f"Telegram getFile failed: {data}")
|
||||
file_path = str(data["result"].get("file_path") or "").strip()
|
||||
if not file_path:
|
||||
raise RuntimeError(f"Telegram getFile did not return file_path: {data}")
|
||||
local_file = Path(file_path)
|
||||
if local_file.is_absolute() and local_file.is_file():
|
||||
if local_file.stat().st_size > MAX_VIDEO_BYTES:
|
||||
raise RuntimeError("telegram local video file is larger than MAX video limit")
|
||||
tmp = tempfile.NamedTemporaryFile(prefix="max-poster-tg-", suffix=suffix, delete=False)
|
||||
path = Path(tmp.name)
|
||||
tmp.close()
|
||||
try:
|
||||
shutil.copyfile(local_file, path)
|
||||
return path
|
||||
except Exception:
|
||||
path.unlink(missing_ok=True)
|
||||
raise
|
||||
download_url = f"{api_base}/file/bot{token}/{file_path}"
|
||||
tmp = tempfile.NamedTemporaryFile(prefix="max-poster-tg-", suffix=suffix, delete=False)
|
||||
path = Path(tmp.name)
|
||||
tmp.close()
|
||||
total = 0
|
||||
try:
|
||||
async with session.get(download_url) as resp:
|
||||
resp.raise_for_status()
|
||||
with path.open("wb") as fh:
|
||||
async for chunk in resp.content.iter_chunked(1024 * 1024):
|
||||
total += len(chunk)
|
||||
if total > MAX_VIDEO_BYTES:
|
||||
raise RuntimeError("telegram video file is larger than MAX video limit")
|
||||
fh.write(chunk)
|
||||
return path
|
||||
except Exception:
|
||||
path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
async def upload_media(self, client: MAXAPIClient, item: dict[str, Any]) -> dict[str, Any] | None:
|
||||
media_type = str(item.get("media_type") or "")
|
||||
if media_type not in {"photo", "video"}:
|
||||
return None
|
||||
temp_path: Path | None = None
|
||||
downloaded = False
|
||||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=180)) as session:
|
||||
temp_path = await self.download_media_to_temp(session, item)
|
||||
downloaded = bool(temp_path and not local_upload_path(str(item.get("original_url") or "")))
|
||||
if not temp_path:
|
||||
return None
|
||||
try:
|
||||
upload_info = await client.get_upload_url("video" if media_type == "video" else "image")
|
||||
upload_url = str(upload_info.get("url") or "").strip()
|
||||
if not upload_url:
|
||||
raise RuntimeError(f"MAX upload URL is empty: {upload_info}")
|
||||
payload = await client.upload_file(upload_url, temp_path)
|
||||
if media_type == "video" and upload_info.get("token") and not payload.get("token"):
|
||||
payload["token"] = upload_info["token"]
|
||||
has_image_token = bool(payload.get("photos")) if isinstance(payload.get("photos"), dict) else False
|
||||
if not payload.get("token") and not has_image_token:
|
||||
raise RuntimeError(f"MAX upload did not return token: {payload}")
|
||||
return {"type": "video" if media_type == "video" else "image", "payload": payload}
|
||||
finally:
|
||||
if downloaded and temp_path:
|
||||
temp_path.unlink(missing_ok=True)
|
||||
|
||||
def message_id_from_response(self, data: dict[str, Any]) -> str:
|
||||
candidates = [
|
||||
data.get("id"),
|
||||
data.get("mid"),
|
||||
(data.get("message") or {}).get("id") if isinstance(data.get("message"), dict) else None,
|
||||
(data.get("message") or {}).get("mid") if isinstance(data.get("message"), dict) else None,
|
||||
]
|
||||
message = data.get("message") if isinstance(data.get("message"), dict) else {}
|
||||
body = message.get("body") if isinstance(message.get("body"), dict) else {}
|
||||
candidates.extend([body.get("id"), body.get("mid"), body.get("message_id")])
|
||||
return next((str(value) for value in candidates if value), "")
|
||||
|
||||
def message_url_from_response(self, data: dict[str, Any]) -> str | None:
|
||||
candidates = [data.get("url")]
|
||||
message = data.get("message") if isinstance(data.get("message"), dict) else {}
|
||||
body = message.get("body") if isinstance(message.get("body"), dict) else {}
|
||||
candidates.extend([message.get("url"), body.get("url")])
|
||||
return next((str(value).strip() for value in candidates if str(value or "").strip()), None)
|
||||
|
||||
async def send_message_waiting_for_media(
|
||||
self,
|
||||
client: MAXAPIClient,
|
||||
text: str,
|
||||
attachments: list[dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
has_video = any(item.get("type") == "video" for item in attachments)
|
||||
attempts = self.video_ready_attempts if has_video else 1
|
||||
delay = self.video_ready_delay_sec
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
return await client.send_message(self.chat_id, text, attachments=attachments or None)
|
||||
except MAXAPIError as exc:
|
||||
if exc.code != "attachment.not.ready" or attempt >= attempts:
|
||||
raise
|
||||
logger.warning(
|
||||
"MAX attachment is not ready, retry send in {}s attempt={}/{}",
|
||||
delay,
|
||||
attempt + 1,
|
||||
attempts,
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
delay = min(delay * 2, self.retry_backoff_max_sec * 4)
|
||||
raise RuntimeError("MAX attachment readiness retry exhausted")
|
||||
|
||||
def video_tokens(self, attachments: list[dict[str, Any]]) -> list[str]:
|
||||
tokens: list[str] = []
|
||||
for item in attachments:
|
||||
if item.get("type") != "video":
|
||||
continue
|
||||
payload = item.get("payload") if isinstance(item.get("payload"), dict) else {}
|
||||
token = str(payload.get("token") or "").strip()
|
||||
if token:
|
||||
tokens.append(token)
|
||||
return tokens
|
||||
|
||||
async def wait_for_videos(self, client: MAXAPIClient, attachments: list[dict[str, Any]]) -> None:
|
||||
tokens = self.video_tokens(attachments)
|
||||
if not tokens:
|
||||
return
|
||||
delay = self.video_ready_delay_sec
|
||||
for attempt in range(1, self.video_ready_attempts + 1):
|
||||
pending: list[str] = []
|
||||
for token in tokens:
|
||||
info = await client.get_video_info(token)
|
||||
if not info.get("urls"):
|
||||
pending.append(token)
|
||||
if not pending:
|
||||
return
|
||||
if attempt >= self.video_ready_attempts:
|
||||
raise RuntimeError(f"MAX video is not ready after {attempt} attempts")
|
||||
logger.warning(
|
||||
"MAX video is not ready, retry info in {}s attempt={}/{} pending={}",
|
||||
delay,
|
||||
attempt + 1,
|
||||
self.video_ready_attempts,
|
||||
len(pending),
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
delay = min(delay * 2, self.retry_backoff_max_sec * 4)
|
||||
|
||||
async def send_post(self, post: dict[str, Any]) -> tuple[list[str], str | None, list[dict[str, Any]], str | None]:
|
||||
text = self.build_text(post)
|
||||
chunks = split_message_chunks(text, self.message_limit)
|
||||
media_rows = await self.load_media(int(post["id"])) if self.media_max_items else []
|
||||
message_ids: list[str] = []
|
||||
attachments: list[dict[str, Any]] = []
|
||||
first_url: str | None = None
|
||||
reaction_error: str | None = None
|
||||
async with MAXAPIClient(
|
||||
self.token,
|
||||
self.api_base_url,
|
||||
timeout_sec=max(30, await fetch_int_setting("max_poster_timeout_sec", 120)),
|
||||
max_attempts=self.max_attempts,
|
||||
retry_backoff_max_sec=self.retry_backoff_max_sec,
|
||||
) as client:
|
||||
for item in media_rows:
|
||||
try:
|
||||
attachment = await self.upload_media(client, item)
|
||||
if attachment:
|
||||
attachments.append(attachment)
|
||||
except Exception as exc:
|
||||
logger.warning("MAX media skipped raw_post={} media_id={}: {}", post["id"], item.get("id"), exc)
|
||||
if len(attachments) >= self.media_max_items:
|
||||
break
|
||||
if self.media_process_delay_sec and attachments:
|
||||
await asyncio.sleep(self.media_process_delay_sec)
|
||||
if self.dry_run:
|
||||
return ["dry-run"], None, attachments, None
|
||||
await self.wait_for_videos(client, attachments)
|
||||
first_text = chunks[0] if chunks else ""
|
||||
first = await self.send_message_waiting_for_media(client, first_text, attachments)
|
||||
message = first.get("message") if isinstance(first.get("message"), dict) else first
|
||||
first_url = self.message_url_from_response(message) if isinstance(message, dict) else None
|
||||
first_message_id = self.message_id_from_response(first)
|
||||
if first_message_id:
|
||||
message_ids.append(first_message_id)
|
||||
if not first_url:
|
||||
try:
|
||||
first_url = self.message_url_from_response(await client.get_message(first_message_id))
|
||||
except Exception as exc:
|
||||
logger.warning("MAX message URL lookup failed message={}: {}", first_message_id, exc)
|
||||
if self.auto_reaction_enabled and self.reaction_path_template:
|
||||
try:
|
||||
await client.react(self.reaction_path_template, first_message_id, self.auto_reaction)
|
||||
except Exception as exc:
|
||||
reaction_error = str(exc)
|
||||
logger.warning("MAX auto reaction failed message={}: {}", first_message_id, exc)
|
||||
for chunk in chunks[1:]:
|
||||
if self.send_delay_sec:
|
||||
await asyncio.sleep(self.send_delay_sec)
|
||||
result = await client.send_message(self.chat_id, chunk)
|
||||
message_id = self.message_id_from_response(result)
|
||||
if message_id:
|
||||
message_ids.append(message_id)
|
||||
return message_ids, first_url, attachments, reaction_error
|
||||
|
||||
async def upsert_publication(
|
||||
self,
|
||||
post_id: int,
|
||||
status: str,
|
||||
external_id: str | None = None,
|
||||
url: str | None = None,
|
||||
error: str | None = None,
|
||||
attachments: list[dict[str, Any]] | None = None,
|
||||
) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
INSERT INTO post_publications(
|
||||
raw_post_id, platform, status, target_id, external_id, url, error, attachments_json, published_at, updated_at
|
||||
)
|
||||
VALUES($1, 'max', $2, $3, $4, $5, $6, $7::jsonb, CASE WHEN $2=$8 THEN NOW() ELSE NULL END, NOW())
|
||||
ON CONFLICT (raw_post_id, platform) DO UPDATE
|
||||
SET status=EXCLUDED.status,
|
||||
target_id=EXCLUDED.target_id,
|
||||
external_id=EXCLUDED.external_id,
|
||||
url=EXCLUDED.url,
|
||||
error=EXCLUDED.error,
|
||||
attachments_json=EXCLUDED.attachments_json,
|
||||
published_at=CASE WHEN EXCLUDED.status=$8 THEN COALESCE(EXCLUDED.published_at, NOW()) ELSE post_publications.published_at END,
|
||||
updated_at=NOW()
|
||||
""",
|
||||
post_id,
|
||||
status,
|
||||
str(self.chat_id),
|
||||
external_id,
|
||||
url,
|
||||
error[:1000] if error else None,
|
||||
json.dumps(attachments or [], ensure_ascii=False),
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
)
|
||||
|
||||
async def process_slot(self, slot: dict[str, Any]) -> int:
|
||||
count = int(slot["count"])
|
||||
recent = await self.load_recent()
|
||||
candidates = await self.load_candidates(max(50, count * 10))
|
||||
posts = self.choose_posts(candidates, recent, count)
|
||||
published = 0
|
||||
for post in posts:
|
||||
await self.heartbeat.beat(self.pool, status="publishing", meta={"post_id": int(post["id"]), "slot": slot["id"]}, force=True)
|
||||
try:
|
||||
message_ids, url, attachments, reaction_error = await self.send_post(post)
|
||||
await self.upsert_publication(
|
||||
int(post["id"]),
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
",".join(message_ids),
|
||||
url,
|
||||
error=None,
|
||||
attachments=attachments,
|
||||
)
|
||||
published += 1
|
||||
logger.info("MAX published raw_post={} messages={} attachments={}", post["id"], message_ids, len(attachments))
|
||||
except Exception as exc:
|
||||
await self.upsert_publication(int(post["id"]), PUBLICATION_STATUS_FAILED, error=str(exc))
|
||||
logger.exception("MAX publish failed raw_post={}: {}", post["id"], exc)
|
||||
if self.send_delay_sec:
|
||||
await asyncio.sleep(self.send_delay_sec)
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE publication_runs
|
||||
SET published_count=$2,
|
||||
status=$3,
|
||||
error=$4,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
int(slot["run_id"]),
|
||||
published,
|
||||
"done" if published == count else "partial",
|
||||
None if published == count else f"published {published} of {count}",
|
||||
)
|
||||
return published
|
||||
|
||||
async def run_once(self) -> bool:
|
||||
await self.reload_settings()
|
||||
enabled = await is_worker_enabled(self.pool, WORKER_MAX_POSTER)
|
||||
app_enabled = await fetch_bool_setting("max_poster_enabled", False)
|
||||
if not enabled or not app_enabled:
|
||||
await self.heartbeat.beat(self.pool, status="disabled", force=True)
|
||||
return False
|
||||
slots = await self.due_slots()
|
||||
if not slots:
|
||||
await self.heartbeat.beat(self.pool, status="idle")
|
||||
return False
|
||||
total = 0
|
||||
for slot in slots:
|
||||
total += await self.process_slot(slot)
|
||||
await self.heartbeat.beat(self.pool, status="idle", meta={"published": total}, force=True)
|
||||
return bool(total)
|
||||
|
||||
async def run_loop(self) -> None:
|
||||
await self.init()
|
||||
logger.info("{} started pid={}", WORKER_MAX_POSTER, os.getpid())
|
||||
while True:
|
||||
try:
|
||||
await self.run_once()
|
||||
except Exception as exc:
|
||||
logger.exception("MAX poster loop error: {}", exc)
|
||||
await asyncio.sleep(self.interval_sec)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
logger.remove()
|
||||
logger.add(sys.stdout, level=settings.log_level)
|
||||
worker = MAXPoster()
|
||||
await worker.run_loop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,449 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from ..config import settings
|
||||
from ..constants import (
|
||||
JOB_TYPE_VK_STORAGE_COPY,
|
||||
PLATFORM_VK,
|
||||
POST_STATUS_SKIPPED,
|
||||
POST_STATUS_STORAGE_PENDING,
|
||||
SOURCE_STATUS_ERROR,
|
||||
SOURCE_STATUS_OK,
|
||||
WORKER_PARSER,
|
||||
)
|
||||
from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, get_pool
|
||||
from ..heartbeat import HeartbeatReporter
|
||||
from ..jobs import is_worker_enabled
|
||||
from ..vk_api import (
|
||||
VKAPIClient,
|
||||
VKAPIError,
|
||||
extract_media,
|
||||
is_deleted_or_invalid,
|
||||
is_fatal_source_error,
|
||||
is_repost,
|
||||
post_vk_url,
|
||||
)
|
||||
|
||||
|
||||
def utc_from_ts(value: int) -> datetime:
|
||||
return datetime.fromtimestamp(int(value), tz=timezone.utc)
|
||||
|
||||
|
||||
def make_hash(*parts: str) -> str:
|
||||
h = hashlib.sha256()
|
||||
for part in parts:
|
||||
h.update((part or "").strip().lower().encode("utf-8"))
|
||||
h.update(b"\0")
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
class VKParserWorker:
|
||||
def __init__(self) -> None:
|
||||
self.pool = None
|
||||
self.heartbeat = HeartbeatReporter(WORKER_PARSER, 30)
|
||||
|
||||
async def init(self) -> None:
|
||||
self.pool = await get_pool()
|
||||
|
||||
async def active_sources(self) -> list[dict]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT *
|
||||
FROM sources
|
||||
WHERE platform=$1
|
||||
AND active=TRUE
|
||||
AND archived_at IS NULL
|
||||
ORDER BY last_checked_at NULLS FIRST, priority ASC, id ASC
|
||||
""",
|
||||
PLATFORM_VK,
|
||||
)
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
async def mark_source_error(self, source_id: int, message: str) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE sources
|
||||
SET status=$2,
|
||||
status_msg=$3,
|
||||
last_checked_at=NOW(),
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
source_id,
|
||||
SOURCE_STATUS_ERROR,
|
||||
message[:1000],
|
||||
)
|
||||
|
||||
async def deactivate_source(self, source_id: int, message: str) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE sources
|
||||
SET active=FALSE,
|
||||
status=$2,
|
||||
status_msg=$3,
|
||||
last_checked_at=NOW(),
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
source_id,
|
||||
SOURCE_STATUS_ERROR,
|
||||
message[:1000],
|
||||
)
|
||||
|
||||
async def mark_source_ok(self, source_id: int, last_parsed_at: datetime | None) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE sources
|
||||
SET status=$2,
|
||||
status_msg=NULL,
|
||||
last_checked_at=NOW(),
|
||||
last_parsed_at=COALESCE($3, last_parsed_at),
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
source_id,
|
||||
SOURCE_STATUS_OK,
|
||||
last_parsed_at,
|
||||
)
|
||||
|
||||
async def resolve_source_if_needed(self, client: VKAPIClient, source: dict) -> dict:
|
||||
if source.get("external_owner_id"):
|
||||
return source
|
||||
external_id, owner_id, resolved_name = await client.resolve_group(source["url"] or source.get("external_id") or "")
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE sources
|
||||
SET external_id=$2,
|
||||
external_owner_id=$3,
|
||||
name=CASE WHEN name='' OR name=url THEN $4 ELSE name END,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
int(source["id"]),
|
||||
external_id,
|
||||
owner_id,
|
||||
resolved_name,
|
||||
)
|
||||
source["external_id"] = external_id
|
||||
source["external_owner_id"] = owner_id
|
||||
if not source.get("name") or source.get("name") == source.get("url"):
|
||||
source["name"] = resolved_name
|
||||
return source
|
||||
|
||||
async def known_post_ids(self, source_id: int, external_post_ids: list[str]) -> set[str]:
|
||||
if not external_post_ids:
|
||||
return set()
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT external_post_id
|
||||
FROM raw_posts
|
||||
WHERE source_id=$1 AND external_post_id=ANY($2::text[])
|
||||
""",
|
||||
source_id,
|
||||
external_post_ids,
|
||||
)
|
||||
return {str(r["external_post_id"]) for r in rows}
|
||||
|
||||
async def known_content_hashes(self, hashes: list[str]) -> set[str]:
|
||||
if not hashes:
|
||||
return set()
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT content_hash
|
||||
FROM raw_posts
|
||||
WHERE content_hash=ANY($1::text[])
|
||||
""",
|
||||
hashes,
|
||||
)
|
||||
return {str(r["content_hash"]) for r in rows}
|
||||
|
||||
async def save_post(
|
||||
self,
|
||||
source: dict,
|
||||
post: dict,
|
||||
*,
|
||||
status: str,
|
||||
skip_reason: str | None = None,
|
||||
create_storage_job: bool = True,
|
||||
) -> int | None:
|
||||
source_id = int(source["id"])
|
||||
owner_id = int(post.get("owner_id") or source["external_owner_id"])
|
||||
external_post_id = str(post["id"])
|
||||
raw_text = str(post.get("text") or "").strip()
|
||||
posted_at = utc_from_ts(int(post["date"]))
|
||||
media = extract_media(post)
|
||||
media_ids = ",".join(m.attachment_id for m in media)
|
||||
text_hash = make_hash(raw_text)
|
||||
content_hash = make_hash(raw_text, media_ids)
|
||||
original_url = post_vk_url(owner_id, external_post_id)
|
||||
|
||||
async with self.pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
raw_post_id = await conn.fetchval(
|
||||
"""
|
||||
INSERT INTO raw_posts(
|
||||
source_id, platform, external_post_id, external_owner_id,
|
||||
original_url, raw_text, raw_json, text_hash, content_hash,
|
||||
posted_at, status, skip_reason
|
||||
)
|
||||
VALUES($1,$2,$3,$4,$5,$6,$7::jsonb,$8,$9,$10,$11,$12)
|
||||
ON CONFLICT (source_id, external_post_id) DO NOTHING
|
||||
RETURNING id
|
||||
""",
|
||||
source_id,
|
||||
PLATFORM_VK,
|
||||
external_post_id,
|
||||
owner_id,
|
||||
original_url,
|
||||
raw_text,
|
||||
json.dumps(post, ensure_ascii=False),
|
||||
text_hash,
|
||||
content_hash,
|
||||
posted_at,
|
||||
status,
|
||||
skip_reason,
|
||||
)
|
||||
if raw_post_id is None:
|
||||
return None
|
||||
|
||||
for m in media:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO raw_post_media(
|
||||
raw_post_id, platform, media_type, original_url, original_attachment_id,
|
||||
width, height, duration_sec, sort_order
|
||||
)
|
||||
VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9)
|
||||
""",
|
||||
raw_post_id,
|
||||
PLATFORM_VK,
|
||||
m.media_type,
|
||||
m.original_url,
|
||||
m.attachment_id,
|
||||
m.width,
|
||||
m.height,
|
||||
m.duration_sec,
|
||||
m.sort_order,
|
||||
)
|
||||
|
||||
if create_storage_job:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO jobs(type, entity_type, entity_id, payload_json, status)
|
||||
VALUES($1, 'raw_post', $2, '{}'::jsonb, 'pending')
|
||||
ON CONFLICT DO NOTHING
|
||||
""",
|
||||
JOB_TYPE_VK_STORAGE_COPY,
|
||||
raw_post_id,
|
||||
)
|
||||
return int(raw_post_id)
|
||||
|
||||
async def parse_source(self, client: VKAPIClient, source: dict) -> int:
|
||||
source = await self.resolve_source_if_needed(client, source)
|
||||
source_id = int(source["id"])
|
||||
owner_id = int(source["external_owner_id"])
|
||||
page_size = max(1, min(100, await fetch_int_setting("vk_wall_page_size", 50)))
|
||||
lookback_days = max(1, await fetch_int_setting("parser_new_source_lookback_days", 14))
|
||||
overlap_minutes = max(0, await fetch_int_setting("parser_reparse_overlap_minutes", 120))
|
||||
|
||||
last_parsed_at = source.get("last_parsed_at")
|
||||
parse_from = source.get("parse_from")
|
||||
if last_parsed_at:
|
||||
since_dt = last_parsed_at - timedelta(minutes=overlap_minutes)
|
||||
elif parse_from:
|
||||
since_dt = parse_from
|
||||
else:
|
||||
since_dt = datetime.now(timezone.utc) - timedelta(days=lookback_days)
|
||||
if since_dt.tzinfo is None:
|
||||
since_dt = since_dt.replace(tzinfo=timezone.utc)
|
||||
since_ts = int(since_dt.timestamp())
|
||||
|
||||
fetched: list[dict] = []
|
||||
offset = 0
|
||||
stop = False
|
||||
while not stop:
|
||||
response = await client.get_wall_posts(owner_id=owner_id, count=page_size, offset=offset)
|
||||
items = response.get("items", []) or []
|
||||
if not items:
|
||||
break
|
||||
for post in items:
|
||||
if is_deleted_or_invalid(post):
|
||||
continue
|
||||
if int(post["date"]) <= since_ts:
|
||||
stop = True
|
||||
break
|
||||
fetched.append(post)
|
||||
if len(items) < page_size:
|
||||
break
|
||||
offset += page_size
|
||||
|
||||
known = await self.known_post_ids(source_id, [str(p["id"]) for p in fetched if p.get("id")])
|
||||
candidates = [p for p in fetched if str(p.get("id")) not in known]
|
||||
saved = 0
|
||||
max_seen: datetime | None = None
|
||||
min_text_length = max(0, await fetch_int_setting("parser_min_text_length", 0))
|
||||
skip_reposts = await fetch_bool_setting("parser_skip_reposts", True)
|
||||
skip_empty_text = await fetch_bool_setting("parser_skip_empty_text", True)
|
||||
skip_no_media = await fetch_bool_setting("parser_skip_no_media", True)
|
||||
skip_short_text = await fetch_bool_setting("parser_skip_text_too_short", True)
|
||||
store_skipped = await fetch_bool_setting("parser_store_skipped_posts", False)
|
||||
dedupe_content_hash = await fetch_bool_setting("parser_dedupe_content_hash", True)
|
||||
|
||||
hash_by_post_id: dict[str, str] = {}
|
||||
if dedupe_content_hash:
|
||||
candidate_hashes: list[str] = []
|
||||
for post in candidates:
|
||||
if is_deleted_or_invalid(post) or (skip_reposts and is_repost(post)):
|
||||
continue
|
||||
media = extract_media(post)
|
||||
media_ids = ",".join(m.attachment_id for m in media)
|
||||
content_hash = make_hash(str(post.get("text") or "").strip(), media_ids)
|
||||
hash_by_post_id[str(post["id"])] = content_hash
|
||||
candidate_hashes.append(content_hash)
|
||||
known_hashes = await self.known_content_hashes(candidate_hashes)
|
||||
else:
|
||||
known_hashes = set()
|
||||
|
||||
for post in candidates:
|
||||
if is_deleted_or_invalid(post):
|
||||
continue
|
||||
text = str(post.get("text") or "").strip()
|
||||
posted_at = utc_from_ts(int(post["date"]))
|
||||
if max_seen is None or posted_at > max_seen:
|
||||
max_seen = posted_at
|
||||
|
||||
if skip_reposts and is_repost(post):
|
||||
continue
|
||||
|
||||
media = extract_media(post)
|
||||
content_hash = hash_by_post_id.get(str(post["id"]))
|
||||
if dedupe_content_hash and content_hash in known_hashes:
|
||||
continue
|
||||
|
||||
skip_reason: str | None = None
|
||||
if skip_empty_text and not text:
|
||||
skip_reason = "empty_text"
|
||||
elif skip_no_media and not media:
|
||||
skip_reason = "no_media"
|
||||
elif skip_short_text and len(text) < min_text_length:
|
||||
skip_reason = "text_too_short"
|
||||
|
||||
if skip_reason:
|
||||
if not store_skipped:
|
||||
continue
|
||||
raw_id = await self.save_post(
|
||||
source,
|
||||
post,
|
||||
status=POST_STATUS_SKIPPED,
|
||||
skip_reason=skip_reason,
|
||||
create_storage_job=False,
|
||||
)
|
||||
else:
|
||||
raw_id = await self.save_post(
|
||||
source,
|
||||
post,
|
||||
status=POST_STATUS_STORAGE_PENDING,
|
||||
skip_reason=None,
|
||||
create_storage_job=True,
|
||||
)
|
||||
if raw_id:
|
||||
saved += 1
|
||||
if content_hash:
|
||||
known_hashes.add(content_hash)
|
||||
|
||||
await self.mark_source_ok(source_id, max_seen or last_parsed_at)
|
||||
logger.info(
|
||||
"Parsed source {}: fetched={} known={} candidates={} saved={}",
|
||||
source.get("name"),
|
||||
len(fetched),
|
||||
len(known),
|
||||
len(candidates),
|
||||
saved,
|
||||
)
|
||||
return saved
|
||||
|
||||
async def run_once(self) -> None:
|
||||
enabled = await is_worker_enabled(self.pool, WORKER_PARSER)
|
||||
if not enabled:
|
||||
await self.heartbeat.beat(self.pool, status="disabled", force=True)
|
||||
return
|
||||
|
||||
rps = await fetch_int_setting("vk_requests_per_second", 3)
|
||||
timeout_total = await fetch_int_setting("vk_api_timeout_total_sec", 15)
|
||||
timeout_connect = await fetch_int_setting("vk_api_timeout_connect_sec", 5)
|
||||
rate_limit_sleep = await fetch_float_setting("vk_rate_limit_sleep_sec", 1.0)
|
||||
retry_attempts = await fetch_int_setting("vk_api_retry_attempts", 3)
|
||||
retry_min_delay = await fetch_float_setting("vk_api_retry_min_delay_sec", 2.0)
|
||||
retry_max_delay = await fetch_float_setting("vk_api_retry_max_delay_sec", 10.0)
|
||||
source_pause = max(0.0, await fetch_float_setting("parser_source_pause_sec", 0.0))
|
||||
sources = await self.active_sources()
|
||||
await self.heartbeat.beat(self.pool, meta={"sources": len(sources)})
|
||||
if not sources:
|
||||
logger.info("No active VK sources")
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"Parser cycle: sources={} rps={} timeout={}/{} retry={}/{}..{} rate_limit_sleep={} source_pause={}",
|
||||
len(sources),
|
||||
rps,
|
||||
timeout_total,
|
||||
timeout_connect,
|
||||
retry_attempts,
|
||||
retry_min_delay,
|
||||
retry_max_delay,
|
||||
rate_limit_sleep,
|
||||
source_pause,
|
||||
)
|
||||
|
||||
async with VKAPIClient(
|
||||
rps=rps,
|
||||
timeout_total_sec=timeout_total,
|
||||
timeout_connect_sec=timeout_connect,
|
||||
rate_limit_sleep_sec=rate_limit_sleep,
|
||||
retry_attempts=retry_attempts,
|
||||
retry_min_delay_sec=retry_min_delay,
|
||||
retry_max_delay_sec=retry_max_delay,
|
||||
) as client:
|
||||
for source in sources:
|
||||
try:
|
||||
await self.parse_source(client, source)
|
||||
except VKAPIError as e:
|
||||
if is_fatal_source_error(e):
|
||||
await self.deactivate_source(int(source["id"]), str(e))
|
||||
logger.warning("VK source deactivated {}: {}", source.get("name"), e)
|
||||
else:
|
||||
await self.mark_source_error(int(source["id"]), str(e))
|
||||
logger.warning("VK source temporary error {}: {}", source.get("name"), e)
|
||||
except Exception as e:
|
||||
await self.mark_source_error(int(source["id"]), str(e))
|
||||
logger.exception("Unexpected source error {}: {}", source.get("name"), e)
|
||||
if source_pause:
|
||||
await asyncio.sleep(source_pause)
|
||||
|
||||
async def run_loop(self) -> None:
|
||||
await self.init()
|
||||
logger.info("{} started", WORKER_PARSER)
|
||||
while True:
|
||||
try:
|
||||
await self.run_once()
|
||||
except Exception as e:
|
||||
logger.exception("Parser loop error: {}", e)
|
||||
interval = max(10, await fetch_int_setting("parser_interval_sec", 300))
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
logger.remove()
|
||||
logger.add(lambda msg: print(msg, end=""), level=settings.log_level)
|
||||
worker = VKParserWorker()
|
||||
await worker.run_loop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,166 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from ..config import settings
|
||||
from ..constants import WORKER_SITE_POSTER
|
||||
from ..db import fetch_bool_setting, fetch_int_setting, fetch_setting, get_pool
|
||||
from ..heartbeat import HeartbeatReporter
|
||||
from ..jobs import is_worker_enabled
|
||||
|
||||
|
||||
class SitePoster:
|
||||
def __init__(self) -> None:
|
||||
self.pool = None
|
||||
self.heartbeat = HeartbeatReporter(WORKER_SITE_POSTER, 30)
|
||||
self.project_root = Path(__file__).resolve().parents[3]
|
||||
self.interval_sec = 300
|
||||
self.photo_batch_size = 10
|
||||
self.video_batch_size = 1
|
||||
self.provider = ""
|
||||
self.base_url = ""
|
||||
self.secret_file = ""
|
||||
self.fallback_title = "Новость"
|
||||
|
||||
async def init(self) -> None:
|
||||
self.pool = await get_pool()
|
||||
|
||||
async def reload_settings(self) -> None:
|
||||
self.interval_sec = max(30, await fetch_int_setting("site_poster_interval_sec", 300))
|
||||
self.photo_batch_size = max(0, await fetch_int_setting("site_poster_photo_batch_size", 10))
|
||||
self.video_batch_size = max(0, await fetch_int_setting("site_poster_video_batch_size", 1))
|
||||
legacy_ghost_url = str(await fetch_setting("site_poster_ghost_url", "") or "").strip()
|
||||
legacy_key_file = str(await fetch_setting("site_poster_key_file", "") or "").strip()
|
||||
self.provider = str(await fetch_setting("site_poster_provider", "") or "").strip().lower()
|
||||
if not self.provider and legacy_ghost_url:
|
||||
self.provider = "ghost"
|
||||
self.base_url = str(
|
||||
await fetch_setting("site_poster_base_url", "")
|
||||
or legacy_ghost_url
|
||||
or ""
|
||||
).rstrip("/")
|
||||
self.secret_file = str(
|
||||
await fetch_setting("site_poster_secret_file", "")
|
||||
or legacy_key_file
|
||||
or ""
|
||||
).strip()
|
||||
self.fallback_title = str(await fetch_setting("site_poster_fallback_title", "Новость") or "Новость").strip()
|
||||
|
||||
def build_publish_command(self, flag: str, limit: int) -> list[str]:
|
||||
if self.provider != "ghost":
|
||||
raise RuntimeError(f"unsupported site_poster_provider: {self.provider or '<empty>'}")
|
||||
if not self.base_url:
|
||||
raise RuntimeError("site_poster_base_url is empty")
|
||||
if not self.secret_file and not os.environ.get("SITE_POSTER_GHOST_ADMIN_KEY", "").strip():
|
||||
raise RuntimeError("site_poster_secret_file and SITE_POSTER_GHOST_ADMIN_KEY are empty")
|
||||
return [
|
||||
sys.executable,
|
||||
str(self.project_root / "scripts" / "publish_site_batch.py"),
|
||||
flag,
|
||||
"--limit",
|
||||
str(limit),
|
||||
"--ghost-url",
|
||||
self.base_url,
|
||||
"--key-file",
|
||||
self.secret_file,
|
||||
"--fallback-title",
|
||||
self.fallback_title or "Новость",
|
||||
]
|
||||
|
||||
async def publish_mode(self, flag: str, limit: int) -> tuple[int, int]:
|
||||
if limit < 1:
|
||||
return 0, 0
|
||||
command = self.build_publish_command(flag, limit)
|
||||
env = os.environ.copy()
|
||||
env["PYTHONPATH"] = str(self.project_root / "src")
|
||||
env["PYTHONUNBUFFERED"] = "1"
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*command,
|
||||
cwd=str(self.project_root),
|
||||
env=env,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
)
|
||||
published = 0
|
||||
failed = 0
|
||||
assert process.stdout is not None
|
||||
while True:
|
||||
try:
|
||||
raw_line = await asyncio.wait_for(process.stdout.readline(), timeout=20)
|
||||
except TimeoutError:
|
||||
await self.heartbeat.beat(
|
||||
self.pool,
|
||||
status="publishing",
|
||||
meta={"mode": flag, "published": published, "failed": failed},
|
||||
force=True,
|
||||
)
|
||||
continue
|
||||
if not raw_line:
|
||||
break
|
||||
line = raw_line.decode("utf-8", errors="replace").strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("PUBLISHED "):
|
||||
published += 1
|
||||
logger.info("Site poster: {}", line)
|
||||
elif line.startswith("FAILED "):
|
||||
failed += 1
|
||||
logger.error("Site poster: {}", line)
|
||||
elif line not in {"NO_READY_POSTS", "ALREADY_RUNNING"}:
|
||||
logger.debug("Site poster child: {}", line)
|
||||
await self.heartbeat.beat(
|
||||
self.pool,
|
||||
status="publishing",
|
||||
meta={"mode": flag, "published": published, "failed": failed},
|
||||
)
|
||||
return_code = await process.wait()
|
||||
if return_code != 0:
|
||||
raise RuntimeError(f"site publisher exited with code {return_code} for {flag}")
|
||||
return published, failed
|
||||
|
||||
async def run_once(self) -> bool:
|
||||
await self.reload_settings()
|
||||
worker_enabled = await is_worker_enabled(self.pool, WORKER_SITE_POSTER)
|
||||
app_enabled = await fetch_bool_setting("site_poster_enabled", False)
|
||||
if not worker_enabled or not app_enabled:
|
||||
await self.heartbeat.beat(self.pool, status="disabled", force=True)
|
||||
return False
|
||||
await self.heartbeat.beat(self.pool, status="checking", force=True)
|
||||
photo_published, photo_failed = await self.publish_mode("--all-ready", self.photo_batch_size)
|
||||
video_published, video_failed = await self.publish_mode("--all-ready-videos", self.video_batch_size)
|
||||
published = photo_published + video_published
|
||||
failed = photo_failed + video_failed
|
||||
await self.heartbeat.beat(
|
||||
self.pool,
|
||||
status="idle" if not failed else "error",
|
||||
meta={"published": published, "failed": failed},
|
||||
force=True,
|
||||
)
|
||||
return bool(published)
|
||||
|
||||
async def run_loop(self) -> None:
|
||||
await self.init()
|
||||
logger.info("{} started pid={}", WORKER_SITE_POSTER, os.getpid())
|
||||
while True:
|
||||
try:
|
||||
await self.run_once()
|
||||
except Exception as exc:
|
||||
logger.exception("Site poster loop error: {}", exc)
|
||||
await self.heartbeat.beat(self.pool, status="error", meta={"error": str(exc)[:500]}, force=True)
|
||||
await asyncio.sleep(self.interval_sec)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
logger.remove()
|
||||
logger.add(sys.stdout, level=settings.log_level)
|
||||
worker = SitePoster()
|
||||
await worker.run_loop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,637 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, time, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from aiogram import Bot
|
||||
from aiogram.exceptions import TelegramRetryAfter
|
||||
from aiogram.types import FSInputFile, InputMediaPhoto, InputMediaVideo
|
||||
from loguru import logger
|
||||
|
||||
from ..config import settings
|
||||
from ..constants import (
|
||||
PUBLICATION_STATUS_FAILED,
|
||||
PUBLICATION_STATUS_PENDING,
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
WORKER_TG_POSTER,
|
||||
)
|
||||
from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool
|
||||
from ..heartbeat import HeartbeatReporter
|
||||
from ..jobs import is_worker_enabled
|
||||
from ..text_utils import build_publication_text
|
||||
|
||||
LOCAL_TZ = ZoneInfo("Asia/Yekaterinburg")
|
||||
MAX_MEDIA_GROUP = 10
|
||||
MAX_RICH_MEDIA = 50
|
||||
MAX_RICH_TEXT = 32768
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
UPLOAD_ROOT = PROJECT_ROOT / "uploads"
|
||||
|
||||
|
||||
def parse_topic(value: str) -> tuple[int, int | None]:
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
raise RuntimeError("tg_poster_chat_id is empty")
|
||||
if ":" in raw:
|
||||
chat_id, thread_id = raw.split(":", 1)
|
||||
return int(chat_id), int(thread_id)
|
||||
return int(raw), None
|
||||
|
||||
|
||||
def valid_time(value: str) -> bool:
|
||||
return bool(re.fullmatch(r"(?:[01]\d|2[0-3]):[0-5]\d", str(value or "").strip()))
|
||||
|
||||
|
||||
def split_message_chunks(text: str, limit: int) -> list[str]:
|
||||
text = str(text or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
chunks: list[str] = []
|
||||
while len(text) > limit:
|
||||
split_at = text.rfind("\n", 0, limit)
|
||||
if split_at < limit // 2:
|
||||
split_at = text.rfind(" ", 0, limit)
|
||||
if split_at < limit // 2:
|
||||
split_at = limit
|
||||
chunks.append(text[:split_at].strip())
|
||||
text = text[split_at:].strip()
|
||||
if text:
|
||||
chunks.append(text)
|
||||
return chunks
|
||||
|
||||
|
||||
def tg_message_url(chat_id: int, message_id: int) -> str:
|
||||
chat = str(abs(int(chat_id)))
|
||||
if chat.startswith("100"):
|
||||
chat = chat[3:]
|
||||
return f"https://t.me/c/{chat}/{message_id}"
|
||||
|
||||
|
||||
def local_upload_path(url: str) -> Path | None:
|
||||
raw = str(url or "").strip()
|
||||
prefix = "/uploads/"
|
||||
if not raw.startswith(prefix):
|
||||
return None
|
||||
path = (UPLOAD_ROOT / raw.removeprefix(prefix)).resolve()
|
||||
try:
|
||||
path.relative_to(UPLOAD_ROOT.resolve())
|
||||
except ValueError:
|
||||
return None
|
||||
return path if path.is_file() else None
|
||||
|
||||
|
||||
class RichMessageUnavailable(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class TelegramPoster:
|
||||
def __init__(self) -> None:
|
||||
self.pool = None
|
||||
self.bot: Bot | None = None
|
||||
self.bot_token = ""
|
||||
self.chat_id: int | None = None
|
||||
self.thread_id: int | None = None
|
||||
self.heartbeat = HeartbeatReporter(WORKER_TG_POSTER, 30)
|
||||
self.schedule: list[dict[str, Any]] = []
|
||||
self.media_group_max_items = MAX_MEDIA_GROUP
|
||||
self.caption_limit = 1024
|
||||
self.message_limit = 4096
|
||||
self.max_attempts = 3
|
||||
self.retry_backoff_max_sec = 30
|
||||
self.send_delay_sec = 1.0
|
||||
self.overflow_caption = "⬇️ Описание"
|
||||
self.recent_window = 20
|
||||
self.category_repeat_penalty = 3.0
|
||||
self.source_repeat_penalty = 4.0
|
||||
|
||||
async def init(self) -> None:
|
||||
self.pool = await get_pool()
|
||||
await self.reload_settings()
|
||||
|
||||
async def reload_settings(self) -> None:
|
||||
token = str(await fetch_setting("tg_poster_bot_token", "") or "").strip() or settings.tg_bot_token
|
||||
if not token:
|
||||
raise RuntimeError("tg_poster_bot_token and TG_BOT_TOKEN are empty")
|
||||
if token != self.bot_token:
|
||||
if self.bot:
|
||||
await self.bot.session.close()
|
||||
self.bot = Bot(token=token)
|
||||
self.bot_token = token
|
||||
self.chat_id, self.thread_id = parse_topic(str(await fetch_setting("tg_poster_chat_id", "") or ""))
|
||||
self.schedule = self.normalize_schedule(await fetch_setting("tg_poster_schedule_json", []))
|
||||
self.media_group_max_items = max(1, min(MAX_MEDIA_GROUP, await fetch_int_setting("tg_poster_media_group_max_items", 10)))
|
||||
self.caption_limit = max(128, min(1024, await fetch_int_setting("tg_poster_caption_limit", 1024)))
|
||||
self.message_limit = max(512, min(4096, await fetch_int_setting("tg_poster_message_limit", 4096)))
|
||||
self.max_attempts = max(1, await fetch_int_setting("tg_poster_max_attempts", 3))
|
||||
self.retry_backoff_max_sec = max(1, await fetch_int_setting("tg_poster_retry_backoff_max_sec", 30))
|
||||
self.send_delay_sec = max(0.0, await fetch_float_setting("tg_poster_send_delay_sec", 1.0))
|
||||
self.overflow_caption = str(await fetch_setting("tg_poster_text_overflow_caption", self.overflow_caption) or self.overflow_caption)
|
||||
self.recent_window = max(1, await fetch_int_setting("tg_poster_recent_window", 20))
|
||||
self.category_repeat_penalty = max(0.0, await fetch_float_setting("tg_poster_category_repeat_penalty", 3.0))
|
||||
self.source_repeat_penalty = max(0.0, await fetch_float_setting("tg_poster_source_repeat_penalty", 4.0))
|
||||
logger.info("TG poster config: chat={} schedule={} caption_limit={}", self.chat_id, self.schedule, self.caption_limit)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.bot:
|
||||
await self.bot.session.close()
|
||||
|
||||
def normalize_schedule(self, value: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(value, list):
|
||||
value = []
|
||||
rows: list[dict[str, Any]] = []
|
||||
for idx, item in enumerate(value):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
time_value = str(item.get("time") or "").strip()
|
||||
if not valid_time(time_value):
|
||||
continue
|
||||
count = max(0, min(20, int(item.get("count") or 0)))
|
||||
if count < 1:
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"id": str(item.get("id") or f"slot-{idx + 1}").strip()[:80],
|
||||
"time": time_value,
|
||||
"count": count,
|
||||
"enabled": bool(item.get("enabled", True)),
|
||||
}
|
||||
)
|
||||
return sorted(rows, key=lambda row: row["time"])
|
||||
|
||||
def chat_kwargs(self) -> dict[str, Any]:
|
||||
kwargs: dict[str, Any] = {"chat_id": int(self.chat_id), "parse_mode": "HTML"}
|
||||
if self.thread_id:
|
||||
kwargs["message_thread_id"] = int(self.thread_id)
|
||||
return kwargs
|
||||
|
||||
async def tg_retry(self, fn):
|
||||
for attempt in range(1, self.max_attempts + 1):
|
||||
try:
|
||||
return await fn()
|
||||
except TelegramRetryAfter as exc:
|
||||
delay = float(exc.retry_after) + 0.5
|
||||
logger.warning("Telegram flood control, sleep {}s", delay)
|
||||
await asyncio.sleep(delay)
|
||||
except Exception as exc:
|
||||
logger.warning("Telegram post error without retry to avoid duplicate send: {}", exc)
|
||||
raise
|
||||
raise RuntimeError("telegram retry exhausted")
|
||||
|
||||
async def due_slots(self) -> list[dict[str, Any]]:
|
||||
now = datetime.now(LOCAL_TZ)
|
||||
today = now.date()
|
||||
due: list[dict[str, Any]] = []
|
||||
for row in self.schedule:
|
||||
if not row.get("enabled"):
|
||||
continue
|
||||
hour, minute = [int(part) for part in str(row["time"]).split(":", 1)]
|
||||
slot_start = datetime.combine(today, time(hour=hour, minute=minute), tzinfo=LOCAL_TZ)
|
||||
slot_end = slot_start + timedelta(minutes=10)
|
||||
if not (slot_start <= now < slot_end):
|
||||
continue
|
||||
inserted = await self.pool.fetchrow(
|
||||
"""
|
||||
INSERT INTO publication_runs(poster, schedule_id, scheduled_for, scheduled_time, planned_count, status)
|
||||
VALUES($1, $2, $3, $4, $5, 'started')
|
||||
ON CONFLICT (poster, schedule_id, scheduled_for) DO NOTHING
|
||||
RETURNING id
|
||||
""",
|
||||
"tg",
|
||||
row["id"],
|
||||
today,
|
||||
row["time"],
|
||||
int(row["count"]),
|
||||
)
|
||||
if inserted:
|
||||
due.append({**row, "run_id": int(inserted["id"]), "date": today})
|
||||
return due
|
||||
|
||||
async def load_recent(self) -> list[dict[str, Any]]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT final_category_tag, rewrite_category_tag, final_source_tag, rewrite_source_tag
|
||||
FROM raw_posts
|
||||
WHERE publication_status=$1
|
||||
AND tg_publication_chat_id=$2
|
||||
ORDER BY published_at DESC NULLS LAST, id DESC
|
||||
LIMIT $3::int
|
||||
""",
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
self.chat_id,
|
||||
self.recent_window,
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
async def load_candidates(self, limit: int) -> list[dict[str, Any]]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT rp.*, s.name AS source_name, s.tag AS source_tag
|
||||
FROM raw_posts rp
|
||||
JOIN sources s ON s.id=rp.source_id
|
||||
WHERE rp.status='storage_ready'
|
||||
AND rp.rewrite_status='ready'
|
||||
AND rp.editorial_status='accepted'
|
||||
AND COALESCE(rp.publication_status, 'pending') = $1
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media rpm
|
||||
WHERE rpm.raw_post_id=rp.id
|
||||
AND COALESCE(rpm.editor_hidden, FALSE)=FALSE
|
||||
AND rpm.media_type IN ('photo', 'video')
|
||||
AND COALESCE(rpm.tg_file_id, rpm.storage_attachment_id, '') <> ''
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media rpm
|
||||
WHERE rpm.raw_post_id=rp.id
|
||||
AND COALESCE(rpm.editor_hidden, FALSE)=FALSE
|
||||
AND rpm.media_type IN ('photo', 'video')
|
||||
AND COALESCE(rpm.tg_file_id, rpm.storage_attachment_id, '') = ''
|
||||
)
|
||||
ORDER BY COALESCE(rp.reviewed_at, rp.edited_at, rp.rewritten_at, rp.created_at) ASC,
|
||||
rp.id ASC
|
||||
LIMIT $2::int
|
||||
""",
|
||||
PUBLICATION_STATUS_PENDING,
|
||||
max(limit, 50),
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def choose_posts(self, candidates: list[dict[str, Any]], recent: list[dict[str, Any]], count: int) -> list[dict[str, Any]]:
|
||||
selected: list[dict[str, Any]] = []
|
||||
recent_categories = [
|
||||
str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "")
|
||||
for row in recent
|
||||
]
|
||||
recent_sources = [
|
||||
str(row.get("final_source_tag") or row.get("rewrite_source_tag") or "")
|
||||
for row in recent
|
||||
]
|
||||
pool = list(candidates)
|
||||
while pool and len(selected) < count:
|
||||
best_idx = 0
|
||||
selected_categories = [str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "") for row in selected]
|
||||
selected_sources = [str(row.get("final_source_tag") or row.get("rewrite_source_tag") or row.get("source_tag") or "") for row in selected]
|
||||
previous_category = selected_categories[-1] if selected_categories else (recent_categories[0] if recent_categories else "")
|
||||
previous_source = selected_sources[-1] if selected_sources else (recent_sources[0] if recent_sources else "")
|
||||
best_rank: tuple[int, float, int] | None = None
|
||||
for idx, post in enumerate(pool):
|
||||
category = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or "")
|
||||
source = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "")
|
||||
immediate_repeats = int(bool(category) and category == previous_category) + int(bool(source) and source == previous_source)
|
||||
repeat_penalty = (
|
||||
(recent_categories.count(category) + selected_categories.count(category)) * self.category_repeat_penalty
|
||||
+ (recent_sources.count(source) + selected_sources.count(source)) * self.source_repeat_penalty
|
||||
)
|
||||
rank = (immediate_repeats, repeat_penalty, idx)
|
||||
if best_rank is None or rank < best_rank:
|
||||
best_idx = idx
|
||||
best_rank = rank
|
||||
selected.append(pool.pop(best_idx))
|
||||
return selected
|
||||
|
||||
async def load_media(self, raw_post_id: int) -> list[dict[str, Any]]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT *
|
||||
FROM raw_post_media
|
||||
WHERE raw_post_id=$1
|
||||
AND COALESCE(editor_hidden, FALSE)=FALSE
|
||||
AND media_type IN ('photo', 'video')
|
||||
ORDER BY sort_order ASC, id ASC
|
||||
""",
|
||||
raw_post_id,
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def media_value(self, item: dict[str, Any]) -> Any | None:
|
||||
stored = item.get("tg_file_id") or item.get("storage_attachment_id")
|
||||
if stored:
|
||||
return str(stored)
|
||||
url = str(item.get("original_url") or "").strip()
|
||||
path = local_upload_path(url)
|
||||
if path:
|
||||
return FSInputFile(path)
|
||||
return None
|
||||
|
||||
def build_text(self, post: dict[str, Any]) -> str:
|
||||
category_tag = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or post.get("final_category") or post.get("rewrite_category") or "")
|
||||
source_tag = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "")
|
||||
return build_publication_text(post.get("final_text") or post.get("rewritten_text") or "", category_tag, source_tag, format_title=True, parse_mode="html")
|
||||
|
||||
async def send_text(self, text: str) -> list[int]:
|
||||
message_ids: list[int] = []
|
||||
for chunk in split_message_chunks(text, self.message_limit):
|
||||
message = await self.tg_retry(lambda text=chunk: self.bot.send_message(text=text, **self.chat_kwargs()))
|
||||
message_ids.append(int(message.message_id))
|
||||
if self.send_delay_sec:
|
||||
await asyncio.sleep(self.send_delay_sec)
|
||||
return message_ids
|
||||
|
||||
async def send_media_items(self, items: list[dict[str, Any]], caption: str | None = None) -> list[int]:
|
||||
if not items:
|
||||
return []
|
||||
if len(items) == 1:
|
||||
item = items[0]
|
||||
if item["type"] == "photo":
|
||||
message = await self.tg_retry(
|
||||
lambda: self.bot.send_photo(photo=item["media"], caption=caption, **self.chat_kwargs())
|
||||
)
|
||||
else:
|
||||
message = await self.tg_retry(
|
||||
lambda: self.bot.send_video(video=item["media"], caption=caption, **self.chat_kwargs())
|
||||
)
|
||||
return [int(message.message_id)]
|
||||
group = []
|
||||
for idx, item in enumerate(items):
|
||||
media_caption = caption if idx == 0 else None
|
||||
if item["type"] == "photo":
|
||||
group.append(InputMediaPhoto(media=item["media"], caption=media_caption))
|
||||
else:
|
||||
group.append(InputMediaVideo(media=item["media"], caption=media_caption))
|
||||
messages = await self.tg_retry(lambda: self.bot.send_media_group(media=group, **self.chat_kwargs()))
|
||||
return [int(message.message_id) for message in messages]
|
||||
|
||||
def rich_media_value(self, item: dict[str, Any]) -> str | None:
|
||||
stored = str(item.get("tg_file_id") or item.get("storage_attachment_id") or "").strip()
|
||||
return stored or None
|
||||
|
||||
def rich_text_html(self, text: str) -> str:
|
||||
paragraphs = []
|
||||
for paragraph in str(text or "").strip().split("\n\n"):
|
||||
body = "<br/>".join(line for line in paragraph.splitlines() if line.strip())
|
||||
if body:
|
||||
paragraphs.append(f"<p>{body}</p>")
|
||||
return "\n".join(paragraphs)
|
||||
|
||||
def build_rich_message(self, text: str, media_rows: list[dict[str, Any]]) -> dict[str, Any] | None:
|
||||
if len(media_rows) > MAX_RICH_MEDIA:
|
||||
return None
|
||||
|
||||
rich_media = []
|
||||
media_tags = []
|
||||
for idx, item in enumerate(media_rows[:MAX_RICH_MEDIA]):
|
||||
media_type = str(item.get("media_type") or "")
|
||||
if media_type not in ("photo", "video"):
|
||||
continue
|
||||
media = self.rich_media_value(item)
|
||||
if not media:
|
||||
return None
|
||||
media_id = f"m{idx}"
|
||||
input_type = "photo" if media_type == "photo" else "video"
|
||||
rich_media.append({"id": media_id, "media": {"type": input_type, "media": media}})
|
||||
if media_type == "photo":
|
||||
media_tags.append(f'<img src="tg://photo?id={media_id}"/>')
|
||||
else:
|
||||
media_tags.append(f'<video src="tg://video?id={media_id}"></video>')
|
||||
if not rich_media:
|
||||
return None
|
||||
|
||||
rich_text = self.rich_text_html(text)
|
||||
if len(rich_text) > MAX_RICH_TEXT:
|
||||
return None
|
||||
|
||||
media_html = media_tags[0] if len(media_tags) == 1 else f"<tg-collage>{''.join(media_tags)}</tg-collage>"
|
||||
return {
|
||||
"html": f"{media_html}\n{rich_text}" if rich_text else media_html,
|
||||
"media": rich_media,
|
||||
}
|
||||
|
||||
async def send_rich_message(self, rich_message: dict[str, Any]) -> list[int]:
|
||||
data: dict[str, Any] = {
|
||||
"chat_id": int(self.chat_id),
|
||||
"rich_message": rich_message,
|
||||
}
|
||||
if self.thread_id:
|
||||
data["message_thread_id"] = int(self.thread_id)
|
||||
|
||||
url = f"https://api.telegram.org/bot{self.bot_token}/sendRichMessage"
|
||||
timeout = aiohttp.ClientTimeout(total=90)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
async with session.post(url, json=data) as response:
|
||||
payload = await response.json(content_type=None)
|
||||
|
||||
if payload.get("ok"):
|
||||
message_id = payload.get("result", {}).get("message_id")
|
||||
if message_id:
|
||||
return [int(message_id)]
|
||||
raise RichMessageUnavailable("sendRichMessage returned no message_id")
|
||||
|
||||
description = str(payload.get("description") or f"HTTP {response.status}")
|
||||
if "Too Many Requests" in description and isinstance(payload.get("parameters"), dict):
|
||||
retry_after = float(payload["parameters"].get("retry_after") or 0)
|
||||
if retry_after > 0:
|
||||
logger.warning("Telegram rich message flood control, sleep {}s", retry_after)
|
||||
await asyncio.sleep(retry_after + 0.5)
|
||||
raise RichMessageUnavailable(description)
|
||||
|
||||
async def send_legacy_media_post(self, text: str, media_rows: list[dict[str, Any]]) -> list[int]:
|
||||
media_items = []
|
||||
for item in media_rows:
|
||||
media = self.media_value(item)
|
||||
if media is None:
|
||||
continue
|
||||
media_type = str(item.get("media_type") or "")
|
||||
if media_type == "photo":
|
||||
media_items.append({"type": "photo", "media": media})
|
||||
elif media_type == "video":
|
||||
media_items.append({"type": "video", "media": media})
|
||||
|
||||
if not media_items:
|
||||
return await self.send_text(text)
|
||||
|
||||
if len(text) <= self.caption_limit:
|
||||
first_caption = text
|
||||
text_chunks: list[str] = []
|
||||
else:
|
||||
first_caption = self.overflow_caption[: self.caption_limit]
|
||||
text_chunks = split_message_chunks(text, self.message_limit)
|
||||
|
||||
first = media_items[: self.media_group_max_items]
|
||||
message_ids = []
|
||||
message_ids.extend(await self.send_media_items(first, first_caption))
|
||||
if self.send_delay_sec:
|
||||
await asyncio.sleep(self.send_delay_sec)
|
||||
|
||||
rest = media_items[self.media_group_max_items :]
|
||||
for start in range(0, len(rest), self.media_group_max_items):
|
||||
chunk = rest[start : start + self.media_group_max_items]
|
||||
message_ids.extend(await self.send_media_items(chunk))
|
||||
if self.send_delay_sec:
|
||||
await asyncio.sleep(self.send_delay_sec)
|
||||
|
||||
for chunk in text_chunks:
|
||||
message_ids.extend(await self.send_text(chunk))
|
||||
|
||||
return message_ids
|
||||
|
||||
async def send_post(self, post: dict[str, Any]) -> list[int]:
|
||||
text = self.build_text(post)
|
||||
media_rows = await self.load_media(int(post["id"]))
|
||||
|
||||
if not media_rows:
|
||||
return await self.send_text(text)
|
||||
|
||||
rich_message = self.build_rich_message(text, media_rows)
|
||||
if rich_message:
|
||||
try:
|
||||
return await self.send_rich_message(rich_message)
|
||||
except RichMessageUnavailable as exc:
|
||||
logger.warning("Telegram rich message failed, fallback to legacy media post: {}", exc)
|
||||
|
||||
return await self.send_legacy_media_post(text, media_rows)
|
||||
|
||||
async def mark_published(self, post_id: int, message_ids: list[int]) -> None:
|
||||
async with self.pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE raw_posts
|
||||
SET publication_status=$2,
|
||||
editorial_status='published',
|
||||
publication_error=NULL,
|
||||
published_at=NOW(),
|
||||
tg_publication_chat_id=$3,
|
||||
tg_publication_thread_id=$4,
|
||||
tg_publication_message_ids=$5,
|
||||
tg_publication_url=$6,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
post_id,
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
self.chat_id,
|
||||
self.thread_id,
|
||||
message_ids,
|
||||
tg_message_url(int(self.chat_id), message_ids[0]) if message_ids else None,
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO post_publications(raw_post_id, platform, status, target_id, external_id, url, attachments_json, published_at, updated_at)
|
||||
VALUES($1, 'tg', $2, $3, $4, $5, $6::jsonb, NOW(), NOW())
|
||||
ON CONFLICT (raw_post_id, platform) DO UPDATE
|
||||
SET status=EXCLUDED.status,
|
||||
target_id=EXCLUDED.target_id,
|
||||
external_id=EXCLUDED.external_id,
|
||||
url=EXCLUDED.url,
|
||||
error=NULL,
|
||||
attachments_json=EXCLUDED.attachments_json,
|
||||
published_at=NOW(),
|
||||
updated_at=NOW()
|
||||
""",
|
||||
post_id,
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
str(self.chat_id),
|
||||
",".join(str(mid) for mid in message_ids),
|
||||
tg_message_url(int(self.chat_id), message_ids[0]) if message_ids else None,
|
||||
json.dumps(message_ids, ensure_ascii=False),
|
||||
)
|
||||
|
||||
async def mark_failed(self, post_id: int, error: str) -> None:
|
||||
async with self.pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE raw_posts
|
||||
SET publication_status=$2,
|
||||
editorial_status='publish_failed',
|
||||
publication_error=$3,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
post_id,
|
||||
PUBLICATION_STATUS_FAILED,
|
||||
error[:1000],
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO post_publications(raw_post_id, platform, status, target_id, error, updated_at)
|
||||
VALUES($1, 'tg', $2, $3, $4, NOW())
|
||||
ON CONFLICT (raw_post_id, platform) DO UPDATE
|
||||
SET status=EXCLUDED.status,
|
||||
target_id=EXCLUDED.target_id,
|
||||
error=EXCLUDED.error,
|
||||
updated_at=NOW()
|
||||
""",
|
||||
post_id,
|
||||
PUBLICATION_STATUS_FAILED,
|
||||
str(self.chat_id),
|
||||
error[:1000],
|
||||
)
|
||||
async def process_slot(self, slot: dict[str, Any]) -> int:
|
||||
count = int(slot["count"])
|
||||
recent = await self.load_recent()
|
||||
candidates = await self.load_candidates(max(50, count * 10))
|
||||
posts = self.choose_posts(candidates, recent, count)
|
||||
published = 0
|
||||
for post in posts:
|
||||
await self.heartbeat.beat(self.pool, status="publishing", meta={"post_id": int(post["id"]), "slot": slot["id"]}, force=True)
|
||||
try:
|
||||
message_ids = await self.send_post(post)
|
||||
await self.mark_published(int(post["id"]), message_ids)
|
||||
published += 1
|
||||
logger.info("TG published raw_post={} messages={}", post["id"], message_ids)
|
||||
except Exception as exc:
|
||||
await self.mark_failed(int(post["id"]), str(exc))
|
||||
logger.exception("TG publish failed raw_post={}: {}", post["id"], exc)
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE publication_runs
|
||||
SET published_count=$2,
|
||||
status=$3,
|
||||
error=$4,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
int(slot["run_id"]),
|
||||
published,
|
||||
"done" if published == count else "partial",
|
||||
None if published == count else f"published {published} of {count}",
|
||||
)
|
||||
return published
|
||||
|
||||
async def run_once(self) -> bool:
|
||||
await self.reload_settings()
|
||||
enabled = await is_worker_enabled(self.pool, WORKER_TG_POSTER)
|
||||
app_enabled = await fetch_bool_setting("tg_poster_enabled", False)
|
||||
if not enabled or not app_enabled:
|
||||
await self.heartbeat.beat(self.pool, status="disabled", force=True)
|
||||
return False
|
||||
slots = await self.due_slots()
|
||||
if not slots:
|
||||
await self.heartbeat.beat(self.pool, status="idle")
|
||||
return False
|
||||
total = 0
|
||||
for slot in slots:
|
||||
total += await self.process_slot(slot)
|
||||
await self.heartbeat.beat(self.pool, status="idle", meta={"published": total}, force=True)
|
||||
return bool(total)
|
||||
|
||||
async def run_loop(self) -> None:
|
||||
await self.init()
|
||||
logger.info("{} started pid={}", WORKER_TG_POSTER, os.getpid())
|
||||
try:
|
||||
while True:
|
||||
await self.run_once()
|
||||
await asyncio.sleep(max(10, await fetch_int_setting("tg_poster_interval_sec", 60)))
|
||||
finally:
|
||||
await self.close()
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
logger.remove()
|
||||
logger.add(sys.stdout, level=settings.log_level)
|
||||
worker = TelegramPoster()
|
||||
await worker.run_loop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,326 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
from loguru import logger
|
||||
|
||||
from ..config import settings
|
||||
from ..constants import PUBLICATION_STATUS_PUBLISHED, WORKER_TG_REACTOR
|
||||
from ..db import fetch_bool_setting, fetch_int_setting, fetch_setting, get_pool
|
||||
from ..heartbeat import HeartbeatReporter
|
||||
from ..jobs import is_worker_enabled
|
||||
|
||||
PLATFORM_TG = "tg"
|
||||
REACTION_STATUS_PENDING = "pending"
|
||||
REACTION_STATUS_IN_PROGRESS = "in_progress"
|
||||
REACTION_STATUS_REACTED = "reacted"
|
||||
REACTION_STATUS_FAILED = "failed"
|
||||
|
||||
|
||||
def split_tokens(value: Any) -> list[str]:
|
||||
if isinstance(value, list):
|
||||
candidates = [str(item or "") for item in value]
|
||||
else:
|
||||
candidates = re.split(r"[\s,;]+", str(value or ""))
|
||||
return [token.strip() for token in candidates if token.strip()]
|
||||
|
||||
|
||||
def normalize_reactions(value: Any) -> list[str]:
|
||||
if not isinstance(value, list):
|
||||
value = []
|
||||
reactions = []
|
||||
for item in value:
|
||||
emoji = str(item or "").strip()
|
||||
if emoji:
|
||||
reactions.append(emoji)
|
||||
return reactions or ["👍"]
|
||||
|
||||
|
||||
def parse_message_ids(row: dict[str, Any]) -> list[str]:
|
||||
raw = row.get("attachments_json")
|
||||
values: list[Any] = []
|
||||
if isinstance(raw, list):
|
||||
values = raw
|
||||
elif isinstance(raw, str):
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
if isinstance(parsed, list):
|
||||
values = parsed
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
if not values and row.get("external_id"):
|
||||
values = str(row["external_id"]).split(",")
|
||||
message_ids = []
|
||||
for value in values:
|
||||
text = str(value or "").strip()
|
||||
if text:
|
||||
message_ids.append(text)
|
||||
return message_ids
|
||||
|
||||
|
||||
class TelegramReactor:
|
||||
def __init__(self) -> None:
|
||||
self.pool = None
|
||||
self.heartbeat = HeartbeatReporter(WORKER_TG_REACTOR, 30)
|
||||
self.bot_tokens: list[str] = []
|
||||
self.reactions: list[str] = ["👍"]
|
||||
self.delay_sec = 60
|
||||
self.interval_sec = 60
|
||||
self.reaction_pause_sec = 2
|
||||
self.react_all_messages = False
|
||||
self.max_attempts = 3
|
||||
self.retry_backoff_max_sec = 30
|
||||
self.since = ""
|
||||
|
||||
async def init(self) -> None:
|
||||
self.pool = await get_pool()
|
||||
await self.reload_settings()
|
||||
|
||||
async def reload_settings(self) -> None:
|
||||
tokens = split_tokens(await fetch_setting("tg_reactor_bot_tokens", ""))
|
||||
if not tokens:
|
||||
fallback = str(await fetch_setting("tg_poster_bot_token", "") or "").strip() or settings.tg_bot_token
|
||||
tokens = split_tokens(fallback)
|
||||
if not tokens:
|
||||
raise RuntimeError("tg_reactor_bot_tokens, tg_poster_bot_token and TG_BOT_TOKEN are empty")
|
||||
self.bot_tokens = tokens
|
||||
self.reactions = normalize_reactions(await fetch_setting("tg_reactor_reactions_json", ["👍"]))
|
||||
self.since = str(await fetch_setting("tg_reactor_since", "") or "").strip()
|
||||
self.delay_sec = max(0, await fetch_int_setting("tg_reactor_delay_sec", 60))
|
||||
self.interval_sec = max(10, await fetch_int_setting("tg_reactor_interval_sec", 60))
|
||||
self.reaction_pause_sec = max(0, await fetch_int_setting("tg_reactor_reaction_pause_sec", 2))
|
||||
self.react_all_messages = await fetch_bool_setting("tg_reactor_react_all_messages", False)
|
||||
self.max_attempts = max(1, await fetch_int_setting("tg_reactor_max_attempts", 3))
|
||||
self.retry_backoff_max_sec = max(1, await fetch_int_setting("tg_reactor_retry_backoff_max_sec", 30))
|
||||
logger.info(
|
||||
"TG reactor config: bots={} reactions={} since={} delay={}s pause={}s all_messages={}",
|
||||
len(self.bot_tokens),
|
||||
self.reactions,
|
||||
self.since or "-",
|
||||
self.delay_sec,
|
||||
self.reaction_pause_sec,
|
||||
self.react_all_messages,
|
||||
)
|
||||
|
||||
async def load_publications(self, limit: int = 50) -> list[dict[str, Any]]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT pp.*
|
||||
FROM post_publications pp
|
||||
WHERE pp.platform=$1
|
||||
AND pp.status=$2
|
||||
AND pp.published_at IS NOT NULL
|
||||
AND pp.published_at <= NOW() - ($3::int * INTERVAL '1 second')
|
||||
AND ($4::text = '' OR pp.published_at >= $4::timestamptz)
|
||||
ORDER BY pp.published_at DESC, pp.id DESC
|
||||
LIMIT $5::int
|
||||
""",
|
||||
PLATFORM_TG,
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
self.delay_sec,
|
||||
self.since,
|
||||
limit,
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
async def ensure_reaction_row(
|
||||
self,
|
||||
publication: dict[str, Any],
|
||||
reactor_key: str,
|
||||
reaction: str,
|
||||
message_id: str,
|
||||
) -> dict[str, Any] | None:
|
||||
row = await self.pool.fetchrow(
|
||||
"""
|
||||
INSERT INTO post_reactions(
|
||||
publication_id, raw_post_id, platform, reactor_key, reaction, target_id, message_id, status, updated_at
|
||||
)
|
||||
VALUES($1, $2, $3, $4, $5, $6, $7, $8, NOW())
|
||||
ON CONFLICT (publication_id, reactor_key, message_id) DO NOTHING
|
||||
RETURNING *
|
||||
""",
|
||||
int(publication["id"]),
|
||||
int(publication["raw_post_id"]),
|
||||
PLATFORM_TG,
|
||||
reactor_key,
|
||||
reaction,
|
||||
str(publication["target_id"]),
|
||||
message_id,
|
||||
REACTION_STATUS_PENDING,
|
||||
)
|
||||
if row:
|
||||
return dict(row)
|
||||
row = await self.pool.fetchrow(
|
||||
"""
|
||||
SELECT *
|
||||
FROM post_reactions
|
||||
WHERE publication_id=$1 AND reactor_key=$2 AND message_id=$3
|
||||
""",
|
||||
int(publication["id"]),
|
||||
reactor_key,
|
||||
message_id,
|
||||
)
|
||||
if not row or row["status"] == REACTION_STATUS_REACTED or int(row["attempts"] or 0) >= self.max_attempts:
|
||||
return None
|
||||
return dict(row)
|
||||
|
||||
async def claim_reaction(self, reaction_id: int) -> dict[str, Any] | None:
|
||||
row = await self.pool.fetchrow(
|
||||
"""
|
||||
UPDATE post_reactions
|
||||
SET status=$2,
|
||||
attempts=attempts + 1,
|
||||
error=NULL,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
AND status IN ($3, $4)
|
||||
AND attempts < $5
|
||||
RETURNING *
|
||||
""",
|
||||
reaction_id,
|
||||
REACTION_STATUS_IN_PROGRESS,
|
||||
REACTION_STATUS_PENDING,
|
||||
REACTION_STATUS_FAILED,
|
||||
self.max_attempts,
|
||||
)
|
||||
return dict(row) if row else None
|
||||
|
||||
async def set_reaction(self, token: str, chat_id: str, message_id: str, reaction: str) -> None:
|
||||
url = f"https://api.telegram.org/bot{token}/setMessageReaction"
|
||||
payload = {
|
||||
"chat_id": int(chat_id),
|
||||
"message_id": int(message_id),
|
||||
"reaction": [{"type": "emoji", "emoji": reaction}],
|
||||
"is_big": False,
|
||||
}
|
||||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=30)) as session:
|
||||
async with session.post(url, json=payload) as resp:
|
||||
data = await resp.json(content_type=None)
|
||||
if not data.get("ok"):
|
||||
description = data.get("description") or data
|
||||
retry_after = ((data.get("parameters") or {}).get("retry_after")) if isinstance(data, dict) else None
|
||||
if retry_after:
|
||||
raise RuntimeError(f"Telegram retry_after={retry_after}: {description}")
|
||||
raise RuntimeError(str(description))
|
||||
|
||||
async def mark_reacted(self, reaction_id: int) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE post_reactions
|
||||
SET status=$2,
|
||||
error=NULL,
|
||||
reacted_at=NOW(),
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
reaction_id,
|
||||
REACTION_STATUS_REACTED,
|
||||
)
|
||||
|
||||
async def mark_failed(self, reaction_id: int, error: str) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE post_reactions
|
||||
SET status=$2,
|
||||
error=$3,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
reaction_id,
|
||||
REACTION_STATUS_FAILED,
|
||||
error[:1000],
|
||||
)
|
||||
|
||||
async def process_publication(self, publication: dict[str, Any]) -> int:
|
||||
message_ids = parse_message_ids(publication)
|
||||
if not message_ids:
|
||||
return 0
|
||||
if not self.react_all_messages:
|
||||
message_ids = message_ids[:1]
|
||||
|
||||
done = 0
|
||||
for message_id in message_ids:
|
||||
for idx, token in enumerate(self.bot_tokens):
|
||||
reaction = self.reactions[idx % len(self.reactions)]
|
||||
reactor_key = f"bot-{idx + 1}:{reaction}"
|
||||
row = await self.ensure_reaction_row(publication, reactor_key, reaction, message_id)
|
||||
if not row:
|
||||
continue
|
||||
claimed = await self.claim_reaction(int(row["id"]))
|
||||
if not claimed:
|
||||
continue
|
||||
try:
|
||||
await self.set_reaction(token, str(publication["target_id"]), message_id, reaction)
|
||||
await self.mark_reacted(int(claimed["id"]))
|
||||
done += 1
|
||||
logger.info(
|
||||
"TG reacted raw_post={} publication={} message={} reactor={}",
|
||||
publication["raw_post_id"],
|
||||
publication["id"],
|
||||
message_id,
|
||||
reactor_key,
|
||||
)
|
||||
if self.reaction_pause_sec:
|
||||
await asyncio.sleep(self.reaction_pause_sec)
|
||||
except Exception as exc:
|
||||
await self.mark_failed(int(claimed["id"]), str(exc))
|
||||
logger.warning(
|
||||
"TG reaction failed raw_post={} publication={} message={} reactor={}: {}",
|
||||
publication["raw_post_id"],
|
||||
publication["id"],
|
||||
message_id,
|
||||
reactor_key,
|
||||
exc,
|
||||
)
|
||||
await asyncio.sleep(min(2 ** int(claimed["attempts"]), self.retry_backoff_max_sec))
|
||||
return done
|
||||
|
||||
async def run_once(self) -> bool:
|
||||
await self.reload_settings()
|
||||
enabled = await is_worker_enabled(self.pool, WORKER_TG_REACTOR)
|
||||
app_enabled = await fetch_bool_setting("tg_reactor_enabled", False)
|
||||
if not enabled or not app_enabled:
|
||||
await self.heartbeat.beat(self.pool, status="disabled", force=True)
|
||||
return False
|
||||
publications = await self.load_publications()
|
||||
if not publications:
|
||||
await self.heartbeat.beat(self.pool, status="idle")
|
||||
return False
|
||||
total = 0
|
||||
for publication in publications:
|
||||
await self.heartbeat.beat(
|
||||
self.pool,
|
||||
status="reacting",
|
||||
meta={"publication_id": int(publication["id"]), "raw_post_id": int(publication["raw_post_id"])},
|
||||
force=True,
|
||||
)
|
||||
total += await self.process_publication(publication)
|
||||
await self.heartbeat.beat(self.pool, status="idle", meta={"reactions": total}, force=True)
|
||||
return bool(total)
|
||||
|
||||
async def run_loop(self) -> None:
|
||||
await self.init()
|
||||
logger.info("{} started pid={}", WORKER_TG_REACTOR, os.getpid())
|
||||
while True:
|
||||
try:
|
||||
await self.run_once()
|
||||
except Exception as exc:
|
||||
logger.exception("TG reactor loop error: {}", exc)
|
||||
await asyncio.sleep(self.interval_sec)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
logger.remove()
|
||||
logger.add(sys.stdout, level=settings.log_level)
|
||||
worker = TelegramReactor()
|
||||
await worker.run_loop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,442 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, time, timedelta, timezone
|
||||
from typing import Any
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import aiohttp
|
||||
from loguru import logger
|
||||
|
||||
from ..config import settings
|
||||
from ..constants import (
|
||||
PUBLICATION_STATUS_FAILED,
|
||||
PUBLICATION_STATUS_PENDING,
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
WORKER_VK_POSTER,
|
||||
)
|
||||
from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool
|
||||
from ..heartbeat import HeartbeatReporter
|
||||
from ..jobs import is_worker_enabled
|
||||
from ..text_utils import build_publication_text
|
||||
from ..vk_api import VKAPIClient, post_vk_url
|
||||
|
||||
LOCAL_TZ = ZoneInfo("Asia/Yekaterinburg")
|
||||
VK_MESSAGE_LIMIT = 16384
|
||||
VK_MAX_ATTACHMENTS = 10
|
||||
|
||||
|
||||
def valid_time(value: str) -> bool:
|
||||
return bool(re.fullmatch(r"(?:[01]\d|2[0-3]):[0-5]\d", str(value or "").strip()))
|
||||
|
||||
|
||||
def normalize_schedule(value: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(value, list):
|
||||
value = []
|
||||
rows: list[dict[str, Any]] = []
|
||||
for idx, item in enumerate(value):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
time_value = str(item.get("time") or "").strip()
|
||||
if not valid_time(time_value):
|
||||
continue
|
||||
count = max(0, min(20, int(item.get("count") or 0)))
|
||||
if count < 1:
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"id": str(item.get("id") or f"slot-{idx + 1}").strip()[:80],
|
||||
"time": time_value,
|
||||
"count": count,
|
||||
"enabled": bool(item.get("enabled", True)),
|
||||
}
|
||||
)
|
||||
return sorted(rows, key=lambda row: row["time"])
|
||||
|
||||
|
||||
class VKPoster:
|
||||
def __init__(self) -> None:
|
||||
self.pool = None
|
||||
self.heartbeat = HeartbeatReporter(WORKER_VK_POSTER, 30)
|
||||
self.token = ""
|
||||
self.owner_id = 0
|
||||
self.from_group = True
|
||||
self.schedule: list[dict[str, Any]] = []
|
||||
self.interval_sec = 60
|
||||
self.send_delay_sec = 1.0
|
||||
self.recent_window = 20
|
||||
self.category_repeat_penalty = 3.0
|
||||
self.source_repeat_penalty = 4.0
|
||||
self.dry_run = False
|
||||
|
||||
async def init(self) -> None:
|
||||
self.pool = await get_pool()
|
||||
|
||||
def token_expires_soon(self, value: Any) -> bool:
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
return False
|
||||
try:
|
||||
expires_at = datetime.fromisoformat(raw)
|
||||
if expires_at.tzinfo is None:
|
||||
expires_at = expires_at.replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
return False
|
||||
return expires_at <= datetime.now(timezone.utc) + timedelta(minutes=15)
|
||||
|
||||
async def save_setting(self, key: str, value: Any) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE app_settings
|
||||
SET value_json=$2::jsonb,
|
||||
updated_at=NOW()
|
||||
WHERE key=$1
|
||||
""",
|
||||
key,
|
||||
json.dumps(value, ensure_ascii=False),
|
||||
)
|
||||
|
||||
async def refresh_access_token_if_needed(self) -> None:
|
||||
expires_at = await fetch_setting("vk_poster_token_expires_at", "")
|
||||
if not self.token_expires_soon(expires_at):
|
||||
return
|
||||
refresh_token = str(await fetch_setting("vk_poster_refresh_token", "") or "").strip()
|
||||
client_id = str(await fetch_setting("vk_poster_app_id", "") or "").strip()
|
||||
client_secret = str(await fetch_setting("vk_poster_client_secret", "") or "").strip()
|
||||
device_id = str(await fetch_setting("vk_poster_token_device_id", "") or "").strip()
|
||||
if not refresh_token or not client_id:
|
||||
return
|
||||
data = {
|
||||
"grant_type": "refresh_token",
|
||||
"refresh_token": refresh_token,
|
||||
"client_id": client_id,
|
||||
"device_id": device_id,
|
||||
"state": "vk-poster-refresh",
|
||||
}
|
||||
if client_secret:
|
||||
data["client_secret"] = client_secret
|
||||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=30)) as session:
|
||||
async with session.post("https://id.vk.ru/oauth2/auth", data=data) as resp:
|
||||
payload = await resp.json(content_type=None)
|
||||
if payload.get("error"):
|
||||
raise RuntimeError(f"VK token refresh failed: {payload.get('error')}: {payload.get('error_description')}")
|
||||
access_token = str(payload.get("access_token") or "").strip()
|
||||
if not access_token:
|
||||
raise RuntimeError(f"VK token refresh did not return access_token: {payload}")
|
||||
new_refresh = str(payload.get("refresh_token") or "").strip() or refresh_token
|
||||
new_device_id = str(payload.get("device_id") or "").strip() or device_id
|
||||
expires_in = payload.get("expires_in")
|
||||
expires_at_value = ""
|
||||
try:
|
||||
expires_at_value = (datetime.now(timezone.utc) + timedelta(seconds=int(expires_in))).isoformat()
|
||||
except Exception:
|
||||
pass
|
||||
await self.save_setting("vk_poster_access_token", access_token)
|
||||
await self.save_setting("vk_poster_refresh_token", new_refresh)
|
||||
await self.save_setting("vk_poster_token_device_id", new_device_id)
|
||||
await self.save_setting("vk_poster_token_expires_at", expires_at_value)
|
||||
self.token = access_token
|
||||
logger.info("VK access token refreshed; expires_at={}", expires_at_value)
|
||||
|
||||
async def reload_settings(self) -> None:
|
||||
self.token = (
|
||||
str(await fetch_setting("vk_poster_access_token", "") or "").strip()
|
||||
or settings.vk_group_access_token
|
||||
or settings.vk_access_token
|
||||
)
|
||||
await self.refresh_access_token_if_needed()
|
||||
self.owner_id = int(await fetch_int_setting("vk_poster_owner_id", settings.vk_storage_owner_id if settings.vk_storage_group_id else 0))
|
||||
self.from_group = await fetch_bool_setting("vk_poster_from_group", True)
|
||||
self.schedule = normalize_schedule(await fetch_setting("vk_poster_schedule_json", []))
|
||||
self.interval_sec = max(10, await fetch_int_setting("vk_poster_interval_sec", 60))
|
||||
self.send_delay_sec = max(0.0, await fetch_float_setting("vk_poster_send_delay_sec", 1.0))
|
||||
self.recent_window = max(1, await fetch_int_setting("vk_poster_recent_window", 20))
|
||||
self.category_repeat_penalty = max(0.0, await fetch_float_setting("vk_poster_category_repeat_penalty", 3.0))
|
||||
self.source_repeat_penalty = max(0.0, await fetch_float_setting("vk_poster_source_repeat_penalty", 4.0))
|
||||
self.dry_run = await fetch_bool_setting("vk_poster_dry_run", False)
|
||||
if not self.token:
|
||||
raise RuntimeError("vk_poster_access_token, VK_GROUP_ACCESS_TOKEN and VK_ACCESS_TOKEN are empty")
|
||||
if not self.owner_id:
|
||||
raise RuntimeError("vk_poster_owner_id is empty")
|
||||
logger.info("VK poster config: owner={} schedule={} dry_run={}", self.owner_id, self.schedule, self.dry_run)
|
||||
|
||||
async def due_slots(self) -> list[dict[str, Any]]:
|
||||
now = datetime.now(LOCAL_TZ)
|
||||
today = now.date()
|
||||
due: list[dict[str, Any]] = []
|
||||
for row in self.schedule:
|
||||
if not row.get("enabled"):
|
||||
continue
|
||||
hour, minute = [int(part) for part in str(row["time"]).split(":", 1)]
|
||||
slot_start = datetime.combine(today, time(hour=hour, minute=minute), tzinfo=LOCAL_TZ)
|
||||
slot_end = slot_start + timedelta(minutes=10)
|
||||
if not (slot_start <= now < slot_end):
|
||||
continue
|
||||
inserted = await self.pool.fetchrow(
|
||||
"""
|
||||
INSERT INTO publication_runs(poster, schedule_id, scheduled_for, scheduled_time, planned_count, status)
|
||||
VALUES($1, $2, $3, $4, $5, 'started')
|
||||
ON CONFLICT (poster, schedule_id, scheduled_for) DO NOTHING
|
||||
RETURNING id
|
||||
""",
|
||||
"vk",
|
||||
row["id"],
|
||||
today,
|
||||
row["time"],
|
||||
int(row["count"]),
|
||||
)
|
||||
if inserted:
|
||||
due.append({**row, "run_id": int(inserted["id"]), "date": today})
|
||||
return due
|
||||
|
||||
async def load_recent(self) -> list[dict[str, Any]]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT rp.final_category_tag, rp.rewrite_category_tag, rp.final_source_tag, rp.rewrite_source_tag
|
||||
FROM post_publications pp
|
||||
JOIN raw_posts rp ON rp.id=pp.raw_post_id
|
||||
WHERE pp.platform='vk'
|
||||
AND pp.status=$1
|
||||
AND pp.target_id=$2
|
||||
ORDER BY pp.published_at DESC NULLS LAST, pp.id DESC
|
||||
LIMIT $3::int
|
||||
""",
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
str(self.owner_id),
|
||||
self.recent_window,
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
async def load_candidates(self, limit: int) -> list[dict[str, Any]]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT rp.*, s.name AS source_name, s.tag AS source_tag
|
||||
FROM raw_posts rp
|
||||
JOIN sources s ON s.id=rp.source_id
|
||||
LEFT JOIN post_publications pp ON pp.raw_post_id=rp.id AND pp.platform='vk'
|
||||
WHERE rp.status='storage_ready'
|
||||
AND rp.rewrite_status='ready'
|
||||
AND COALESCE(rp.editorial_status, 'review') IN ('accepted', 'published', 'publish_failed')
|
||||
AND COALESCE(pp.status, $1) = $1
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media rpm
|
||||
WHERE rpm.raw_post_id=rp.id
|
||||
AND COALESCE(rpm.editor_hidden, FALSE)=FALSE
|
||||
AND COALESCE(rpm.editor_added, FALSE)=FALSE
|
||||
AND (
|
||||
(rpm.media_type='photo' AND rpm.original_attachment_id LIKE 'photo%')
|
||||
OR (
|
||||
rpm.media_type='video'
|
||||
AND rpm.original_attachment_id LIKE 'video%'
|
||||
AND COALESCE(rpm.status, '') <> 'link_only'
|
||||
)
|
||||
)
|
||||
)
|
||||
ORDER BY COALESCE(rp.reviewed_at, rp.edited_at, rp.rewritten_at, rp.created_at) ASC,
|
||||
rp.id ASC
|
||||
LIMIT $2::int
|
||||
""",
|
||||
PUBLICATION_STATUS_PENDING,
|
||||
max(limit, 50),
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def choose_posts(self, candidates: list[dict[str, Any]], recent: list[dict[str, Any]], count: int) -> list[dict[str, Any]]:
|
||||
selected: list[dict[str, Any]] = []
|
||||
recent_categories = [str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "") for row in recent]
|
||||
recent_sources = [str(row.get("final_source_tag") or row.get("rewrite_source_tag") or "") for row in recent]
|
||||
pool = list(candidates)
|
||||
while pool and len(selected) < count:
|
||||
best_idx = 0
|
||||
selected_categories = [str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "") for row in selected]
|
||||
selected_sources = [str(row.get("final_source_tag") or row.get("rewrite_source_tag") or row.get("source_tag") or "") for row in selected]
|
||||
previous_category = selected_categories[-1] if selected_categories else (recent_categories[0] if recent_categories else "")
|
||||
previous_source = selected_sources[-1] if selected_sources else (recent_sources[0] if recent_sources else "")
|
||||
best_rank: tuple[int, float, int] | None = None
|
||||
for idx, post in enumerate(pool):
|
||||
category = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or "")
|
||||
source = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "")
|
||||
immediate_repeats = int(bool(category) and category == previous_category) + int(bool(source) and source == previous_source)
|
||||
repeat_penalty = (
|
||||
(recent_categories.count(category) + selected_categories.count(category)) * self.category_repeat_penalty
|
||||
+ (recent_sources.count(source) + selected_sources.count(source)) * self.source_repeat_penalty
|
||||
)
|
||||
rank = (immediate_repeats, repeat_penalty, idx)
|
||||
if best_rank is None or rank < best_rank:
|
||||
best_idx = idx
|
||||
best_rank = rank
|
||||
selected.append(pool.pop(best_idx))
|
||||
return selected
|
||||
|
||||
async def load_media(self, raw_post_id: int) -> list[dict[str, Any]]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT *
|
||||
FROM raw_post_media
|
||||
WHERE raw_post_id=$1
|
||||
AND COALESCE(editor_hidden, FALSE)=FALSE
|
||||
AND media_type IN ('photo', 'video')
|
||||
ORDER BY sort_order ASC, id ASC
|
||||
LIMIT $2::int
|
||||
""",
|
||||
raw_post_id,
|
||||
VK_MAX_ATTACHMENTS,
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def build_text(self, post: dict[str, Any]) -> str:
|
||||
category_tag = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or post.get("final_category") or post.get("rewrite_category") or "")
|
||||
source_tag = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "")
|
||||
return build_publication_text(post.get("final_text") or post.get("rewritten_text") or "", category_tag, source_tag)[:VK_MESSAGE_LIMIT]
|
||||
|
||||
async def media_attachment(self, client: VKAPIClient, item: dict[str, Any]) -> str | None:
|
||||
attachment_id = str(item.get("original_attachment_id") or "").strip()
|
||||
media_type = str(item.get("media_type") or "").strip()
|
||||
if media_type == "video":
|
||||
if str(item.get("status") or "").strip() == "link_only":
|
||||
return None
|
||||
return attachment_id if attachment_id.startswith("video") else None
|
||||
if media_type != "photo":
|
||||
return None
|
||||
if attachment_id.startswith("photo") and not item.get("editor_added"):
|
||||
return attachment_id
|
||||
return None
|
||||
|
||||
async def send_post(self, post: dict[str, Any]) -> tuple[int, str, list[str]]:
|
||||
text = self.build_text(post)
|
||||
media_rows = await self.load_media(int(post["id"]))
|
||||
async with VKAPIClient(
|
||||
token=self.token,
|
||||
rps=2,
|
||||
timeout_total_sec=max(30, await fetch_int_setting("vk_poster_timeout_sec", 90)),
|
||||
retry_attempts=max(1, await fetch_int_setting("vk_poster_max_attempts", 3)),
|
||||
retry_min_delay_sec=2,
|
||||
retry_max_delay_sec=max(2, await fetch_int_setting("vk_poster_retry_backoff_max_sec", 30)),
|
||||
) as client:
|
||||
attachments: list[str] = []
|
||||
for item in media_rows:
|
||||
try:
|
||||
attachment = await self.media_attachment(client, item)
|
||||
except Exception as exc:
|
||||
logger.warning("VK media skipped raw_post={} media_id={}: {}", post["id"], item.get("id"), exc)
|
||||
continue
|
||||
if attachment:
|
||||
attachments.append(attachment)
|
||||
if len(attachments) >= VK_MAX_ATTACHMENTS:
|
||||
break
|
||||
if self.dry_run:
|
||||
return 0, "", attachments
|
||||
post_id = await client.create_wall_post(self.owner_id, text, attachments, from_group=self.from_group)
|
||||
return post_id, post_vk_url(self.owner_id, post_id), attachments
|
||||
|
||||
async def upsert_publication(
|
||||
self,
|
||||
post_id: int,
|
||||
status: str,
|
||||
external_id: str | None = None,
|
||||
url: str | None = None,
|
||||
error: str | None = None,
|
||||
attachments: list[str] | None = None,
|
||||
) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
INSERT INTO post_publications(
|
||||
raw_post_id, platform, status, target_id, external_id, url, error, attachments_json, published_at, updated_at
|
||||
)
|
||||
VALUES($1, 'vk', $2, $3, $4, $5, $6, $7::jsonb, CASE WHEN $2=$8 THEN NOW() ELSE NULL END, NOW())
|
||||
ON CONFLICT (raw_post_id, platform) DO UPDATE
|
||||
SET status=EXCLUDED.status,
|
||||
target_id=EXCLUDED.target_id,
|
||||
external_id=EXCLUDED.external_id,
|
||||
url=EXCLUDED.url,
|
||||
error=EXCLUDED.error,
|
||||
attachments_json=EXCLUDED.attachments_json,
|
||||
published_at=CASE WHEN EXCLUDED.status=$8 THEN COALESCE(EXCLUDED.published_at, NOW()) ELSE post_publications.published_at END,
|
||||
updated_at=NOW()
|
||||
""",
|
||||
post_id,
|
||||
status,
|
||||
str(self.owner_id),
|
||||
external_id,
|
||||
url,
|
||||
error[:1000] if error else None,
|
||||
json.dumps(attachments or [], ensure_ascii=False),
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
)
|
||||
|
||||
async def process_slot(self, slot: dict[str, Any]) -> int:
|
||||
count = int(slot["count"])
|
||||
recent = await self.load_recent()
|
||||
candidates = await self.load_candidates(max(50, count * 10))
|
||||
posts = self.choose_posts(candidates, recent, count)
|
||||
published = 0
|
||||
for post in posts:
|
||||
await self.heartbeat.beat(self.pool, status="publishing", meta={"post_id": int(post["id"]), "slot": slot["id"]}, force=True)
|
||||
try:
|
||||
vk_post_id, url, attachments = await self.send_post(post)
|
||||
await self.upsert_publication(int(post["id"]), PUBLICATION_STATUS_PUBLISHED, str(vk_post_id), url, attachments=attachments)
|
||||
published += 1
|
||||
logger.info("VK published raw_post={} post_id={} attachments={}", post["id"], vk_post_id, len(attachments))
|
||||
except Exception as exc:
|
||||
await self.upsert_publication(int(post["id"]), PUBLICATION_STATUS_FAILED, error=str(exc))
|
||||
logger.exception("VK publish failed raw_post={}: {}", post["id"], exc)
|
||||
if self.send_delay_sec:
|
||||
await asyncio.sleep(self.send_delay_sec)
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE publication_runs
|
||||
SET published_count=$2,
|
||||
status=$3,
|
||||
error=$4,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
int(slot["run_id"]),
|
||||
published,
|
||||
"done" if published == count else "partial",
|
||||
None if published == count else f"published {published} of {count}",
|
||||
)
|
||||
return published
|
||||
|
||||
async def run_once(self) -> bool:
|
||||
await self.reload_settings()
|
||||
enabled = await is_worker_enabled(self.pool, WORKER_VK_POSTER)
|
||||
app_enabled = await fetch_bool_setting("vk_poster_enabled", False)
|
||||
if not enabled or not app_enabled:
|
||||
await self.heartbeat.beat(self.pool, status="disabled", force=True)
|
||||
return False
|
||||
slots = await self.due_slots()
|
||||
if not slots:
|
||||
await self.heartbeat.beat(self.pool, status="idle")
|
||||
return False
|
||||
total = 0
|
||||
for slot in slots:
|
||||
total += await self.process_slot(slot)
|
||||
await self.heartbeat.beat(self.pool, status="idle", meta={"published": total}, force=True)
|
||||
return bool(total)
|
||||
|
||||
async def run_loop(self) -> None:
|
||||
await self.init()
|
||||
logger.info("{} started pid={}", WORKER_VK_POSTER, os.getpid())
|
||||
while True:
|
||||
try:
|
||||
await self.run_once()
|
||||
except Exception as exc:
|
||||
logger.exception("VK poster loop error: {}", exc)
|
||||
await asyncio.sleep(self.interval_sec)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
logger.remove()
|
||||
logger.add(sys.stdout, level=settings.log_level)
|
||||
worker = VKPoster()
|
||||
await worker.run_loop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,662 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import html
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import aiohttp
|
||||
from aiogram import Bot
|
||||
from aiogram.client.session.aiohttp import AiohttpSession
|
||||
from aiogram.client.telegram import TelegramAPIServer
|
||||
from aiogram.exceptions import TelegramRetryAfter
|
||||
from aiogram.types import BufferedInputFile, FSInputFile, InputMediaPhoto, InputMediaVideo
|
||||
from loguru import logger
|
||||
|
||||
from ..config import settings
|
||||
from ..constants import (
|
||||
JOB_TYPE_VK_STORAGE_COPY,
|
||||
MEDIA_STATUS_FAILED,
|
||||
MEDIA_STATUS_LINK_ONLY,
|
||||
MEDIA_STATUS_UPLOADED,
|
||||
POST_STATUS_FAILED,
|
||||
POST_STATUS_STORAGE_READY,
|
||||
WORKER_STORAGE_UPLOADER,
|
||||
)
|
||||
from ..db import fetch_float_setting, fetch_int_setting, fetch_setting, get_pool
|
||||
from ..heartbeat import HeartbeatReporter
|
||||
from ..jobs import ack_done, ack_retry, claim_job, is_worker_enabled, recover_stale_jobs
|
||||
|
||||
TMP_DIR = Path("/tmp")
|
||||
TMP_PREFIX = "vkparser_tg_media_"
|
||||
MAX_MEDIA_GROUP = 10
|
||||
|
||||
|
||||
def parse_topic(value: str) -> tuple[int, int | None]:
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
raise RuntimeError("TG_MEDIA_CHANNEL_ID is empty")
|
||||
if ":" in raw:
|
||||
chat_id, thread_id = raw.split(":", 1)
|
||||
return int(chat_id), int(thread_id)
|
||||
return int(raw), None
|
||||
|
||||
|
||||
def parse_vk_video_url(vk_url: str) -> tuple[int, int] | None:
|
||||
match = re.search(r"video(-?\d+)_(\d+)", vk_url or "")
|
||||
if not match:
|
||||
return None
|
||||
return int(match.group(1)), int(match.group(2))
|
||||
|
||||
|
||||
def split_message_chunks(text: str, limit: int) -> list[str]:
|
||||
text = str(text or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
chunks: list[str] = []
|
||||
while len(text) > limit:
|
||||
split_at = text.rfind("\n", 0, limit)
|
||||
if split_at < limit // 2:
|
||||
split_at = text.rfind(" ", 0, limit)
|
||||
if split_at < limit // 2:
|
||||
split_at = limit
|
||||
chunks.append(text[:split_at].strip())
|
||||
text = text[split_at:].strip()
|
||||
if text:
|
||||
chunks.append(text)
|
||||
return chunks
|
||||
|
||||
|
||||
def original_link(post: dict) -> str:
|
||||
url = str(post.get("original_url") or "").strip()
|
||||
return f'<a href="{url}">#{int(post["id"])}</a>' if url else f'#{int(post["id"])}'
|
||||
|
||||
|
||||
class TelegramStorageUploader:
|
||||
def __init__(self) -> None:
|
||||
self.pool = None
|
||||
self.bot: Bot | None = None
|
||||
self.storage_chat_id: int | None = None
|
||||
self.storage_thread_id: int | None = None
|
||||
self.heartbeat = HeartbeatReporter(WORKER_STORAGE_UPLOADER, 30)
|
||||
self.media_group_max_items = MAX_MEDIA_GROUP
|
||||
self.media_upload_delay_sec = 1.0
|
||||
self.tg_retry_max_attempts = 4
|
||||
self.tg_retry_backoff_max_sec = 15
|
||||
self.caption_limit = 1024
|
||||
self.message_limit = 4096
|
||||
self.text_overflow_marker = "Продолжение следующим сообщением."
|
||||
self.download_timeout_sec = 45
|
||||
self.video_max_size_mb = 49
|
||||
self.video_max_height = 720
|
||||
self.video_max_duration_sec = 300
|
||||
self.yt_dlp_timeout_sec = 300
|
||||
self.max_media_attempts = 3
|
||||
self.post_job_pause_sec = 0.2
|
||||
self.using_local_bot_api = False
|
||||
|
||||
async def init(self) -> None:
|
||||
self.pool = await get_pool()
|
||||
recovered = await recover_stale_jobs(self.pool, JOB_TYPE_VK_STORAGE_COPY, stale_minutes=20)
|
||||
if recovered:
|
||||
logger.warning("Recovered stale storage jobs: {}", recovered)
|
||||
|
||||
if not settings.tg_bot_token:
|
||||
raise RuntimeError("TG_BOT_TOKEN is empty")
|
||||
media_channel = str(await fetch_setting("tg_media_channel_id", settings.tg_media_channel_id) or "").strip()
|
||||
self.storage_chat_id, self.storage_thread_id = parse_topic(media_channel)
|
||||
|
||||
local_bot_api_url = str(await fetch_setting("local_bot_api_url", settings.local_bot_api_url) or "").strip()
|
||||
if local_bot_api_url:
|
||||
session = AiohttpSession(
|
||||
api=TelegramAPIServer.from_base(local_bot_api_url.rstrip("/"), is_local=True)
|
||||
)
|
||||
self.bot = Bot(token=settings.tg_bot_token, session=session)
|
||||
try:
|
||||
await self.bot.get_me()
|
||||
self.using_local_bot_api = True
|
||||
logger.info("Using local Telegram Bot API: {}", local_bot_api_url)
|
||||
except Exception as exc:
|
||||
logger.warning("Local Telegram Bot API unavailable at {}: {}. Falling back to cloud Bot API.", local_bot_api_url, exc)
|
||||
await self.bot.session.close()
|
||||
self.bot = Bot(token=settings.tg_bot_token)
|
||||
else:
|
||||
self.bot = Bot(token=settings.tg_bot_token)
|
||||
|
||||
self.media_group_max_items = max(1, min(MAX_MEDIA_GROUP, await fetch_int_setting("telegram_media_group_max_items", 10)))
|
||||
self.media_upload_delay_sec = max(0.0, await fetch_float_setting("telegram_media_upload_delay_sec", 1.0))
|
||||
self.tg_retry_max_attempts = max(1, await fetch_int_setting("telegram_retry_max_attempts", 4))
|
||||
self.tg_retry_backoff_max_sec = max(1, await fetch_int_setting("telegram_retry_backoff_max_sec", 15))
|
||||
self.caption_limit = max(128, await fetch_int_setting("telegram_caption_limit", 1024))
|
||||
self.message_limit = max(512, await fetch_int_setting("telegram_message_limit", 4096))
|
||||
self.text_overflow_marker = str(await fetch_setting("telegram_text_overflow_marker", self.text_overflow_marker))
|
||||
self.download_timeout_sec = max(5, await fetch_int_setting("uploader_download_timeout_sec", 45))
|
||||
self.video_max_size_mb = max(1, await fetch_int_setting("video_max_size_mb", 49))
|
||||
if not self.using_local_bot_api:
|
||||
self.video_max_size_mb = min(self.video_max_size_mb, 49)
|
||||
self.video_max_height = max(144, await fetch_int_setting("video_max_height", 720))
|
||||
self.video_max_duration_sec = max(1, await fetch_int_setting("video_max_duration_sec", 300))
|
||||
self.yt_dlp_timeout_sec = max(30, await fetch_int_setting("uploader_yt_dlp_timeout_sec", 300))
|
||||
self.max_media_attempts = max(1, await fetch_int_setting("uploader_max_media_attempts", 3))
|
||||
self.post_job_pause_sec = max(0.0, await fetch_float_setting("media_post_job_pause_sec", 0.2))
|
||||
|
||||
logger.info(
|
||||
"Telegram storage config: media_delay={}s, media_group_max_items={}, tg_retry={}, tg_backoff_max={}s, dl_timeout={}s, ytdlp_timeout={}s, post_pause={}s",
|
||||
self.media_upload_delay_sec,
|
||||
self.media_group_max_items,
|
||||
self.tg_retry_max_attempts,
|
||||
self.tg_retry_backoff_max_sec,
|
||||
self.download_timeout_sec,
|
||||
self.yt_dlp_timeout_sec,
|
||||
self.post_job_pause_sec,
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.bot:
|
||||
await self.bot.session.close()
|
||||
|
||||
def chat_kwargs(self) -> dict:
|
||||
kwargs = {"chat_id": self.storage_chat_id}
|
||||
if self.storage_thread_id:
|
||||
kwargs["message_thread_id"] = self.storage_thread_id
|
||||
return kwargs
|
||||
|
||||
async def tg_retry(self, fn):
|
||||
for attempt in range(1, self.tg_retry_max_attempts + 1):
|
||||
try:
|
||||
return await fn()
|
||||
except TelegramRetryAfter as exc:
|
||||
delay = float(exc.retry_after) + 0.5
|
||||
logger.warning("Telegram flood control, sleep {}s", delay)
|
||||
await asyncio.sleep(delay)
|
||||
except Exception as exc:
|
||||
if attempt >= self.tg_retry_max_attempts:
|
||||
raise
|
||||
delay = min(2**attempt, self.tg_retry_backoff_max_sec)
|
||||
logger.warning("Telegram send error: {}. retry in {}s", exc, delay)
|
||||
await asyncio.sleep(delay)
|
||||
raise RuntimeError("telegram retry exhausted")
|
||||
|
||||
async def load_raw_post(self, raw_post_id: int) -> dict | None:
|
||||
row = await self.pool.fetchrow(
|
||||
"""
|
||||
SELECT rp.*, s.name AS source_name
|
||||
FROM raw_posts rp
|
||||
JOIN sources s ON s.id=rp.source_id
|
||||
WHERE rp.id=$1
|
||||
""",
|
||||
raw_post_id,
|
||||
)
|
||||
return dict(row) if row else None
|
||||
|
||||
async def load_media(self, raw_post_id: int) -> list[dict]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT *
|
||||
FROM raw_post_media
|
||||
WHERE raw_post_id=$1
|
||||
ORDER BY sort_order ASC, id ASC
|
||||
""",
|
||||
raw_post_id,
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def media_blocking_error(self, media: list[dict], prepared_media_ids: set[int] | None = None) -> str | None:
|
||||
prepared_media_ids = prepared_media_ids or set()
|
||||
visible_media = [
|
||||
item
|
||||
for item in media
|
||||
if str(item.get("media_type") or "") in ("photo", "video")
|
||||
and not bool(item.get("editor_hidden"))
|
||||
]
|
||||
if not visible_media:
|
||||
return "no publishable media"
|
||||
pending = [
|
||||
item
|
||||
for item in visible_media
|
||||
if str(item.get("status") or "") == "pending"
|
||||
and int(item["id"]) not in prepared_media_ids
|
||||
]
|
||||
if pending:
|
||||
details = ", ".join(
|
||||
f"{item.get('media_type')}#{item.get('id')}: {item.get('error') or 'pending'}"
|
||||
for item in pending[:5]
|
||||
)
|
||||
return f"media still pending: {details}"
|
||||
failed = [
|
||||
item
|
||||
for item in visible_media
|
||||
if str(item.get("status") or "") == MEDIA_STATUS_FAILED
|
||||
]
|
||||
if failed:
|
||||
details = ", ".join(
|
||||
f"{item.get('media_type')}#{item.get('id')}: {item.get('error') or 'failed'}"
|
||||
for item in failed[:5]
|
||||
)
|
||||
return f"media failed: {details}"
|
||||
link_only = [
|
||||
item
|
||||
for item in visible_media
|
||||
if str(item.get("status") or "") == MEDIA_STATUS_LINK_ONLY
|
||||
]
|
||||
if link_only:
|
||||
details = ", ".join(
|
||||
f"{item.get('media_type')}#{item.get('id')}: {item.get('error') or 'link only'}"
|
||||
for item in link_only[:5]
|
||||
)
|
||||
return f"media link only: {details}"
|
||||
missing_upload = [
|
||||
item
|
||||
for item in visible_media
|
||||
if not (item.get("tg_file_id") or item.get("storage_attachment_id"))
|
||||
and int(item["id"]) not in prepared_media_ids
|
||||
]
|
||||
if missing_upload:
|
||||
details = ", ".join(
|
||||
f"{item.get('media_type')}#{item.get('id')}: missing tg_file_id"
|
||||
for item in missing_upload[:5]
|
||||
)
|
||||
return f"media not uploaded: {details}"
|
||||
return None
|
||||
|
||||
async def download_bytes(self, session: aiohttp.ClientSession, url: str) -> bytes | None:
|
||||
try:
|
||||
async with session.get(url, timeout=aiohttp.ClientTimeout(total=self.download_timeout_sec)) as response:
|
||||
if response.status != 200:
|
||||
return None
|
||||
return await response.read()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
async def mark_media_uploaded(self, media_id: int, file_id: str, unique_id: str | None) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE raw_post_media
|
||||
SET status=$2,
|
||||
storage_attachment_id=$3,
|
||||
storage_url=$3,
|
||||
tg_file_id=$3,
|
||||
tg_file_unique_id=$4,
|
||||
error=NULL,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
media_id,
|
||||
MEDIA_STATUS_UPLOADED,
|
||||
file_id,
|
||||
unique_id,
|
||||
)
|
||||
|
||||
async def mark_media_link_only(self, media_id: int, error: str | None = None) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE raw_post_media
|
||||
SET status=$2,
|
||||
error=COALESCE($3, error),
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
media_id,
|
||||
MEDIA_STATUS_LINK_ONLY,
|
||||
error,
|
||||
)
|
||||
|
||||
async def mark_media_failed_attempt(self, media_id: int, error: str) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE raw_post_media
|
||||
SET attempts=attempts+1,
|
||||
status=CASE WHEN attempts + 1 >= $3 THEN $2 ELSE status END,
|
||||
error=$4,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
media_id,
|
||||
MEDIA_STATUS_FAILED,
|
||||
self.max_media_attempts,
|
||||
error[:1000],
|
||||
)
|
||||
|
||||
async def download_video(self, vk_url: str, output_path: str) -> dict | None:
|
||||
parsed = parse_vk_video_url(vk_url)
|
||||
if not parsed:
|
||||
return None
|
||||
owner_id, video_id = parsed
|
||||
max_size = self.video_max_size_mb * 1024 * 1024
|
||||
netrc_path = f"{output_path}.netrc"
|
||||
fd = os.open(netrc_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
||||
fh.write(f"machine vk.com login vk_token password {settings.vk_access_token}\n")
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"yt_dlp",
|
||||
"--netrc-location",
|
||||
netrc_path,
|
||||
f"https://vk.com/video{owner_id}_{video_id}",
|
||||
"-o",
|
||||
output_path,
|
||||
"--no-playlist",
|
||||
"-f",
|
||||
(
|
||||
f"best[height<={self.video_max_height}][filesize<{max_size}]"
|
||||
f"/best[height<={self.video_max_height}]"
|
||||
f"/bestvideo[height<={self.video_max_height}][filesize<{max_size}]+bestaudio/best"
|
||||
f"/bestvideo[height<={self.video_max_height}]+bestaudio/best"
|
||||
f"/best[filesize<{max_size}]"
|
||||
),
|
||||
"--quiet",
|
||||
"--no-warnings",
|
||||
]
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
_, stderr = await asyncio.wait_for(proc.communicate(), timeout=self.yt_dlp_timeout_sec)
|
||||
except asyncio.TimeoutError:
|
||||
proc.kill()
|
||||
await proc.communicate()
|
||||
return {"error": "video download timeout", "permanent": False}
|
||||
finally:
|
||||
try:
|
||||
os.unlink(netrc_path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
if proc.returncode != 0 or not os.path.exists(output_path):
|
||||
err = (stderr or b"").decode("utf-8", errors="ignore").lower()
|
||||
permanent = any(marker in err for marker in ("removed", "unavailable", "private", "access denied"))
|
||||
return {"error": "video unavailable or download failed", "permanent": permanent}
|
||||
size = os.path.getsize(output_path)
|
||||
if size > max_size:
|
||||
return {"error": "video too large", "permanent": True}
|
||||
return {"path": output_path, "size_bytes": size}
|
||||
|
||||
async def prepare_media(self, raw_post_id: int, media: list[dict]) -> list[dict]:
|
||||
prepared: list[dict] = []
|
||||
async with aiohttp.ClientSession() as session:
|
||||
for item in media:
|
||||
media_id = int(item["id"])
|
||||
media_type = str(item["media_type"])
|
||||
url = str(item.get("original_url") or "")
|
||||
|
||||
if item.get("tg_file_id"):
|
||||
prepared.append({"media_id": media_id, "media_type": media_type, "media": item["tg_file_id"]})
|
||||
continue
|
||||
if int(item.get("attempts") or 0) >= self.max_media_attempts:
|
||||
continue
|
||||
|
||||
if media_type == "photo":
|
||||
data = await self.download_bytes(session, url)
|
||||
if not data:
|
||||
await self.mark_media_failed_attempt(media_id, "photo download failed")
|
||||
continue
|
||||
prepared.append(
|
||||
{
|
||||
"media_id": media_id,
|
||||
"media_type": "photo",
|
||||
"media": BufferedInputFile(data, filename=f"photo_{media_id}.jpg"),
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
if media_type == "video":
|
||||
duration = item.get("duration_sec")
|
||||
if duration and int(duration) > self.video_max_duration_sec:
|
||||
await self.mark_media_link_only(media_id, "video too long")
|
||||
continue
|
||||
temp_path = str(TMP_DIR / f"{TMP_PREFIX}{raw_post_id}_{media_id}.mp4")
|
||||
info = await self.download_video(url, temp_path)
|
||||
if not info:
|
||||
await self.mark_media_failed_attempt(media_id, "video download failed")
|
||||
continue
|
||||
if info.get("error"):
|
||||
if info.get("permanent"):
|
||||
await self.mark_media_link_only(media_id, str(info["error"]))
|
||||
else:
|
||||
await self.mark_media_failed_attempt(media_id, str(info["error"]))
|
||||
continue
|
||||
prepared.append(
|
||||
{
|
||||
"media_id": media_id,
|
||||
"media_type": "video",
|
||||
"media": FSInputFile(temp_path, filename="video.mp4"),
|
||||
"tmp_path": temp_path,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
await self.mark_media_link_only(media_id, "unsupported media type")
|
||||
return prepared
|
||||
|
||||
async def send_text_chunk(self, text: str) -> int:
|
||||
message = await self.tg_retry(
|
||||
lambda: self.bot.send_message(
|
||||
text=text,
|
||||
parse_mode="HTML",
|
||||
disable_web_page_preview=True,
|
||||
**self.chat_kwargs(),
|
||||
)
|
||||
)
|
||||
return int(message.message_id)
|
||||
|
||||
def build_text_parts(self, post: dict, has_media: bool, link_only_media: list[dict] | None = None) -> tuple[str | None, list[str]]:
|
||||
link = original_link(post)
|
||||
raw_text = html.escape(str(post.get("raw_text") or "").strip())
|
||||
link_only_media = link_only_media or []
|
||||
media_lines = []
|
||||
for item in link_only_media:
|
||||
url = str(item.get("original_url") or "").strip()
|
||||
if not url:
|
||||
continue
|
||||
media_type = html.escape(str(item.get("media_type") or "media"))
|
||||
reason = html.escape(str(item.get("error") or "link only"))
|
||||
media_lines.append(f"- {media_type}: <a href=\"{html.escape(url, quote=True)}\">ссылка</a> ({reason})")
|
||||
media_note = "\n\nМедиа по ссылке:\n" + "\n".join(media_lines) if media_lines else ""
|
||||
suffix = f"\n\n{link}"
|
||||
body = raw_text + media_note
|
||||
if has_media:
|
||||
if len(body + suffix) <= self.caption_limit:
|
||||
return body + suffix if body else link, []
|
||||
chunks = split_message_chunks(body + suffix, self.message_limit)
|
||||
return self.text_overflow_marker[: self.caption_limit], chunks
|
||||
chunks = split_message_chunks(body + suffix if body else link, self.message_limit)
|
||||
return None, chunks
|
||||
|
||||
async def save_sent_media_ids(self, prepared: list[dict], messages: list) -> None:
|
||||
for item, message in zip(prepared, messages):
|
||||
if item["media_type"] == "photo" and message.photo:
|
||||
photo = message.photo[-1]
|
||||
await self.mark_media_uploaded(int(item["media_id"]), photo.file_id, photo.file_unique_id)
|
||||
elif item["media_type"] == "video" and message.video:
|
||||
video = message.video
|
||||
await self.mark_media_uploaded(int(item["media_id"]), video.file_id, video.file_unique_id)
|
||||
|
||||
async def send_storage_post(self, post: dict, prepared: list[dict], media: list[dict]) -> tuple[list[int], int | None]:
|
||||
uploaded_media_ids = {int(item["media_id"]) for item in prepared}
|
||||
link_only_media = [
|
||||
item
|
||||
for item in media
|
||||
if int(item["id"]) not in uploaded_media_ids and str(item.get("status") or "") == MEDIA_STATUS_LINK_ONLY
|
||||
]
|
||||
caption, text_chunks = self.build_text_parts(post, bool(prepared), link_only_media)
|
||||
message_ids: list[int] = []
|
||||
|
||||
if prepared:
|
||||
first = prepared[: self.media_group_max_items]
|
||||
group = []
|
||||
for idx, item in enumerate(first):
|
||||
cap = caption if idx == 0 else None
|
||||
if item["media_type"] == "photo":
|
||||
group.append(InputMediaPhoto(media=item["media"], caption=cap, parse_mode="HTML"))
|
||||
elif item["media_type"] == "video":
|
||||
group.append(InputMediaVideo(media=item["media"], caption=cap, parse_mode="HTML"))
|
||||
if group:
|
||||
messages = await self.tg_retry(lambda: self.bot.send_media_group(media=group, **self.chat_kwargs()))
|
||||
await self.save_sent_media_ids(first, messages)
|
||||
message_ids.extend(int(msg.message_id) for msg in messages)
|
||||
await asyncio.sleep(self.media_upload_delay_sec * len(first))
|
||||
|
||||
rest = prepared[self.media_group_max_items :]
|
||||
for start in range(0, len(rest), self.media_group_max_items):
|
||||
chunk = rest[start : start + self.media_group_max_items]
|
||||
group = [
|
||||
InputMediaPhoto(media=item["media"])
|
||||
if item["media_type"] == "photo"
|
||||
else InputMediaVideo(media=item["media"])
|
||||
for item in chunk
|
||||
if item["media_type"] in ("photo", "video")
|
||||
]
|
||||
if group:
|
||||
messages = await self.tg_retry(lambda g=group: self.bot.send_media_group(media=g, **self.chat_kwargs()))
|
||||
await self.save_sent_media_ids(chunk, messages)
|
||||
message_ids.extend(int(msg.message_id) for msg in messages)
|
||||
await asyncio.sleep(self.media_upload_delay_sec * len(chunk))
|
||||
for chunk in text_chunks:
|
||||
message_ids.append(await self.send_text_chunk(chunk))
|
||||
return message_ids, None
|
||||
|
||||
async def mark_post_ready(self, raw_post_id: int, message_ids: list[int], meta_message_id: int | None) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE raw_posts
|
||||
SET status=$2,
|
||||
tg_storage_chat_id=$3,
|
||||
tg_storage_thread_id=$4,
|
||||
tg_storage_message_ids=$5,
|
||||
tg_storage_meta_message_id=$6,
|
||||
storage_post_url=$7,
|
||||
copied_at=NOW(),
|
||||
error_reason=NULL,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
raw_post_id,
|
||||
POST_STATUS_STORAGE_READY,
|
||||
self.storage_chat_id,
|
||||
self.storage_thread_id,
|
||||
message_ids,
|
||||
meta_message_id,
|
||||
f"tg://resolve?domain=c/{str(abs(int(self.storage_chat_id or 0))).removeprefix('100')}/{message_ids[0]}" if message_ids else None,
|
||||
)
|
||||
|
||||
async def mark_post_failed(self, raw_post_id: int, error: str) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE raw_posts
|
||||
SET status=$2,
|
||||
error_reason=$3,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
raw_post_id,
|
||||
POST_STATUS_FAILED,
|
||||
error[:1000],
|
||||
)
|
||||
|
||||
async def process_job(self, job: dict, worker_id: str) -> None:
|
||||
job_id = int(job["id"])
|
||||
raw_post_id = int(job["entity_id"])
|
||||
await self.heartbeat.beat(self.pool, current_job_id=job_id, force=True)
|
||||
|
||||
post = await self.load_raw_post(raw_post_id)
|
||||
if not post:
|
||||
await ack_retry(self.pool, job, f"raw_post {raw_post_id} not found", delay_sec=60)
|
||||
return
|
||||
if post.get("tg_storage_message_ids"):
|
||||
await ack_done(self.pool, job_id)
|
||||
return
|
||||
|
||||
media = await self.load_media(raw_post_id)
|
||||
prepared = await self.prepare_media(raw_post_id, media)
|
||||
media = await self.load_media(raw_post_id)
|
||||
prepared_media_ids = {int(item["media_id"]) for item in prepared}
|
||||
blocking_error = self.media_blocking_error(media, prepared_media_ids)
|
||||
if blocking_error:
|
||||
for item in prepared:
|
||||
tmp_path = item.get("tmp_path")
|
||||
if tmp_path:
|
||||
try:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
except Exception:
|
||||
pass
|
||||
if not blocking_error.startswith("media still pending:"):
|
||||
await self.mark_post_failed(raw_post_id, blocking_error)
|
||||
await ack_done(self.pool, job_id)
|
||||
logger.warning("Telegram storage blocked raw_post={}: {}", raw_post_id, blocking_error)
|
||||
return
|
||||
await ack_retry(self.pool, job, blocking_error, delay_sec=120)
|
||||
logger.warning("Telegram storage retry raw_post={}: {}", raw_post_id, blocking_error)
|
||||
return
|
||||
try:
|
||||
message_ids, meta_message_id = await self.send_storage_post(post, prepared, media)
|
||||
finally:
|
||||
for item in prepared:
|
||||
tmp_path = item.get("tmp_path")
|
||||
if tmp_path:
|
||||
try:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
except Exception:
|
||||
pass
|
||||
await self.mark_post_ready(raw_post_id, message_ids, meta_message_id)
|
||||
await ack_done(self.pool, job_id)
|
||||
logger.info("Telegram storage done: raw_post={} messages={}", raw_post_id, message_ids)
|
||||
|
||||
async def run_once(self, worker_id: str) -> bool:
|
||||
enabled = await is_worker_enabled(self.pool, WORKER_STORAGE_UPLOADER)
|
||||
if not enabled:
|
||||
await self.heartbeat.beat(self.pool, status="disabled", force=True)
|
||||
return False
|
||||
|
||||
job = await claim_job(self.pool, JOB_TYPE_VK_STORAGE_COPY, worker_id)
|
||||
if not job:
|
||||
await self.heartbeat.beat(self.pool, status="idle")
|
||||
return False
|
||||
try:
|
||||
await self.process_job(job, worker_id)
|
||||
except Exception as exc:
|
||||
await self.mark_post_failed(int(job["entity_id"]), str(exc))
|
||||
await ack_retry(self.pool, job, str(exc), delay_sec=120)
|
||||
logger.exception("Telegram storage job {} failed: {}", job["id"], exc)
|
||||
return True
|
||||
|
||||
async def cleanup_tmp_files(self) -> None:
|
||||
now = time.time()
|
||||
for path in TMP_DIR.glob(f"{TMP_PREFIX}*"):
|
||||
try:
|
||||
if path.is_file() and path.stat().st_mtime < now - 3600:
|
||||
path.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def run_loop(self) -> None:
|
||||
await self.init()
|
||||
worker_id = f"{WORKER_STORAGE_UPLOADER}:{os.getpid()}"
|
||||
logger.info("{} started", worker_id)
|
||||
try:
|
||||
while True:
|
||||
await self.cleanup_tmp_files()
|
||||
had_job = await self.run_once(worker_id)
|
||||
if not had_job:
|
||||
await asyncio.sleep(max(1, await fetch_int_setting("uploader_interval_sec", 5)))
|
||||
else:
|
||||
await asyncio.sleep(self.post_job_pause_sec)
|
||||
finally:
|
||||
await self.close()
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
logger.remove()
|
||||
logger.add(sys.stdout, level=settings.log_level)
|
||||
worker = TelegramStorageUploader()
|
||||
await worker.run_loop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user