Initial parser admin implementation
This commit is contained in:
Your Name
@@ -0,0 +1,23 @@
|
||||
APP_ENV=production
|
||||
APP_SECRET_KEY=change-me
|
||||
ADMIN_BOOTSTRAP_LOGIN=admin
|
||||
ADMIN_BOOTSTRAP_PASSWORD=change-me-long-password
|
||||
|
||||
DB_HOST=localhost
|
||||
DB_PORT=5432
|
||||
DB_NAME=vk_parser
|
||||
DB_USER=vk_parser_user
|
||||
DB_PASSWORD=change-me
|
||||
|
||||
VK_ACCESS_TOKEN=
|
||||
VK_GROUP_ACCESS_TOKEN=
|
||||
VK_API_VERSION=5.199
|
||||
VK_STORAGE_GROUP_ID=239548476
|
||||
|
||||
TG_BOT_TOKEN=
|
||||
TG_MEDIA_CHANNEL_ID=
|
||||
LOCAL_BOT_API_URL=
|
||||
|
||||
ADMIN_HOST=0.0.0.0
|
||||
ADMIN_PORT=8080
|
||||
LOG_LEVEL=INFO
|
||||
@@ -0,0 +1,5 @@
|
||||
* text=auto eol=lf
|
||||
|
||||
*.bat text eol=crlf
|
||||
*.cmd text eol=crlf
|
||||
*.ps1 text eol=crlf
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
.venv/
|
||||
logs/
|
||||
uploads/
|
||||
original_project/
|
||||
ai_worker.py
|
||||
media_worker.py
|
||||
parser.py
|
||||
raw_feed_worker.py
|
||||
vkdev_main.js
|
||||
sample_photo.jpg
|
||||
@@ -0,0 +1,199 @@
|
||||
# VK Parser New Core
|
||||
|
||||
Первый чистый слой проекта:
|
||||
|
||||
- админка с источниками, пользователями, воркерами и raw-лентой;
|
||||
- VK parser worker;
|
||||
- VK storage uploader worker;
|
||||
- PostgreSQL job queue.
|
||||
|
||||
Старый проект лежит в `original_project/` как справочник. Новый код живёт в `src/vk_parser_app/`.
|
||||
|
||||
## Что нужно на сервере
|
||||
|
||||
Ubuntu 22.04/24.04:
|
||||
|
||||
```bash
|
||||
sudo apt update
|
||||
sudo apt install -y python3 python3-venv python3-pip postgresql postgresql-contrib nginx git curl
|
||||
```
|
||||
|
||||
Для будущей работы с видео:
|
||||
|
||||
```bash
|
||||
sudo apt install -y ffmpeg
|
||||
```
|
||||
|
||||
## БД
|
||||
|
||||
Пример:
|
||||
|
||||
```bash
|
||||
sudo -u postgres psql
|
||||
```
|
||||
|
||||
```sql
|
||||
CREATE DATABASE vk_parser;
|
||||
CREATE USER vk_parser_user WITH PASSWORD 'CHANGE_ME';
|
||||
GRANT ALL PRIVILEGES ON DATABASE vk_parser TO vk_parser_user;
|
||||
\c vk_parser
|
||||
GRANT ALL ON SCHEMA public TO vk_parser_user;
|
||||
```
|
||||
|
||||
## Установка проекта
|
||||
|
||||
```bash
|
||||
cd /opt
|
||||
sudo git clone <repo-url> vk-parser
|
||||
sudo chown -R $USER:$USER /opt/vk-parser
|
||||
cd /opt/vk-parser
|
||||
|
||||
python3 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
Заполнить `.env`:
|
||||
|
||||
```env
|
||||
APP_SECRET_KEY=long-random-string
|
||||
ADMIN_BOOTSTRAP_LOGIN=admin
|
||||
ADMIN_BOOTSTRAP_PASSWORD=long-admin-password
|
||||
|
||||
DB_HOST=localhost
|
||||
DB_PORT=5432
|
||||
DB_NAME=vk_parser
|
||||
DB_USER=vk_parser_user
|
||||
DB_PASSWORD=...
|
||||
|
||||
VK_ACCESS_TOKEN=...
|
||||
VK_STORAGE_GROUP_ID=239548476
|
||||
```
|
||||
|
||||
Важно: для `wall.post`, `wall.edit`, `photos.saveWallPhoto` нужен VK user token пользователя с правами в storage-группе. Community token для этого контура не подходит.
|
||||
|
||||
## Миграции
|
||||
|
||||
```bash
|
||||
PYTHONPATH=src .venv/bin/python scripts/apply_migrations.py
|
||||
```
|
||||
|
||||
Первый админ создаётся из `ADMIN_BOOTSTRAP_LOGIN` / `ADMIN_BOOTSTRAP_PASSWORD` при старте админки.
|
||||
|
||||
## Проверка VK storage
|
||||
|
||||
Без фото:
|
||||
|
||||
```bash
|
||||
PYTHONPATH=src .venv/bin/python scripts/vk_storage_spike.py
|
||||
```
|
||||
|
||||
С фото:
|
||||
|
||||
```bash
|
||||
cp /path/to/photo.jpg sample_photo.jpg
|
||||
PYTHONPATH=src .venv/bin/python scripts/vk_storage_spike.py
|
||||
```
|
||||
|
||||
Скрипт должен вывести ссылку вида:
|
||||
|
||||
```text
|
||||
https://vk.com/wall-239548476_123
|
||||
```
|
||||
|
||||
После этого надо руками проверить:
|
||||
|
||||
- пост создан от имени storage-группы;
|
||||
- админ/редактор группы может открыть и отредактировать пост;
|
||||
- вложенное фото видно в браузере.
|
||||
|
||||
## Запуск локально
|
||||
|
||||
Админка:
|
||||
|
||||
```bash
|
||||
PYTHONPATH=src .venv/bin/uvicorn vk_parser_app.admin:app --host 0.0.0.0 --port 8080
|
||||
```
|
||||
|
||||
Парсер:
|
||||
|
||||
```bash
|
||||
PYTHONPATH=src .venv/bin/python -m vk_parser_app.workers.parser
|
||||
```
|
||||
|
||||
VK storage uploader:
|
||||
|
||||
```bash
|
||||
PYTHONPATH=src .venv/bin/python -m vk_parser_app.workers.vk_storage_uploader
|
||||
```
|
||||
|
||||
## systemd units
|
||||
|
||||
Минимальный `vk-admin.service`:
|
||||
|
||||
```ini
|
||||
[Unit]
|
||||
Description=VK Parser Admin
|
||||
After=network.target postgresql.service
|
||||
|
||||
[Service]
|
||||
WorkingDirectory=/opt/vk-parser
|
||||
Environment=PYTHONPATH=/opt/vk-parser/src
|
||||
ExecStart=/opt/vk-parser/.venv/bin/uvicorn vk_parser_app.admin:app --host 0.0.0.0 --port 8080
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
`vk-parser.service`:
|
||||
|
||||
```ini
|
||||
[Unit]
|
||||
Description=VK Parser Worker
|
||||
After=network.target postgresql.service
|
||||
|
||||
[Service]
|
||||
WorkingDirectory=/opt/vk-parser
|
||||
Environment=PYTHONPATH=/opt/vk-parser/src
|
||||
ExecStart=/opt/vk-parser/.venv/bin/python -m vk_parser_app.workers.parser
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
`vk-storage-uploader.service`:
|
||||
|
||||
```ini
|
||||
[Unit]
|
||||
Description=VK Storage Uploader Worker
|
||||
After=network.target postgresql.service
|
||||
|
||||
[Service]
|
||||
WorkingDirectory=/opt/vk-parser
|
||||
Environment=PYTHONPATH=/opt/vk-parser/src
|
||||
ExecStart=/opt/vk-parser/.venv/bin/python -m vk_parser_app.workers.vk_storage_uploader
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
## Текущий pipeline
|
||||
|
||||
```text
|
||||
sources(vk)
|
||||
-> vk-parser
|
||||
-> raw_posts + raw_post_media
|
||||
-> jobs(type='vk.storage.copy')
|
||||
-> vk-storage-uploader
|
||||
-> wall.post в storage-группе
|
||||
-> raw_posts.storage_post_url
|
||||
```
|
||||
|
||||
Фото копируются в storage-группу. Видео в первой версии сохраняются как `link_only`/исходное VK-вложение, чтобы сначала стабилизировать основной контур.
|
||||
@@ -0,0 +1,194 @@
|
||||
CREATE TABLE IF NOT EXISTS admin_users (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
login TEXT NOT NULL UNIQUE,
|
||||
password_hash TEXT NOT NULL,
|
||||
role TEXT NOT NULL DEFAULT 'admin' CHECK (role IN ('admin','editor','viewer')),
|
||||
is_active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS admin_sessions (
|
||||
token_hash TEXT PRIMARY KEY,
|
||||
user_id BIGINT NOT NULL REFERENCES admin_users(id) ON DELETE CASCADE,
|
||||
csrf_token TEXT NOT NULL,
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_admin_sessions_user_id ON admin_sessions(user_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_admin_sessions_expires_at ON admin_sessions(expires_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sources (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
platform TEXT NOT NULL DEFAULT 'vk',
|
||||
name TEXT NOT NULL,
|
||||
url TEXT NOT NULL,
|
||||
external_id TEXT,
|
||||
external_owner_id BIGINT,
|
||||
active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
status TEXT NOT NULL DEFAULT 'new',
|
||||
status_msg TEXT,
|
||||
parse_from TIMESTAMPTZ,
|
||||
last_checked_at TIMESTAMPTZ,
|
||||
last_parsed_at TIMESTAMPTZ,
|
||||
priority INTEGER NOT NULL DEFAULT 100,
|
||||
tags TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
|
||||
settings_json JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_by BIGINT REFERENCES admin_users(id) ON DELETE SET NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
archived_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS ux_sources_platform_url_active
|
||||
ON sources(platform, lower(url))
|
||||
WHERE archived_at IS NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_sources_active_platform ON sources(active, platform, priority, id)
|
||||
WHERE archived_at IS NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS raw_posts (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
source_id BIGINT NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
|
||||
platform TEXT NOT NULL,
|
||||
external_post_id TEXT NOT NULL,
|
||||
external_owner_id BIGINT,
|
||||
original_url TEXT NOT NULL,
|
||||
storage_owner_id BIGINT,
|
||||
storage_post_id BIGINT,
|
||||
storage_post_url TEXT,
|
||||
raw_text TEXT NOT NULL DEFAULT '',
|
||||
raw_json JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
text_hash TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL,
|
||||
posted_at TIMESTAMPTZ,
|
||||
copied_at TIMESTAMPTZ,
|
||||
status TEXT NOT NULL DEFAULT 'raw_saved',
|
||||
skip_reason TEXT,
|
||||
error_reason TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE(source_id, external_post_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_posts_status_created ON raw_posts(status, created_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_posts_source_posted ON raw_posts(source_id, posted_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_posts_content_hash ON raw_posts(content_hash);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS raw_post_media (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
raw_post_id BIGINT NOT NULL REFERENCES raw_posts(id) ON DELETE CASCADE,
|
||||
platform TEXT NOT NULL,
|
||||
media_type TEXT NOT NULL CHECK (media_type IN ('photo','video','link','doc','unknown')),
|
||||
original_url TEXT,
|
||||
original_attachment_id TEXT,
|
||||
storage_attachment_id TEXT,
|
||||
storage_url TEXT,
|
||||
preview_url TEXT,
|
||||
width INTEGER,
|
||||
height INTEGER,
|
||||
duration_sec INTEGER,
|
||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
error TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_post_media_post ON raw_post_media(raw_post_id, sort_order, id);
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_post_media_status ON raw_post_media(status);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS jobs (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
type TEXT NOT NULL,
|
||||
entity_type TEXT NOT NULL,
|
||||
entity_id BIGINT NOT NULL,
|
||||
payload_json JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
max_attempts INTEGER NOT NULL DEFAULT 5,
|
||||
next_run_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
locked_by TEXT,
|
||||
locked_at TIMESTAMPTZ,
|
||||
last_error TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS ux_jobs_open_entity_type
|
||||
ON jobs(type, entity_type, entity_id)
|
||||
WHERE status IN ('pending','retry','in_progress');
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_jobs_claim ON jobs(type, status, next_run_at, id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS worker_controls (
|
||||
name TEXT PRIMARY KEY,
|
||||
enabled BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
settings_json JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
updated_by BIGINT REFERENCES admin_users(id) ON DELETE SET NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS worker_heartbeats (
|
||||
name TEXT PRIMARY KEY,
|
||||
heartbeat_at TIMESTAMPTZ NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'running',
|
||||
current_job_id BIGINT,
|
||||
meta_json JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS app_settings (
|
||||
key TEXT PRIMARY KEY,
|
||||
value_json JSONB NOT NULL,
|
||||
value_type TEXT NOT NULL DEFAULT 'str',
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
description TEXT NOT NULL DEFAULT '',
|
||||
category TEXT NOT NULL DEFAULT 'General',
|
||||
updated_by BIGINT REFERENCES admin_users(id) ON DELETE SET NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS audit_log (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
actor_id BIGINT REFERENCES admin_users(id) ON DELETE SET NULL,
|
||||
action TEXT NOT NULL,
|
||||
entity_type TEXT NOT NULL,
|
||||
entity_id BIGINT,
|
||||
before_json JSONB,
|
||||
after_json JSONB,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
INSERT INTO worker_controls(name, enabled, settings_json)
|
||||
VALUES
|
||||
('vk-parser', TRUE, '{}'::jsonb),
|
||||
('vk-storage-uploader', TRUE, '{}'::jsonb)
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
|
||||
INSERT INTO app_settings(key, value_json, value_type, title, description, category)
|
||||
VALUES
|
||||
('parser_interval_sec', '300'::jsonb, 'int', 'Parser interval, sec', 'Pause between parser cycles.', 'Parser'),
|
||||
('parser_new_source_lookback_days', '14'::jsonb, 'int', 'New source lookback, days', 'How far back to read a new source.', 'Parser'),
|
||||
('parser_reparse_overlap_minutes', '120'::jsonb, 'int', 'Reparse overlap, min', 'Overlap window for known sources.', 'Parser'),
|
||||
('parser_min_text_length', '50'::jsonb, 'int', 'Minimum text length', 'Posts with shorter text are treated as junk.', 'Parser'),
|
||||
('parser_skip_reposts', 'true'::jsonb, 'bool', 'Skip reposts', 'Do not save reposts from VK copy_history.', 'Parser'),
|
||||
('parser_skip_empty_text', 'true'::jsonb, 'bool', 'Skip empty text posts', 'Treat posts without text as junk.', 'Parser'),
|
||||
('parser_skip_no_media', 'true'::jsonb, 'bool', 'Skip posts without media', 'Treat posts without media as junk.', 'Parser'),
|
||||
('parser_skip_text_too_short', 'true'::jsonb, 'bool', 'Skip short text posts', 'Treat posts shorter than parser_min_text_length as junk.', 'Parser'),
|
||||
('parser_store_skipped_posts', 'false'::jsonb, 'bool', 'Store skipped posts', 'Keep junk posts in raw_posts instead of dropping them.', 'Parser'),
|
||||
('parser_dedupe_content_hash', 'true'::jsonb, 'bool', 'Dedupe by content hash', 'Skip posts whose text and media hash already exists in raw_posts.', 'Parser'),
|
||||
('parser_source_pause_sec', '0'::jsonb, 'float', 'Source pause, sec', 'Pause between VK sources inside one parser cycle.', 'Parser'),
|
||||
('vk_requests_per_second', '3'::jsonb, 'int', 'VK requests per second', 'Global VK API rate limit per worker.', 'VK'),
|
||||
('vk_wall_page_size', '50'::jsonb, 'int', 'VK wall page size', 'wall.get count per page.', 'VK'),
|
||||
('vk_api_timeout_total_sec', '15'::jsonb, 'int', 'VK total timeout, sec', 'Total timeout for one VK API request.', 'VK'),
|
||||
('vk_api_timeout_connect_sec', '5'::jsonb, 'int', 'VK connect timeout, sec', 'Connection timeout for VK API.', 'VK'),
|
||||
('vk_rate_limit_sleep_sec', '1'::jsonb, 'float', 'VK rate limit sleep, sec', 'Sleep after VK API rate limit error code 6.', 'VK'),
|
||||
('vk_api_retry_attempts', '3'::jsonb, 'int', 'VK retry attempts', 'Attempts for temporary VK API/network errors.', 'VK'),
|
||||
('vk_api_retry_min_delay_sec', '2'::jsonb, 'float', 'VK retry min delay, sec', 'Initial retry delay for temporary VK errors.', 'VK'),
|
||||
('vk_api_retry_max_delay_sec', '10'::jsonb, 'float', 'VK retry max delay, sec', 'Maximum retry delay for temporary VK errors.', 'VK'),
|
||||
('uploader_interval_sec', '5'::jsonb, 'int', 'Uploader idle interval, sec', 'Pause when no jobs are available.', 'Uploader'),
|
||||
('uploader_download_timeout_sec', '45'::jsonb, 'int', 'Download timeout, sec', 'Timeout for source media download.', 'Uploader')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
@@ -0,0 +1,27 @@
|
||||
ALTER TABLE raw_posts
|
||||
ADD COLUMN IF NOT EXISTS tg_storage_chat_id BIGINT,
|
||||
ADD COLUMN IF NOT EXISTS tg_storage_thread_id BIGINT,
|
||||
ADD COLUMN IF NOT EXISTS tg_storage_message_ids BIGINT[] NOT NULL DEFAULT ARRAY[]::BIGINT[],
|
||||
ADD COLUMN IF NOT EXISTS tg_storage_meta_message_id BIGINT;
|
||||
|
||||
ALTER TABLE raw_post_media
|
||||
ADD COLUMN IF NOT EXISTS tg_file_id TEXT,
|
||||
ADD COLUMN IF NOT EXISTS tg_file_unique_id TEXT;
|
||||
|
||||
INSERT INTO app_settings(key, value_json, value_type, title, description, category)
|
||||
VALUES
|
||||
('tg_media_channel_id', '""'::jsonb, 'str', 'Telegram media storage chat', 'Chat id or chat_id:topic_id for raw media storage.', 'Telegram'),
|
||||
('local_bot_api_url', '""'::jsonb, 'str', 'Local Telegram Bot API URL', 'Optional local Bot API server URL.', 'Telegram'),
|
||||
('telegram_media_group_max_items', '10'::jsonb, 'int', 'Telegram media group max items', 'Maximum media items per Telegram media group.', 'Telegram'),
|
||||
('telegram_media_upload_delay_sec', '1'::jsonb, 'float', 'Telegram media upload delay, sec', 'Pause between Telegram upload operations.', 'Telegram'),
|
||||
('telegram_retry_max_attempts', '4'::jsonb, 'int', 'Telegram retry attempts', 'Retry attempts for Telegram calls.', 'Telegram'),
|
||||
('telegram_retry_backoff_max_sec', '15'::jsonb, 'int', 'Telegram retry max backoff, sec', 'Maximum retry backoff for Telegram calls.', 'Telegram'),
|
||||
('telegram_caption_limit', '1024'::jsonb, 'int', 'Telegram caption limit', 'Caption character limit for media messages.', 'Telegram'),
|
||||
('telegram_message_limit', '4096'::jsonb, 'int', 'Telegram message limit', 'Text message character limit.', 'Telegram'),
|
||||
('telegram_text_overflow_marker', '"Продолжение следующим сообщением."'::jsonb, 'str', 'Telegram overflow marker', 'Caption marker when text does not fit.', 'Telegram'),
|
||||
('media_post_job_pause_sec', '0.2'::jsonb, 'float', 'Post job pause, sec', 'Pause after each processed media storage job.', 'Uploader'),
|
||||
('video_max_size_mb', '49'::jsonb, 'int', 'Video max size, MB', 'Maximum video size to upload to Telegram.', 'Uploader'),
|
||||
('video_max_duration_sec', '300'::jsonb, 'int', 'Video max duration, sec', 'Maximum video duration to upload to Telegram.', 'Uploader'),
|
||||
('uploader_yt_dlp_timeout_sec', '300'::jsonb, 'int', 'yt-dlp timeout, sec', 'Timeout for VK video downloads.', 'Uploader'),
|
||||
('uploader_max_media_attempts', '3'::jsonb, 'int', 'Media max attempts', 'Maximum attempts per media item.', 'Uploader')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
@@ -0,0 +1,16 @@
|
||||
INSERT INTO app_settings(key, value_json, value_type, title, description, category)
|
||||
VALUES
|
||||
('parser_skip_reposts', 'true'::jsonb, 'bool', 'Skip reposts', 'Do not save reposts from VK copy_history.', 'Parser'),
|
||||
('parser_skip_empty_text', 'true'::jsonb, 'bool', 'Skip empty text posts', 'Treat posts without text as junk.', 'Parser'),
|
||||
('parser_skip_no_media', 'true'::jsonb, 'bool', 'Skip posts without media', 'Treat posts without media as junk.', 'Parser'),
|
||||
('parser_skip_text_too_short', 'true'::jsonb, 'bool', 'Skip short text posts', 'Treat posts shorter than parser_min_text_length as junk.', 'Parser'),
|
||||
('parser_store_skipped_posts', 'false'::jsonb, 'bool', 'Store skipped posts', 'Keep junk posts in raw_posts instead of dropping them.', 'Parser'),
|
||||
('parser_dedupe_content_hash', 'true'::jsonb, 'bool', 'Dedupe by content hash', 'Skip posts whose text and media hash already exists in raw_posts.', 'Parser'),
|
||||
('parser_source_pause_sec', '0'::jsonb, 'float', 'Source pause, sec', 'Pause between VK sources inside one parser cycle.', 'Parser'),
|
||||
('vk_api_timeout_total_sec', '15'::jsonb, 'int', 'VK total timeout, sec', 'Total timeout for one VK API request.', 'VK'),
|
||||
('vk_api_timeout_connect_sec', '5'::jsonb, 'int', 'VK connect timeout, sec', 'Connection timeout for VK API.', 'VK'),
|
||||
('vk_rate_limit_sleep_sec', '1'::jsonb, 'float', 'VK rate limit sleep, sec', 'Sleep after VK API rate limit error code 6.', 'VK'),
|
||||
('vk_api_retry_attempts', '3'::jsonb, 'int', 'VK retry attempts', 'Attempts for temporary VK API/network errors.', 'VK'),
|
||||
('vk_api_retry_min_delay_sec', '2'::jsonb, 'float', 'VK retry min delay, sec', 'Initial retry delay for temporary VK errors.', 'VK'),
|
||||
('vk_api_retry_max_delay_sec', '10'::jsonb, 'float', 'VK retry max delay, sec', 'Maximum retry delay for temporary VK errors.', 'VK')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
@@ -0,0 +1,28 @@
|
||||
INSERT INTO app_settings(key, value_json, value_type, title, description, category)
|
||||
VALUES
|
||||
('parser_skip_empty_text', 'true'::jsonb, 'bool', 'Skip empty text posts', 'Treat posts without text as junk.', 'Parser')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
UPDATE app_settings
|
||||
SET value_json='50'::jsonb,
|
||||
description='Posts with shorter text are treated as junk.',
|
||||
updated_at=NOW()
|
||||
WHERE key='parser_min_text_length'
|
||||
AND value_json='0'::jsonb;
|
||||
|
||||
UPDATE app_settings
|
||||
SET value_json='false'::jsonb,
|
||||
description='Keep junk posts in raw_posts instead of dropping them.',
|
||||
updated_at=NOW()
|
||||
WHERE key='parser_store_skipped_posts'
|
||||
AND value_json='true'::jsonb;
|
||||
|
||||
UPDATE app_settings
|
||||
SET description='Treat posts without media as junk.',
|
||||
updated_at=NOW()
|
||||
WHERE key='parser_skip_no_media';
|
||||
|
||||
UPDATE app_settings
|
||||
SET description='Treat posts shorter than parser_min_text_length as junk.',
|
||||
updated_at=NOW()
|
||||
WHERE key='parser_skip_text_too_short';
|
||||
@@ -0,0 +1,50 @@
|
||||
ALTER TABLE raw_posts
|
||||
ADD COLUMN IF NOT EXISTS qualification_status TEXT,
|
||||
ADD COLUMN IF NOT EXISTS qualification_score INTEGER,
|
||||
ADD COLUMN IF NOT EXISTS qualification_decision TEXT,
|
||||
ADD COLUMN IF NOT EXISTS qualification_reason TEXT,
|
||||
ADD COLUMN IF NOT EXISTS qualification_model TEXT,
|
||||
ADD COLUMN IF NOT EXISTS qualification_prompt_hash TEXT,
|
||||
ADD COLUMN IF NOT EXISTS qualification_batch_id BIGINT,
|
||||
ADD COLUMN IF NOT EXISTS qualified_at TIMESTAMPTZ;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_posts_qualification_claim
|
||||
ON raw_posts(status, qualification_status, created_at)
|
||||
WHERE status='storage_ready';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ai_qualification_batches (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
status TEXT NOT NULL DEFAULT 'processing',
|
||||
provider TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
prompt_hash TEXT NOT NULL,
|
||||
posts_count INTEGER NOT NULL DEFAULT 0,
|
||||
accepted_count INTEGER NOT NULL DEFAULT 0,
|
||||
rejected_count INTEGER NOT NULL DEFAULT 0,
|
||||
maybe_count INTEGER NOT NULL DEFAULT 0,
|
||||
request_json JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
response_json JSONB,
|
||||
error TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
completed_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
INSERT INTO worker_controls(name, enabled, settings_json)
|
||||
VALUES ('ai-qualifier', FALSE, '{}'::jsonb)
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
|
||||
INSERT INTO app_settings(key, value_json, value_type, title, description, category)
|
||||
VALUES
|
||||
('ai_qualifier_enabled', 'false'::jsonb, 'bool', 'AI qualifier enabled', 'Enable qualification requests to AI API.', 'AI Qualifier'),
|
||||
('ai_qualifier_provider', '"openrouter"'::jsonb, 'str', 'Provider', 'LiteLLM provider: openrouter, openai, anthropic, gemini, openai_compatible.', 'AI Qualifier'),
|
||||
('ai_qualifier_model', '"openrouter/openai/gpt-4.1-mini"'::jsonb, 'str', 'Model', 'Model name in LiteLLM format.', 'AI Qualifier'),
|
||||
('ai_qualifier_api_key', '""'::jsonb, 'secret', 'API key', 'AI provider API key.', 'AI Qualifier'),
|
||||
('ai_qualifier_api_base', '""'::jsonb, 'str', 'API base', 'Optional base URL for OpenAI-compatible providers.', 'AI Qualifier'),
|
||||
('ai_qualifier_batch_size', '30'::jsonb, 'int', 'Batch size', 'How many raw posts to classify in one request.', 'AI Qualifier'),
|
||||
('ai_qualifier_min_score', '7'::jsonb, 'int', 'Minimum accepted score', 'Posts with score >= this value are accepted.', 'AI Qualifier'),
|
||||
('ai_qualifier_interval_sec', '60'::jsonb, 'int', 'Idle interval, sec', 'Pause between qualifier checks.', 'AI Qualifier'),
|
||||
('ai_qualifier_max_text_chars', '2000'::jsonb, 'int', 'Max text chars per post', 'Trim raw text before sending it to AI.', 'AI Qualifier'),
|
||||
('ai_qualifier_temperature', '0'::jsonb, 'float', 'Temperature', 'AI sampling temperature for stable classification.', 'AI Qualifier'),
|
||||
('ai_qualifier_timeout_sec', '120'::jsonb, 'int', 'Timeout, sec', 'AI request timeout.', 'AI Qualifier'),
|
||||
('ai_qualifier_prompt', '"Ты квалификатор постов для проекта, который собирает посты производителей и магазинов милитари, тактической и airsoft-снаряги.\\n\\nОцени каждый raw-пост по пригодности для дальнейшего рерайта и публикации.\\n\\nПодходит: товарные посты, новинки, обзоры, производство, экипировка, тактическая одежда, броня, подсумки, рюкзаки, разгрузки, ремни, патчи, аксессуары, airsoft/страйкбольная снаряга, полезные анонсы производителя.\\n\\nМусор: мемы, поздравления, вакансии, политика, бытовуха, пустые посты, посты без конкретного товара/снаряжения, розыгрыши без предметного контента, чистая реклама скидки без понятного товара, нерелевантные новости.\\n\\nВерни только JSON-объект по схеме: {\\\"results\\\":[{\\\"id\\\":123,\\\"score\\\":8,\\\"decision\\\":\\\"accepted\\\",\\\"reason\\\":\\\"короткая причина\\\"}]}. score — целое число от 1 до 10. decision: accepted, rejected или maybe. Не добавляй markdown и пояснения вне JSON."'::jsonb, 'text', 'Prompt', 'Main qualification prompt.', 'AI Qualifier')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
@@ -0,0 +1,47 @@
|
||||
UPDATE app_settings SET title='Интервал парсера, сек', description='Пауза между полными циклами парсинга источников.' WHERE key='parser_interval_sec';
|
||||
UPDATE app_settings SET title='Глубина нового источника, дней', description='Сколько дней назад читать посты при первом запуске источника.' WHERE key='parser_new_source_lookback_days';
|
||||
UPDATE app_settings SET title='Перекрытие повторного чтения, мин', description='Насколько назад перечитывать источник, чтобы не пропустить посты.' WHERE key='parser_reparse_overlap_minutes';
|
||||
UPDATE app_settings SET title='Минимальная длина текста', description='Посты с текстом короче этого значения считаются мусором.' WHERE key='parser_min_text_length';
|
||||
UPDATE app_settings SET title='Пропускать репосты', description='Не брать VK-посты с copy_history.' WHERE key='parser_skip_reposts';
|
||||
UPDATE app_settings SET title='Пропускать пустой текст', description='Не брать посты без текста.' WHERE key='parser_skip_empty_text';
|
||||
UPDATE app_settings SET title='Пропускать без медиа', description='Не брать посты без фото/видео.' WHERE key='parser_skip_no_media';
|
||||
UPDATE app_settings SET title='Пропускать короткий текст', description='Не брать посты короче минимальной длины текста.' WHERE key='parser_skip_text_too_short';
|
||||
UPDATE app_settings SET title='Сохранять мусорные посты', description='Если включено, мусорные посты попадут в raw_posts со статусом skipped. Обычно выключено.' WHERE key='parser_store_skipped_posts';
|
||||
UPDATE app_settings SET title='Дедупликация по контенту', description='Не брать пост, если такой текст и набор медиа уже есть в raw_posts.' WHERE key='parser_dedupe_content_hash';
|
||||
UPDATE app_settings SET title='Пауза между источниками, сек', description='Задержка между чтением разных VK-источников внутри одного цикла.' WHERE key='parser_source_pause_sec';
|
||||
|
||||
UPDATE app_settings SET title='VK запросов в секунду', description='Ограничение частоты запросов к VK API.' WHERE key='vk_requests_per_second';
|
||||
UPDATE app_settings SET title='Размер страницы VK wall.get', description='Сколько постов запрашивать за один вызов wall.get.' WHERE key='vk_wall_page_size';
|
||||
UPDATE app_settings SET title='VK общий таймаут, сек', description='Общий таймаут одного запроса к VK API.' WHERE key='vk_api_timeout_total_sec';
|
||||
UPDATE app_settings SET title='VK таймаут соединения, сек', description='Таймаут установки соединения с VK API.' WHERE key='vk_api_timeout_connect_sec';
|
||||
UPDATE app_settings SET title='Пауза при VK rate limit, сек', description='Сколько ждать после ошибки VK API error_code=6.' WHERE key='vk_rate_limit_sleep_sec';
|
||||
UPDATE app_settings SET title='VK retry попытки', description='Сколько раз повторять временно упавший VK-запрос.' WHERE key='vk_api_retry_attempts';
|
||||
UPDATE app_settings SET title='VK retry минимум, сек', description='Начальная задержка между повторными VK-запросами.' WHERE key='vk_api_retry_min_delay_sec';
|
||||
UPDATE app_settings SET title='VK retry максимум, сек', description='Максимальная задержка между повторными VK-запросами.' WHERE key='vk_api_retry_max_delay_sec';
|
||||
|
||||
UPDATE app_settings SET title='Интервал аплоадера, сек', description='Пауза, когда нет задач на загрузку raw-постов в Telegram.' WHERE key='uploader_interval_sec';
|
||||
UPDATE app_settings SET title='Таймаут скачивания, сек', description='Таймаут скачивания исходного медиа.' WHERE key='uploader_download_timeout_sec';
|
||||
UPDATE app_settings SET title='Telegram канал raw', description='ID Telegram-канала/топика для хранения raw-постов.' WHERE key='tg_media_channel_id';
|
||||
UPDATE app_settings SET title='Локальный Bot API URL', description='Опциональный URL локального Telegram Bot API сервера.' WHERE key='local_bot_api_url';
|
||||
UPDATE app_settings SET title='Задержка на медиа, сек', description='Пауза после отправки медиа, умножается на количество файлов.' WHERE key='media_upload_delay_sec';
|
||||
UPDATE app_settings SET title='Пауза после поста, сек', description='Короткая пауза после полной обработки одного raw-поста.' WHERE key='media_post_job_pause_sec';
|
||||
UPDATE app_settings SET title='Максимум медиа в группе', description='Максимум файлов в одном Telegram media group.' WHERE key='media_group_max_items';
|
||||
UPDATE app_settings SET title='Максимум попыток медиа', description='Сколько раз пытаться скачать/загрузить отдельное медиа.' WHERE key='max_media_attempts';
|
||||
UPDATE app_settings SET title='Telegram retry попытки', description='Сколько раз повторять временно упавший Telegram-запрос.' WHERE key='tg_retry_attempts';
|
||||
UPDATE app_settings SET title='Telegram backoff максимум, сек', description='Максимальная задержка между retry Telegram-запросов.' WHERE key='tg_retry_backoff_max_sec';
|
||||
UPDATE app_settings SET title='Максимум видео, МБ', description='Максимальный размер видео для загрузки в Telegram.' WHERE key='video_max_size_mb';
|
||||
UPDATE app_settings SET title='Максимум видео, сек', description='Видео длиннее этого значения помечается как link_only.' WHERE key='video_max_duration_sec';
|
||||
UPDATE app_settings SET title='yt-dlp таймаут, сек', description='Сколько ждать скачивания VK-видео через yt-dlp.' WHERE key='uploader_yt_dlp_timeout_sec';
|
||||
|
||||
UPDATE app_settings SET title='Включить AI-квалификатор', description='Разрешить воркеру отправлять raw-посты в AI API.' WHERE key='ai_qualifier_enabled';
|
||||
UPDATE app_settings SET title='Провайдер LLM', description='Провайдер для LiteLLM. Для OpenRouter список моделей подгружается live.' WHERE key='ai_qualifier_provider';
|
||||
UPDATE app_settings SET title='Модель LLM', description='Модель для квалификации. Можно выбрать из списка или вписать вручную.' WHERE key='ai_qualifier_model';
|
||||
UPDATE app_settings SET title='API токен', description='Токен выбранного LLM-провайдера. Сохраняется в настройках.' WHERE key='ai_qualifier_api_key';
|
||||
UPDATE app_settings SET title='API base URL', description='Опциональный base URL для OpenAI-compatible провайдера.' WHERE key='ai_qualifier_api_base';
|
||||
UPDATE app_settings SET title='Размер пачки', description='Сколько raw-постов отправлять в один AI-запрос.' WHERE key='ai_qualifier_batch_size';
|
||||
UPDATE app_settings SET title='Минимальная оценка', description='Пост принимается, если score больше или равен этому значению.' WHERE key='ai_qualifier_min_score';
|
||||
UPDATE app_settings SET title='Интервал проверки, сек', description='Пауза между проверками новых raw-постов.' WHERE key='ai_qualifier_interval_sec';
|
||||
UPDATE app_settings SET title='Максимум символов текста', description='Сколько символов raw_text отправлять в AI на один пост.' WHERE key='ai_qualifier_max_text_chars';
|
||||
UPDATE app_settings SET title='Температура', description='Температура генерации. Для классификации обычно 0.' WHERE key='ai_qualifier_temperature';
|
||||
UPDATE app_settings SET title='AI таймаут, сек', description='Таймаут одного AI-запроса.' WHERE key='ai_qualifier_timeout_sec';
|
||||
UPDATE app_settings SET title='Промпт квалификатора', description='Основная инструкция для оценки пригодности raw-постов.' WHERE key='ai_qualifier_prompt';
|
||||
@@ -0,0 +1,57 @@
|
||||
ALTER TABLE ai_qualification_batches
|
||||
ADD COLUMN IF NOT EXISTS prompt_tokens INTEGER,
|
||||
ADD COLUMN IF NOT EXISTS completion_tokens INTEGER,
|
||||
ADD COLUMN IF NOT EXISTS total_tokens INTEGER,
|
||||
ADD COLUMN IF NOT EXISTS estimated_cost_usd NUMERIC(12, 6);
|
||||
|
||||
ALTER TABLE raw_posts
|
||||
ADD COLUMN IF NOT EXISTS rewrite_status TEXT,
|
||||
ADD COLUMN IF NOT EXISTS rewritten_text TEXT,
|
||||
ADD COLUMN IF NOT EXISTS rewrite_notes TEXT,
|
||||
ADD COLUMN IF NOT EXISTS rewrite_model TEXT,
|
||||
ADD COLUMN IF NOT EXISTS rewrite_prompt_hash TEXT,
|
||||
ADD COLUMN IF NOT EXISTS rewrite_batch_id BIGINT,
|
||||
ADD COLUMN IF NOT EXISTS rewritten_at TIMESTAMPTZ;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_posts_writer_claim
|
||||
ON raw_posts(qualification_status, rewrite_status, qualified_at, id)
|
||||
WHERE qualification_status='accepted';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ai_writer_batches (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
status TEXT NOT NULL DEFAULT 'processing',
|
||||
provider TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
prompt_hash TEXT NOT NULL,
|
||||
posts_count INTEGER NOT NULL DEFAULT 0,
|
||||
ready_count INTEGER NOT NULL DEFAULT 0,
|
||||
failed_count INTEGER NOT NULL DEFAULT 0,
|
||||
request_json JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
response_json JSONB,
|
||||
error TEXT,
|
||||
prompt_tokens INTEGER,
|
||||
completion_tokens INTEGER,
|
||||
total_tokens INTEGER,
|
||||
estimated_cost_usd NUMERIC(12, 6),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
completed_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
INSERT INTO worker_controls(name, enabled, settings_json)
|
||||
VALUES ('ai-writer', FALSE, '{}'::jsonb)
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
|
||||
INSERT INTO app_settings(key, value_json, value_type, title, description, category)
|
||||
VALUES
|
||||
('ai_writer_enabled', 'false'::jsonb, 'bool', 'AI-райтер включён', 'Разрешает отправлять прошедшие квалификацию посты на рерайт.', 'AI Writer'),
|
||||
('ai_writer_provider', '"anthropic"'::jsonb, 'str', 'Провайдер LLM', 'Через какого провайдера LiteLLM отправлять запросы райтера.', 'AI Writer'),
|
||||
('ai_writer_model', '"claude-haiku-4-5-20251001"'::jsonb, 'str', 'Модель', 'Модель для рерайта. Для отладки используем дешёвую Haiku.', 'AI Writer'),
|
||||
('ai_writer_api_key', '""'::jsonb, 'secret', 'API-ключ', 'Ключ провайдера для AI-райтера. Можно использовать тот же Anthropic-ключ, что и у квалификатора.', 'AI Writer'),
|
||||
('ai_writer_api_base', '""'::jsonb, 'str', 'API base URL', 'Необязательный base URL для OpenAI-compatible провайдеров.', 'AI Writer'),
|
||||
('ai_writer_prompt', '"Ты редактор Telegram/VK-канала про милитари, тактическую и airsoft-снарягу. Перепиши исходный пост производителя в готовый к публикации пост на русском языке.\\n\\nСтиль: живой, понятный, без канцелярита, без чрезмерной рекламы, без выдуманных фактов. Сохраняй конкретику: модель, материал, цвета, размеры, назначение, цену и условия, если они есть в исходнике. Не добавляй то, чего нет в исходном посте.\\n\\nФормат результата: 1 короткий заголовок первой строкой, затем 2-5 абзацев или аккуратный список, если исходник товарный. В конце не добавляй ссылки и хэштеги.\\n\\nВерни только JSON-объект по схеме: {\\\"rewrites\\\":[{\\\"id\\\":123,\\\"text\\\":\\\"готовый текст\\\",\\\"notes\\\":\\\"короткая заметка для редактора\\\"}]}. Без markdown вне JSON."'::jsonb, 'text', 'Промпт райтера', 'Главный промпт рерайта. Поддерживает обычные переносы строк.', 'AI Writer'),
|
||||
('ai_writer_batch_size', '1'::jsonb, 'int', 'Размер batch', 'Сколько постов переписывать за один запрос. Для отладки лучше 1.', 'AI Writer'),
|
||||
('ai_writer_max_text_chars', '3500'::jsonb, 'int', 'Максимум символов исходника', 'Обрезает исходный raw-текст перед отправкой в LLM.', 'AI Writer'),
|
||||
('ai_writer_temperature', '0.4'::jsonb, 'float', 'Температура генерации', 'Ниже — стабильнее и суше, выше — свободнее и разнообразнее.', 'AI Writer'),
|
||||
('ai_writer_timeout_sec', '180'::jsonb, 'int', 'Таймаут, сек', 'Сколько ждать ответ модели.', 'AI Writer'),
|
||||
('ai_writer_interval_sec', '60'::jsonb, 'int', 'Интервал ожидания, сек', 'Пауза между проверками очереди writer, когда работы нет.', 'AI Writer')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
@@ -0,0 +1,5 @@
|
||||
ALTER TABLE ai_qualification_batches
|
||||
ADD COLUMN IF NOT EXISTS prompt_text TEXT;
|
||||
|
||||
ALTER TABLE ai_writer_batches
|
||||
ADD COLUMN IF NOT EXISTS prompt_text TEXT;
|
||||
@@ -0,0 +1,6 @@
|
||||
ALTER TABLE raw_posts
|
||||
ADD COLUMN IF NOT EXISTS qualification_reject_tag TEXT;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_posts_qualification_reject_tag
|
||||
ON raw_posts(qualification_reject_tag)
|
||||
WHERE qualification_reject_tag IS NOT NULL;
|
||||
@@ -0,0 +1,53 @@
|
||||
ALTER TABLE sources
|
||||
ADD COLUMN IF NOT EXISTS tag TEXT;
|
||||
|
||||
UPDATE sources
|
||||
SET tag = lower(regexp_replace(COALESCE(NULLIF(external_id, ''), name), '[^[:alnum:]_]+', '_', 'g'))
|
||||
WHERE tag IS NULL OR btrim(tag) = '';
|
||||
|
||||
ALTER TABLE raw_posts
|
||||
ADD COLUMN IF NOT EXISTS rewrite_category TEXT,
|
||||
ADD COLUMN IF NOT EXISTS rewrite_category_tag TEXT,
|
||||
ADD COLUMN IF NOT EXISTS rewrite_source_tag TEXT;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_sources_tag ON sources(lower(tag))
|
||||
WHERE archived_at IS NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_posts_rewrite_category ON raw_posts(rewrite_category)
|
||||
WHERE rewrite_category IS NOT NULL;
|
||||
|
||||
INSERT INTO app_settings(key, value_json, value_type, title, description, category)
|
||||
VALUES
|
||||
('ai_writer_categories', '["защита","одежда","разгрузка","рюкзаки","airsoft","патчи","электроника","аксессуары","производство"]'::jsonb, 'text', 'Категории райтера', 'Список категорий, из которых AI-райтер выбирает одну. Можно писать через запятую или с новой строки.', 'AI Writer')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
UPDATE app_settings
|
||||
SET value_json = to_jsonb($prompt$You are an editor of a Russian Telegram channel about military, tactical and airsoft gear.
|
||||
Write on behalf of the channel — as an expert observer, not the manufacturer.
|
||||
Source posts come from manufacturers/shops; your job is to reframe them as channel news.
|
||||
|
||||
VOICE: concise, expert, matter-of-fact. Like telling a knowledgeable friend what dropped.
|
||||
Example opener framing: "У [Producer] вышел...", "Wartech показали...", "[Producer] добавили в линейку..."
|
||||
|
||||
RULES:
|
||||
- Use only facts from source text. Never invent specs, properties or claims.
|
||||
- If source is thin — write a short post, do not pad it.
|
||||
- No hype, no exclamation spam, no manufacturer PR tone.
|
||||
- Post length: 3-6 lines. Short is better than padded.
|
||||
- No bullet lists unless source has 4+ distinct specs worth listing.
|
||||
- Do not add hashtags. The system will add category and producer hashtags separately.
|
||||
|
||||
CATEGORIES: pick exactly one from the categories field in input.
|
||||
|
||||
INPUT fields used: id, producer_name, producer_tag, text.
|
||||
qualification_score and media_count are metadata; ignore them for rewrite unless they help understand context.
|
||||
|
||||
OUTPUT: JSON only, no markdown outside JSON.$prompt$::text),
|
||||
description = 'Главный промпт рерайта. Модель выбирает category, но не пишет хэштеги.',
|
||||
updated_at = NOW()
|
||||
WHERE key = 'ai_writer_prompt'
|
||||
AND (
|
||||
value_json::text ILIKE '%хэштег%'
|
||||
OR value_json::text ILIKE '%hashtag%'
|
||||
OR value_json::text ILIKE '%#CATEGORY%'
|
||||
);
|
||||
@@ -0,0 +1,39 @@
|
||||
UPDATE app_settings
|
||||
SET title='Инструкция квалификатора',
|
||||
description='Редактируемая человеческая инструкция: критерии оценки и мусора. JSON-контракт ответа добавляется кодом автоматически.'
|
||||
WHERE key='ai_qualifier_prompt';
|
||||
|
||||
UPDATE app_settings
|
||||
SET title='Инструкция райтера',
|
||||
description='Редактируемая человеческая инструкция: стиль, факты, длина и голос канала. Категорию и JSON-контракт ответа контролирует код.'
|
||||
WHERE key='ai_writer_prompt';
|
||||
|
||||
UPDATE app_settings
|
||||
SET description='Модель для квалификации. Список берётся live из API провайдера, если доступен ключ; fallback помечается отдельно.'
|
||||
WHERE key='ai_qualifier_model';
|
||||
|
||||
UPDATE app_settings
|
||||
SET description='Модель для рерайта. Список берётся live из API провайдера, если доступен ключ; fallback помечается отдельно.'
|
||||
WHERE key='ai_writer_model';
|
||||
|
||||
UPDATE app_settings
|
||||
SET description='Токен выбранного LLM-провайдера. В админке показывается полностью, поэтому доступ к настройкам должен быть только у доверенных пользователей.'
|
||||
WHERE key IN ('ai_qualifier_api_key', 'ai_writer_api_key');
|
||||
|
||||
UPDATE app_settings m
|
||||
SET value_json = to_jsonb('anthropic/' || trim(both '"' from m.value_json::text))
|
||||
FROM app_settings p
|
||||
WHERE p.key = replace(m.key, '_model', '_provider')
|
||||
AND p.value_json = '"anthropic"'::jsonb
|
||||
AND m.key IN ('ai_qualifier_model', 'ai_writer_model')
|
||||
AND m.value_json::text NOT LIKE '"anthropic/%'
|
||||
AND m.value_json::text NOT LIKE '"%/%"';
|
||||
|
||||
UPDATE app_settings m
|
||||
SET value_json = to_jsonb('gemini/' || trim(both '"' from m.value_json::text))
|
||||
FROM app_settings p
|
||||
WHERE p.key = replace(m.key, '_model', '_provider')
|
||||
AND p.value_json = '"gemini"'::jsonb
|
||||
AND m.key IN ('ai_qualifier_model', 'ai_writer_model')
|
||||
AND m.value_json::text NOT LIKE '"gemini/%'
|
||||
AND m.value_json::text NOT LIKE '"%/%"';
|
||||
@@ -0,0 +1,15 @@
|
||||
ALTER TABLE raw_posts
|
||||
ADD COLUMN IF NOT EXISTS qualification_model_decision TEXT;
|
||||
|
||||
UPDATE raw_posts rp
|
||||
SET qualification_model_decision = item->>'decision'
|
||||
FROM ai_qualification_batches b,
|
||||
jsonb_array_elements(COALESCE(b.response_json->'results', '[]'::jsonb)) AS item
|
||||
WHERE rp.qualification_batch_id = b.id
|
||||
AND (item->>'id')::bigint = rp.id
|
||||
AND rp.qualification_model_decision IS NULL
|
||||
AND item ? 'decision';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_posts_qualification_model_decision
|
||||
ON raw_posts(qualification_model_decision)
|
||||
WHERE qualification_model_decision IS NOT NULL;
|
||||
@@ -0,0 +1,32 @@
|
||||
ALTER TABLE raw_posts
|
||||
ADD COLUMN IF NOT EXISTS editorial_status TEXT,
|
||||
ADD COLUMN IF NOT EXISTS final_text TEXT,
|
||||
ADD COLUMN IF NOT EXISTS final_category TEXT,
|
||||
ADD COLUMN IF NOT EXISTS final_category_tag TEXT,
|
||||
ADD COLUMN IF NOT EXISTS final_source_tag TEXT,
|
||||
ADD COLUMN IF NOT EXISTS editor_notes TEXT,
|
||||
ADD COLUMN IF NOT EXISTS regeneration_prompt TEXT,
|
||||
ADD COLUMN IF NOT EXISTS reviewed_by BIGINT REFERENCES admin_users(id) ON DELETE SET NULL,
|
||||
ADD COLUMN IF NOT EXISTS reviewed_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS edited_at TIMESTAMPTZ;
|
||||
|
||||
UPDATE raw_posts
|
||||
SET editorial_status = 'review'
|
||||
WHERE editorial_status IS NULL
|
||||
AND rewrite_status = 'ready';
|
||||
|
||||
UPDATE raw_posts
|
||||
SET final_text = rewritten_text,
|
||||
final_category = rewrite_category,
|
||||
final_category_tag = rewrite_category_tag,
|
||||
final_source_tag = rewrite_source_tag
|
||||
WHERE rewrite_status = 'ready'
|
||||
AND final_text IS NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_posts_editorial_status
|
||||
ON raw_posts(editorial_status, rewritten_at DESC, id)
|
||||
WHERE rewrite_status = 'ready';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_posts_final_category
|
||||
ON raw_posts(final_category)
|
||||
WHERE final_category IS NOT NULL;
|
||||
@@ -0,0 +1,43 @@
|
||||
INSERT INTO app_settings(key, value_json, value_type, title, description, category)
|
||||
VALUES
|
||||
(
|
||||
'ai_qualifier_contract',
|
||||
'"OUTPUT SCHEMA (return array matching input order):\n{\"results\":[{\"id\":123,\"score\":8,\"decision\":\"accepted\",\"reason\":\"до 10 слов на русском\",\"reject_tag\":null}]}\n\nRules:\n- Input is a JSON array of posts.\n- Return JSON only. No markdown. No text outside JSON.\n- score must be integer 1..10.\n- decision must be exactly one of: \"accepted\", \"rejected\", \"maybe\".\n- reject_tag (rejected/maybe only) must be one of: \"meme\", \"no_product\", \"politics\", \"discount_only\", \"off_topic\", \"vacancy\", \"low_content\", \"wrong_language\", \"injection\", \"weapon\", null.\n- Return one result for every input post id."'::jsonb,
|
||||
'text',
|
||||
'Технический контракт квалификатора',
|
||||
'JSON-схема и строгие правила ответа. Обычно не меняется при правке смыслового промпта.',
|
||||
'AI Qualifier'
|
||||
),
|
||||
(
|
||||
'ai_writer_contract',
|
||||
'"OUTPUT SCHEMA (return array matching input order):\n{\"rewrites\":[{\"id\":123,\"category\":\"разгрузка\",\"text\":\"готовый текст без хэштегов\",\"notes\":\"короткая заметка для редактора или null\"}]}\n\nRules:\n- Input is a JSON object with key \"posts\" containing accepted posts.\n- Each post includes producer_name and producer_tag. Use producer_name when it helps, but do not invent facts.\n- If post includes editor_regeneration_prompt, follow it as an additional editor instruction while still obeying facts and output schema.\n- Do not add hashtags to rewritten text. Pick exactly one category from the categories list in input.\n- Return JSON only. No markdown. No text outside JSON.\n- Return one rewrite for every input post id."'::jsonb,
|
||||
'text',
|
||||
'Технический контракт райтера',
|
||||
'JSON-схема, выбор категории и запрет хэштегов в тексте. Обычно не меняется при правке стиля.',
|
||||
'AI Writer'
|
||||
)
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
UPDATE app_settings
|
||||
SET title='Свободная инструкция квалификатора',
|
||||
description='Роль, тон, критерии и примеры оценки. Технический JSON-контракт вынесен отдельно.'
|
||||
WHERE key='ai_qualifier_prompt';
|
||||
|
||||
UPDATE app_settings
|
||||
SET title='Свободная инструкция райтера',
|
||||
description='Стиль, голос канала, правила фактов и примеры текста. Технический JSON-контракт вынесен отдельно.'
|
||||
WHERE key='ai_writer_prompt';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS admin_login_attempts (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
ip TEXT NOT NULL,
|
||||
login TEXT NOT NULL DEFAULT '',
|
||||
success BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_admin_login_attempts_ip_time
|
||||
ON admin_login_attempts(ip, created_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_admin_login_attempts_login_time
|
||||
ON admin_login_attempts(login, created_at DESC);
|
||||
@@ -0,0 +1,9 @@
|
||||
ALTER TABLE raw_post_media
|
||||
ADD COLUMN IF NOT EXISTS editor_hidden BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
ADD COLUMN IF NOT EXISTS editor_added BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
ADD COLUMN IF NOT EXISTS editor_uploaded_by BIGINT REFERENCES admin_users(id) ON DELETE SET NULL,
|
||||
ADD COLUMN IF NOT EXISTS editor_uploaded_at TIMESTAMPTZ;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_post_media_editor_visible
|
||||
ON raw_post_media(raw_post_id, sort_order, id)
|
||||
WHERE editor_hidden = FALSE;
|
||||
@@ -0,0 +1,56 @@
|
||||
UPDATE raw_posts
|
||||
SET rewrite_status='ready',
|
||||
editorial_status='review',
|
||||
regeneration_prompt=NULL,
|
||||
updated_at=NOW()
|
||||
WHERE COALESCE(editorial_status, '')='regenerating';
|
||||
|
||||
UPDATE app_settings
|
||||
SET value_json = to_jsonb($prompt$
|
||||
Ты редактор русского Telegram-канала про тактическую, милитари и airsoft-экипировку.
|
||||
Пиши от лица канала: как экспертный наблюдатель, а не как производитель или магазин.
|
||||
|
||||
Задача: превратить исходный пост в короткую новостную публикацию для канала.
|
||||
|
||||
Стиль:
|
||||
- коротко, уверенно, по делу;
|
||||
- без рекламного восторга и без выдуманных характеристик;
|
||||
- только факты из исходного текста;
|
||||
- если фактов мало, делай короткий пост, не растягивай;
|
||||
- текст должен быть структурированным, не простынёй.
|
||||
|
||||
Формат текста:
|
||||
- 3-6 коротких строк или 2-4 компактных абзаца;
|
||||
- можно использовать 1-3 тематических эмоджи как маркеры структуры, например: 🧵 материал/конструкция, 🎒 переноска, 🛡 защита, ⚙️ детали, 📦 комплект/наличие, 🎯 назначение;
|
||||
- эмоджи должны помогать читать текст, а не украшать каждую строку;
|
||||
- не добавляй ссылки и хэштеги в text.
|
||||
|
||||
Важное правило про источник:
|
||||
- producer_name — это источник поста: производитель, магазин или площадка.
|
||||
- Не утверждай, что producer_name «выпустил», «сделал», «представил» товар, если из текста не ясно, что это именно производитель этого товара.
|
||||
- Если источник выглядит как магазин/площадка или в тексте упомянут другой бренд, формулируй нейтрально: «у [producer_name] появился...», «[producer_name] показали/добавили...», «в продаже появился...».
|
||||
- Если из текста явно видно, что источник сам производит товар, можно писать: «[producer_name] выпустили...».
|
||||
|
||||
Категория:
|
||||
- выбери ровно одну category из списка categories во входных данных;
|
||||
- если сомневаешься, выбирай ближайшую по назначению товара.
|
||||
|
||||
OUTPUT: JSON only.
|
||||
$prompt$::text),
|
||||
updated_at = NOW()
|
||||
WHERE key='ai_writer_prompt';
|
||||
|
||||
UPDATE app_settings
|
||||
SET value_json = to_jsonb($contract$
|
||||
OUTPUT SCHEMA (return array matching input order):
|
||||
{"rewrites":[{"id":123,"category":"разгрузка","text":"готовый текст без хэштегов","notes":"короткая заметка для редактора или null"}]}
|
||||
|
||||
Rules:
|
||||
- Input is a JSON object with key "posts" containing accepted posts.
|
||||
- Each post includes producer_name and producer_tag. Use producer_name carefully: it can be a manufacturer, shop, or publishing source.
|
||||
- Do not add hashtags to rewritten text. Pick exactly one category from the categories list in input.
|
||||
- Return JSON only. No markdown. No text outside JSON.
|
||||
- Return one rewrite for every input post id.
|
||||
$contract$::text),
|
||||
updated_at = NOW()
|
||||
WHERE key='ai_writer_contract';
|
||||
@@ -0,0 +1,24 @@
|
||||
CREATE TABLE IF NOT EXISTS content_categories (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
tag TEXT NOT NULL,
|
||||
is_active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_content_categories_active_order
|
||||
ON content_categories(is_active, sort_order, name);
|
||||
|
||||
INSERT INTO content_categories(name, tag, sort_order)
|
||||
SELECT value, regexp_replace(lower(value), '[^[:alnum:]_а-яё]+', '_', 'g'), ordinality::int
|
||||
FROM app_settings s
|
||||
CROSS JOIN LATERAL jsonb_array_elements_text(
|
||||
CASE WHEN jsonb_typeof(s.value_json) = 'array' THEN s.value_json ELSE '[]'::jsonb END
|
||||
) WITH ORDINALITY AS items(value, ordinality)
|
||||
WHERE s.key = 'ai_writer_categories'
|
||||
ON CONFLICT (name) DO UPDATE
|
||||
SET tag = EXCLUDED.tag,
|
||||
sort_order = LEAST(content_categories.sort_order, EXCLUDED.sort_order),
|
||||
updated_at = NOW();
|
||||
@@ -0,0 +1,11 @@
|
||||
aiohttp==3.12.13
|
||||
aiogram==3.21.0
|
||||
asyncpg==0.30.0
|
||||
fastapi==0.115.14
|
||||
jinja2==3.1.6
|
||||
loguru==0.7.3
|
||||
litellm==1.76.0
|
||||
pydantic-settings==2.10.1
|
||||
python-multipart==0.0.20
|
||||
uvicorn[standard]==0.35.0
|
||||
yt-dlp==2026.6.9
|
||||
@@ -0,0 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
from vk_parser_app.db import apply_migrations, close_pool
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
await apply_migrations(root / "db" / "migrations")
|
||||
await close_pool()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,37 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
from vk_parser_app.config import settings
|
||||
from vk_parser_app.vk_api import VKAPIClient, post_vk_url
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
if not settings.vk_access_token:
|
||||
raise RuntimeError("VK_ACCESS_TOKEN is empty")
|
||||
if not settings.vk_storage_group_id:
|
||||
raise RuntimeError("VK_STORAGE_GROUP_ID is empty")
|
||||
|
||||
async with VKAPIClient() as client:
|
||||
attachments: list[str] = []
|
||||
sample = Path("sample_photo.jpg")
|
||||
if sample.exists():
|
||||
saved = await client.upload_wall_photo_bytes(
|
||||
abs(settings.vk_storage_group_id),
|
||||
sample.read_bytes(),
|
||||
filename="sample_photo.jpg",
|
||||
)
|
||||
attachments.append(f"photo{saved['owner_id']}_{saved['id']}")
|
||||
|
||||
post_id = await client.create_wall_post(
|
||||
owner_id=settings.vk_storage_owner_id,
|
||||
message="VK storage spike: test raw copy post",
|
||||
attachments=attachments,
|
||||
from_group=True,
|
||||
)
|
||||
print(post_vk_url(settings.vk_storage_owner_id, post_id))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,3 @@
|
||||
__all__ = ["__version__"]
|
||||
|
||||
__version__ = "0.1.0"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,45 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
app_env: str = "production"
|
||||
app_secret_key: str = "change-me"
|
||||
admin_bootstrap_login: str = "admin"
|
||||
admin_bootstrap_password: str = ""
|
||||
|
||||
db_host: str = "localhost"
|
||||
db_port: int = 5432
|
||||
db_name: str = "vk_parser"
|
||||
db_user: str = "vk_parser_user"
|
||||
db_password: str = ""
|
||||
|
||||
vk_access_token: str = ""
|
||||
vk_group_access_token: str = ""
|
||||
vk_api_version: str = "5.199"
|
||||
vk_storage_group_id: int = 0
|
||||
|
||||
tg_bot_token: str = ""
|
||||
tg_media_channel_id: str = ""
|
||||
local_bot_api_url: str = ""
|
||||
|
||||
admin_host: str = "0.0.0.0"
|
||||
admin_port: int = 8080
|
||||
log_level: str = "INFO"
|
||||
|
||||
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
|
||||
|
||||
@property
|
||||
def db_dsn(self) -> str:
|
||||
return (
|
||||
f"postgresql://{self.db_user}:{self.db_password}"
|
||||
f"@{self.db_host}:{self.db_port}/{self.db_name}"
|
||||
)
|
||||
|
||||
@property
|
||||
def vk_storage_owner_id(self) -> int:
|
||||
return -abs(int(self.vk_storage_group_id))
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -0,0 +1,30 @@
|
||||
PLATFORM_VK = "vk"
|
||||
|
||||
SOURCE_STATUS_NEW = "new"
|
||||
SOURCE_STATUS_OK = "ok"
|
||||
SOURCE_STATUS_ERROR = "error"
|
||||
SOURCE_STATUS_PAUSED = "paused"
|
||||
|
||||
POST_STATUS_RAW_SAVED = "raw_saved"
|
||||
POST_STATUS_STORAGE_PENDING = "storage_pending"
|
||||
POST_STATUS_STORAGE_READY = "storage_ready"
|
||||
POST_STATUS_SKIPPED = "skipped"
|
||||
POST_STATUS_FAILED = "failed"
|
||||
|
||||
MEDIA_STATUS_PENDING = "pending"
|
||||
MEDIA_STATUS_UPLOADED = "uploaded"
|
||||
MEDIA_STATUS_LINK_ONLY = "link_only"
|
||||
MEDIA_STATUS_FAILED = "failed"
|
||||
|
||||
JOB_STATUS_PENDING = "pending"
|
||||
JOB_STATUS_IN_PROGRESS = "in_progress"
|
||||
JOB_STATUS_RETRY = "retry"
|
||||
JOB_STATUS_DONE = "done"
|
||||
JOB_STATUS_DEAD = "dead"
|
||||
|
||||
JOB_TYPE_VK_STORAGE_COPY = "vk.storage.copy"
|
||||
|
||||
WORKER_PARSER = "vk-parser"
|
||||
WORKER_STORAGE_UPLOADER = "vk-storage-uploader"
|
||||
WORKER_AI_QUALIFIER = "ai-qualifier"
|
||||
WORKER_AI_WRITER = "ai-writer"
|
||||
@@ -0,0 +1,99 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import asyncpg
|
||||
from loguru import logger
|
||||
|
||||
from .config import settings
|
||||
|
||||
_pool: asyncpg.Pool | None = None
|
||||
|
||||
|
||||
async def get_pool() -> asyncpg.Pool:
|
||||
global _pool
|
||||
if _pool is None:
|
||||
_pool = await asyncpg.create_pool(
|
||||
host=settings.db_host,
|
||||
port=settings.db_port,
|
||||
database=settings.db_name,
|
||||
user=settings.db_user,
|
||||
password=settings.db_password,
|
||||
min_size=1,
|
||||
max_size=10,
|
||||
command_timeout=60,
|
||||
)
|
||||
logger.info("Database pool created: {}:{}/{}", settings.db_host, settings.db_port, settings.db_name)
|
||||
return _pool
|
||||
|
||||
|
||||
async def close_pool() -> None:
|
||||
global _pool
|
||||
if _pool is not None:
|
||||
await _pool.close()
|
||||
_pool = None
|
||||
|
||||
|
||||
async def fetch_setting(key: str, default: Any = None) -> Any:
|
||||
pool = await get_pool()
|
||||
row = await pool.fetchrow("SELECT value_json FROM app_settings WHERE key=$1", key)
|
||||
if not row:
|
||||
return default
|
||||
value = row["value_json"]
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
return value
|
||||
return value
|
||||
|
||||
|
||||
async def fetch_int_setting(key: str, default: int) -> int:
|
||||
try:
|
||||
return int(await fetch_setting(key, default))
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
async def fetch_float_setting(key: str, default: float) -> float:
|
||||
try:
|
||||
return float(await fetch_setting(key, default))
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
async def fetch_bool_setting(key: str, default: bool) -> bool:
|
||||
try:
|
||||
value = await fetch_setting(key, default)
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
return value.strip().lower() in {"1", "true", "yes", "on"}
|
||||
return bool(value)
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
async def apply_migrations(migrations_dir: Path) -> None:
|
||||
pool = await get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version TEXT PRIMARY KEY,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
"""
|
||||
)
|
||||
for path in sorted(migrations_dir.glob("*.sql")):
|
||||
version = path.name
|
||||
exists = await conn.fetchval("SELECT 1 FROM schema_migrations WHERE version=$1", version)
|
||||
if exists:
|
||||
continue
|
||||
sql = path.read_text(encoding="utf-8")
|
||||
async with conn.transaction():
|
||||
await conn.execute(sql)
|
||||
await conn.execute("INSERT INTO schema_migrations(version) VALUES($1)", version)
|
||||
logger.info("Applied migration {}", version)
|
||||
@@ -0,0 +1,45 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
|
||||
class HeartbeatReporter:
|
||||
def __init__(self, name: str, interval_sec: int = 30) -> None:
|
||||
self.name = name
|
||||
self.interval_sec = max(5, int(interval_sec))
|
||||
self._last = 0.0
|
||||
|
||||
async def beat(
|
||||
self,
|
||||
pool,
|
||||
status: str = "running",
|
||||
current_job_id: int | None = None,
|
||||
meta: dict[str, Any] | None = None,
|
||||
force: bool = False,
|
||||
) -> None:
|
||||
now = time.monotonic()
|
||||
if not force and now - self._last < self.interval_sec:
|
||||
return
|
||||
self._last = now
|
||||
try:
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO worker_heartbeats(name, heartbeat_at, status, current_job_id, meta_json, updated_at)
|
||||
VALUES ($1, NOW(), $2, $3, $4::jsonb, NOW())
|
||||
ON CONFLICT (name) DO UPDATE
|
||||
SET heartbeat_at=NOW(),
|
||||
status=EXCLUDED.status,
|
||||
current_job_id=EXCLUDED.current_job_id,
|
||||
meta_json=EXCLUDED.meta_json,
|
||||
updated_at=NOW()
|
||||
""",
|
||||
self.name,
|
||||
status,
|
||||
current_job_id,
|
||||
json.dumps(meta or {}, ensure_ascii=False),
|
||||
)
|
||||
except Exception:
|
||||
return
|
||||
@@ -0,0 +1,98 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from .constants import JOB_STATUS_IN_PROGRESS, JOB_STATUS_PENDING, JOB_STATUS_RETRY
|
||||
|
||||
|
||||
async def is_worker_enabled(pool, name: str) -> bool:
|
||||
value = await pool.fetchval("SELECT enabled FROM worker_controls WHERE name=$1", name)
|
||||
return bool(value)
|
||||
|
||||
|
||||
async def claim_job(pool, job_type: str, worker_id: str) -> dict | None:
|
||||
row = await pool.fetchrow(
|
||||
"""
|
||||
WITH cte AS (
|
||||
SELECT id
|
||||
FROM jobs
|
||||
WHERE type=$1
|
||||
AND status IN ($2, $3)
|
||||
AND next_run_at <= NOW()
|
||||
ORDER BY next_run_at ASC, id ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
)
|
||||
UPDATE jobs j
|
||||
SET status=$4,
|
||||
locked_by=$5,
|
||||
locked_at=NOW(),
|
||||
updated_at=NOW()
|
||||
FROM cte
|
||||
WHERE j.id=cte.id
|
||||
RETURNING j.*
|
||||
""",
|
||||
job_type,
|
||||
JOB_STATUS_PENDING,
|
||||
JOB_STATUS_RETRY,
|
||||
JOB_STATUS_IN_PROGRESS,
|
||||
worker_id,
|
||||
)
|
||||
return dict(row) if row else None
|
||||
|
||||
|
||||
async def ack_done(pool, job_id: int) -> None:
|
||||
await pool.execute(
|
||||
"""
|
||||
UPDATE jobs
|
||||
SET status='done',
|
||||
locked_by=NULL,
|
||||
locked_at=NULL,
|
||||
last_error=NULL,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
job_id,
|
||||
)
|
||||
|
||||
|
||||
async def ack_retry(pool, job: dict, error: str, delay_sec: int = 60) -> None:
|
||||
attempt = int(job.get("attempts") or 0) + 1
|
||||
max_attempts = int(job.get("max_attempts") or 5)
|
||||
status = "dead" if attempt >= max_attempts else "retry"
|
||||
await pool.execute(
|
||||
"""
|
||||
UPDATE jobs
|
||||
SET status=$2,
|
||||
attempts=$3,
|
||||
next_run_at=CASE WHEN $2='retry' THEN NOW() + ($4 * INTERVAL '1 second') ELSE next_run_at END,
|
||||
locked_by=NULL,
|
||||
locked_at=NULL,
|
||||
last_error=$5,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
int(job["id"]),
|
||||
status,
|
||||
attempt,
|
||||
int(delay_sec),
|
||||
error[:1000],
|
||||
)
|
||||
|
||||
|
||||
async def recover_stale_jobs(pool, job_type: str, stale_minutes: int = 20) -> int:
|
||||
result = await pool.execute(
|
||||
"""
|
||||
UPDATE jobs
|
||||
SET status='retry',
|
||||
locked_by=NULL,
|
||||
locked_at=NULL,
|
||||
next_run_at=NOW(),
|
||||
last_error=COALESCE(last_error, 'recovered stale lock'),
|
||||
updated_at=NOW()
|
||||
WHERE type=$1
|
||||
AND status='in_progress'
|
||||
AND locked_at < NOW() - ($2 * INTERVAL '1 minute')
|
||||
""",
|
||||
job_type,
|
||||
stale_minutes,
|
||||
)
|
||||
return int(str(result).split()[-1])
|
||||
@@ -0,0 +1,3 @@
|
||||
from .admin import app
|
||||
|
||||
__all__ = ["app"]
|
||||
@@ -0,0 +1,37 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import secrets
|
||||
|
||||
|
||||
def hash_password(password: str, iterations: int = 390_000) -> str:
|
||||
salt = secrets.token_bytes(16)
|
||||
digest = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, iterations)
|
||||
return "pbkdf2_sha256${}${}${}".format(
|
||||
iterations,
|
||||
base64.urlsafe_b64encode(salt).decode("ascii"),
|
||||
base64.urlsafe_b64encode(digest).decode("ascii"),
|
||||
)
|
||||
|
||||
|
||||
def verify_password(password: str, encoded: str) -> bool:
|
||||
try:
|
||||
algo, iterations_raw, salt_raw, digest_raw = encoded.split("$", 3)
|
||||
if algo != "pbkdf2_sha256":
|
||||
return False
|
||||
iterations = int(iterations_raw)
|
||||
salt = base64.urlsafe_b64decode(salt_raw.encode("ascii"))
|
||||
expected = base64.urlsafe_b64decode(digest_raw.encode("ascii"))
|
||||
except Exception:
|
||||
return False
|
||||
actual = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, iterations)
|
||||
return secrets.compare_digest(actual, expected)
|
||||
|
||||
|
||||
def token_hash(token: str, secret: str) -> str:
|
||||
return hashlib.sha256((secret + ":" + token).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def new_token() -> str:
|
||||
return secrets.token_urlsafe(32)
|
||||
@@ -0,0 +1,229 @@
|
||||
<!doctype html>
|
||||
<html lang="ru">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>{{ title or "VK Parser Admin" }}</title>
|
||||
<style>
|
||||
:root { color-scheme: light; --bg:#f4f6f8; --panel:#fff; --panel-soft:#f8fafc; --line:#d8dee8; --line-strong:#c8d0dc; --text:#20242b; --muted:#647086; --accent:#245fd6; --accent-soft:#eef4ff; --danger:#b42318; --ok:#027a48; --shadow:0 1px 2px rgba(16,24,40,.04), 0 10px 30px rgba(16,24,40,.06); }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin:0; font:14px/1.45 Inter, -apple-system, BlinkMacSystemFont, "Segoe UI", Arial, sans-serif; color:var(--text); background:var(--bg); }
|
||||
header { position:sticky; top:0; z-index:40; min-height:58px; display:flex; align-items:center; justify-content:space-between; gap:16px; padding:0 24px; background:rgba(255,255,255,.92); border-bottom:1px solid var(--line); backdrop-filter:blur(12px); }
|
||||
nav { display:flex; gap:4px; align-items:center; flex-wrap:wrap; }
|
||||
nav a { color:var(--muted); text-decoration:none; font-weight:700; padding:8px 10px; border-radius:7px; }
|
||||
nav a:hover { color:var(--text); background:var(--panel-soft); }
|
||||
main { padding:26px 24px 40px; max-width:1440px; margin:0 auto; }
|
||||
h1 { font-size:34px; line-height:1.1; margin:0 0 8px; letter-spacing:0; }
|
||||
h2 { font-size:21px; line-height:1.2; margin:0 0 8px; letter-spacing:0; }
|
||||
.panel { background:var(--panel); border:1px solid var(--line); border-radius:8px; padding:16px; margin-bottom:16px; box-shadow:var(--shadow); }
|
||||
details.panel summary { cursor:pointer; list-style:none; font-size:16px; margin:-4px 0 12px; }
|
||||
details.panel summary::-webkit-details-marker { display:none; }
|
||||
details.panel summary:before { content:"▸"; display:inline-block; width:18px; color:var(--muted); }
|
||||
details.panel[open] summary:before { content:"▾"; }
|
||||
.toolbar { display:flex; gap:10px; align-items:end; justify-content:space-between; flex-wrap:wrap; margin-bottom:14px; }
|
||||
.row { display:flex; gap:8px; align-items:center; flex-wrap:wrap; }
|
||||
table { width:100%; border-collapse:separate; border-spacing:0; background:var(--panel); }
|
||||
th, td { padding:10px; border-bottom:1px solid var(--line); text-align:left; vertical-align:top; }
|
||||
th { color:var(--muted); font-size:12px; text-transform:uppercase; }
|
||||
input, select, textarea { width:100%; padding:9px 10px; border:1px solid var(--line); border-radius:7px; font:inherit; background:#fff; color:var(--text); outline:none; }
|
||||
input:focus, select:focus, textarea:focus { border-color:var(--accent); box-shadow:0 0 0 3px rgba(36,95,214,.12); }
|
||||
label { display:block; color:var(--muted); font-weight:600; margin-bottom:8px; }
|
||||
label span { display:block; margin-bottom:4px; }
|
||||
.form-grid { display:grid; grid-template-columns:repeat(2, minmax(0,1fr)); gap:14px; }
|
||||
.filter-grid { display:grid; grid-template-columns:repeat(auto-fit, minmax(150px, 1fr)); gap:12px; align-items:end; }
|
||||
.sources-filter { grid-template-columns:minmax(260px, 1fr) 160px 140px auto; }
|
||||
.filter-submit { height:38px; margin-bottom:0; }
|
||||
.btn { display:inline-flex; align-items:center; justify-content:center; gap:6px; min-height:36px; padding:8px 12px; border:1px solid var(--line); border-radius:7px; background:#fff; color:var(--text); text-decoration:none; cursor:pointer; font-weight:700; white-space:nowrap; }
|
||||
.btn:hover { border-color:var(--line-strong); background:var(--panel-soft); }
|
||||
.btn.primary { background:var(--accent); border-color:var(--accent); color:#fff; }
|
||||
.btn.danger { color:var(--danger); }
|
||||
.pill { display:inline-flex; padding:3px 8px; border-radius:999px; border:1px solid var(--line); color:var(--muted); font-size:12px; }
|
||||
.pill.ok { color:var(--ok); border-color:#a6f4c5; background:#ecfdf3; }
|
||||
.pill.bad { color:var(--danger); border-color:#fecdca; background:#fef3f2; }
|
||||
.pill.warn { color:#92400e; border-color:#fedf89; background:#fffaeb; }
|
||||
.muted { color:var(--muted); }
|
||||
.mono { font-family: Consolas, monospace; }
|
||||
.actions { display:flex; gap:6px; flex-wrap:wrap; }
|
||||
.compact-table th, .compact-table td { padding:7px 8px; }
|
||||
.source-state { margin-left:6px; }
|
||||
.source-url { max-width:300px; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; }
|
||||
.status-msg { max-width:220px; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; }
|
||||
.icon-actions { display:flex; gap:4px; align-items:center; justify-content:flex-end; white-space:nowrap; }
|
||||
.icon-actions form { margin:0; }
|
||||
.icon-btn { width:30px; height:30px; display:inline-flex; align-items:center; justify-content:center; border:1px solid var(--line); border-radius:6px; background:#fff; color:var(--text); text-decoration:none; cursor:pointer; font:18px/1 Arial, sans-serif; }
|
||||
.icon-btn:hover { border-color:#b9c0cc; background:#f8fafc; }
|
||||
.icon-btn.danger { color:var(--danger); }
|
||||
.source-add-grid { display:grid; grid-template-columns:minmax(260px, 360px) minmax(360px, 1fr); gap:18px; align-items:start; }
|
||||
.checkline { display:flex; gap:8px; align-items:center; color:var(--text); font-weight:600; }
|
||||
.checkline input { width:auto; }
|
||||
.preview-block { margin-top:16px; border-top:1px solid var(--line); padding-top:14px; }
|
||||
.pagination { display:flex; justify-content:flex-end; margin-top:12px; }
|
||||
.pagination-form { display:flex; gap:8px; align-items:center; }
|
||||
.pagination button[disabled] { opacity:.45; cursor:not-allowed; }
|
||||
.raw-id { color:var(--text); font-weight:700; text-decoration:none; }
|
||||
.source-cell { min-width:150px; }
|
||||
.source-cell a { font-weight:700; }
|
||||
.stage-cell { min-width:112px; }
|
||||
.raw-table { table-layout:fixed; }
|
||||
.raw-table th:nth-child(1), .raw-table td:nth-child(1) { width:58px; }
|
||||
.raw-table th:nth-child(2), .raw-table td:nth-child(2) { width:170px; }
|
||||
.raw-table th:nth-child(3), .raw-table td:nth-child(3) { width:120px; }
|
||||
.raw-table th:nth-child(5), .raw-table td:nth-child(5) { width:96px; }
|
||||
.raw-content-cell { max-width:none; min-width:0; overflow-wrap:anywhere; }
|
||||
.raw-text-details { margin-top:8px; }
|
||||
.raw-text-details summary { cursor:pointer; color:var(--text); white-space:pre-wrap; }
|
||||
.raw-full-text { margin-top:8px; white-space:pre-wrap; overflow-wrap:anywhere; }
|
||||
.raw-text-details summary { overflow-wrap:anywhere; }
|
||||
.raw-content-cell > .raw-full-text:first-child,
|
||||
.media-strip + .raw-full-text { margin-top:8px; }
|
||||
.media-strip { display:flex; gap:6px; max-width:100%; overflow-x:auto; padding:1px 0 5px; margin-bottom:2px; }
|
||||
.media-thumb { flex:0 0 48px; width:48px; height:48px; padding:0; border:1px solid var(--line); border-radius:6px; overflow:hidden; display:flex; align-items:center; justify-content:center; background:#eef1f6; color:var(--text); text-decoration:none; cursor:pointer; }
|
||||
.media-thumb img { width:100%; height:100%; object-fit:cover; display:block; }
|
||||
.media-video span { font-size:18px; line-height:1; }
|
||||
.media-count { font-size:12px; margin-bottom:4px; }
|
||||
.raw-meta { display:grid; grid-template-columns:1fr; gap:8px; min-width:130px; }
|
||||
.raw-links { display:flex; flex-direction:column; gap:3px; }
|
||||
.raw-dates { display:grid; gap:2px; color:var(--text); }
|
||||
.post-meta { margin-top:8px; color:var(--muted); font-size:12px; }
|
||||
.ai-cell { width:96px; }
|
||||
.ai-badge { width:82px; min-height:58px; display:flex; flex-direction:column; align-items:center; justify-content:center; gap:2px; border:1px solid var(--line); border-radius:8px; color:var(--muted); text-decoration:none; background:#f8fafc; }
|
||||
.ai-badge span { color:var(--text); font-size:18px; line-height:1; font-weight:800; }
|
||||
.ai-badge small { max-width:72px; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; font-size:11px; }
|
||||
.ai-badge.ok { border-color:#a6f4c5; background:#ecfdf3; color:var(--ok); }
|
||||
.ai-badge.ok span { color:var(--ok); }
|
||||
.ai-badge.warn { border-color:#fedf89; background:#fffaeb; color:#92400e; }
|
||||
.ai-badge.warn span { color:#92400e; }
|
||||
.ai-badge.bad { border-color:#fecdca; background:#fef3f2; color:var(--danger); }
|
||||
.ai-badge.bad span { color:var(--danger); }
|
||||
.ai-model-decision { width:92px; margin-top:5px; font-size:12px; overflow:hidden; text-overflow:ellipsis; }
|
||||
.detail-grid { display:grid; grid-template-columns:minmax(0, 1fr) 320px; gap:16px; align-items:start; }
|
||||
.side-panel { position:sticky; top:72px; }
|
||||
.detail-media { padding-bottom:8px; }
|
||||
.detail-thumb { flex-basis:72px; width:72px; height:72px; }
|
||||
.editable-media { position:relative; flex:0 0 auto; }
|
||||
.editable-media.marked-delete { opacity:.38; filter:grayscale(1); }
|
||||
.media-remove { position:absolute; top:-6px; right:-6px; width:22px; height:22px; border:1px solid var(--line); border-radius:999px; background:#fff; color:var(--danger); font-weight:900; cursor:pointer; box-shadow:var(--shadow); }
|
||||
.media-upload-form { display:block; margin:8px 0 8px; }
|
||||
.detail-text { font-size:15px; line-height:1.55; }
|
||||
.detail-ai { width:110px; min-height:72px; margin:8px 0 16px; }
|
||||
.rewrite-text { white-space:pre-wrap; font-size:16px; line-height:1.55; max-width:820px; }
|
||||
.rewrite-meta { margin-top:14px; }
|
||||
.kv { display:grid; grid-template-columns:96px minmax(0,1fr); gap:8px 12px; margin:0; }
|
||||
.kv dt { color:var(--muted); font-weight:700; }
|
||||
.kv dd { margin:0; overflow-wrap:anywhere; }
|
||||
.kv-wide { grid-template-columns:180px minmax(0,1fr); margin-bottom:14px; }
|
||||
.json-box { white-space:pre-wrap; overflow:auto; max-height:420px; padding:12px; border:1px solid var(--line); border-radius:8px; background:#0f172a; color:#e2e8f0; }
|
||||
.prompt-editor { display:grid; gap:8px; }
|
||||
.prompt-hint { max-width:720px; border:1px solid var(--line); border-radius:8px; padding:10px 12px; background:#f8fafc; display:grid; gap:8px; color:var(--text); }
|
||||
.prompt-hint summary { cursor:pointer; font-weight:700; }
|
||||
.prompt-hint pre, .prompt-box { white-space:pre-wrap; overflow:auto; padding:10px; border:1px solid var(--line); border-radius:8px; background:#f8fafc; color:var(--text); }
|
||||
.hint-grid { display:grid; grid-template-columns:repeat(2, minmax(0,1fr)); gap:10px; }
|
||||
.hint-label { color:var(--muted); font-size:12px; font-weight:700; text-transform:uppercase; margin-bottom:3px; }
|
||||
.small-help { font-size:12px; margin-top:4px; }
|
||||
.secret-editor { display:grid; gap:3px; }
|
||||
.secret-visible { font-family:Consolas, monospace; }
|
||||
.category-create { display:grid; grid-template-columns:minmax(180px, 1fr) minmax(180px, 1fr) auto; gap:10px; align-items:end; margin:10px 0 14px; }
|
||||
.category-create label { margin-bottom:0; }
|
||||
.category-table input { min-height:34px; padding:7px 8px; }
|
||||
.category-table th:nth-child(3), .category-table td:nth-child(3) { width:96px; }
|
||||
.category-table th:nth-child(4), .category-table td:nth-child(4) { width:110px; }
|
||||
.category-table th:nth-child(5), .category-table td:nth-child(5) { width:120px; }
|
||||
.advanced-tools { grid-column:1 / -1; border-top:1px solid var(--line); margin-top:4px; padding-top:10px; }
|
||||
.advanced-tools summary { cursor:pointer; font-weight:700; color:var(--text); }
|
||||
.advanced-grid { display:grid; grid-template-columns:minmax(0,1fr) minmax(0,1fr); gap:16px; margin-top:10px; }
|
||||
.tool-title { font-weight:700; margin-bottom:6px; }
|
||||
.tool-row { display:grid; grid-template-columns:minmax(130px, 1.1fr) 120px minmax(130px, 1fr); gap:8px; margin-bottom:8px; }
|
||||
.tool-row.sort-row { grid-template-columns:minmax(170px, 1fr) 150px; }
|
||||
.nested-details { margin:12px 0; }
|
||||
.nested-details summary { cursor:pointer; font-weight:700; }
|
||||
.gallery-modal { position:fixed; inset:0; z-index:5000; display:none; align-items:center; justify-content:center; padding:48px; background:rgba(15,18,25,.86); border:0; width:100vw; height:100vh; max-width:none; max-height:none; }
|
||||
.gallery-modal::backdrop { background:rgba(15,18,25,.86); }
|
||||
.gallery-modal.open { display:flex; }
|
||||
.gallery-modal img { max-width:calc(100vw - 130px); max-height:calc(100vh - 120px); object-fit:contain; border-radius:8px; box-shadow:0 20px 60px rgba(0,0,0,.45); background:#111; }
|
||||
.gallery-close, .gallery-nav { position:absolute; border:0; background:rgba(255,255,255,.12); color:#fff; cursor:pointer; }
|
||||
.gallery-close { top:18px; right:22px; width:38px; height:38px; border-radius:50%; font-size:30px; line-height:1; }
|
||||
.gallery-nav { top:50%; transform:translateY(-50%); width:44px; height:64px; border-radius:8px; font-size:42px; line-height:1; }
|
||||
.gallery-prev { left:22px; }
|
||||
.gallery-next { right:22px; }
|
||||
.gallery-title { position:absolute; left:24px; bottom:18px; color:#fff; font-weight:600; }
|
||||
.page-head { display:flex; align-items:flex-end; justify-content:space-between; gap:16px; margin-bottom:16px; }
|
||||
.status-tabs { display:flex; gap:6px; flex-wrap:wrap; justify-content:flex-end; }
|
||||
.tab { display:inline-flex; align-items:center; gap:7px; min-height:34px; padding:7px 10px; border:1px solid var(--line); border-radius:999px; color:var(--muted); background:#fff; text-decoration:none; font-weight:700; }
|
||||
.tab span { color:var(--text); background:var(--panel-soft); border-radius:999px; padding:1px 7px; font-size:12px; }
|
||||
.tab.active { color:var(--accent); border-color:#b7caf8; background:var(--accent-soft); }
|
||||
.editor-filter { grid-template-columns:180px 180px 140px auto; }
|
||||
.catalog-layout { display:grid; grid-template-columns:minmax(0, 1fr) 306px; gap:16px; align-items:start; }
|
||||
.catalog-main { min-width:0; overflow:hidden; }
|
||||
.filter-sidebar { position:sticky; top:74px; max-height:calc(100vh - 92px); overflow:auto; overflow-x:hidden; padding:14px; }
|
||||
.filter-sidebar input, .filter-sidebar select { min-width:0; }
|
||||
.filter-head { display:flex; align-items:center; justify-content:space-between; gap:10px; margin-bottom:8px; }
|
||||
.filter-head h2 { margin:0; }
|
||||
.filter-head a { color:var(--muted); font-weight:700; text-decoration:none; }
|
||||
.filter-section { border-top:1px solid var(--line); padding-top:12px; margin-top:12px; }
|
||||
.filter-title { font-weight:800; margin-bottom:8px; }
|
||||
.range-row { display:grid; grid-template-columns:minmax(0,1fr) minmax(0,1fr); gap:8px; }
|
||||
.facet-search { margin-bottom:8px; }
|
||||
.facet-list { display:grid; gap:4px; max-height:220px; overflow:auto; padding-right:3px; }
|
||||
.facet-list.compact { max-height:160px; }
|
||||
.facet-option { display:grid; grid-template-columns:auto minmax(0,1fr) auto; align-items:center; gap:8px; margin:0; padding:6px 7px; border-radius:7px; color:var(--text); font-weight:600; cursor:pointer; }
|
||||
.facet-option:hover { background:var(--panel-soft); }
|
||||
.facet-option input { width:16px; height:16px; margin:0; }
|
||||
.facet-option span { overflow:hidden; text-overflow:ellipsis; white-space:nowrap; margin:0; }
|
||||
.facet-option small { color:var(--muted); font-weight:700; }
|
||||
.filter-apply { width:100%; margin-top:14px; }
|
||||
.list-toolbar { display:flex; justify-content:space-between; align-items:end; gap:12px; margin-bottom:12px; }
|
||||
.list-footer { display:flex; align-items:center; justify-content:space-between; gap:12px; flex-wrap:wrap; margin-top:12px; }
|
||||
.list-footer .pagination { margin-top:0; }
|
||||
.per-page-form label { width:126px; margin-bottom:0; }
|
||||
.compact-controls { justify-content:flex-end; }
|
||||
.compact-controls label { width:170px; margin-bottom:0; }
|
||||
.sort-head { display:inline-flex; align-items:center; gap:4px; color:inherit; text-decoration:none; }
|
||||
.sort-head:hover { color:var(--accent); }
|
||||
.sort-head.active:after { content:"↓"; font-size:11px; color:var(--accent); }
|
||||
.sort-head.active.asc:after { content:"↑"; }
|
||||
.editor-list { display:grid; gap:12px; }
|
||||
.editor-card { display:grid; grid-template-columns:112px minmax(0,1fr) 132px; gap:16px; align-items:start; background:var(--panel); border:1px solid var(--line); border-radius:8px; padding:14px; box-shadow:var(--shadow); }
|
||||
.editor-media .media-strip { display:grid; grid-template-columns:repeat(2, 48px); overflow:visible; }
|
||||
.editor-main { min-width:0; }
|
||||
.editor-meta { display:flex; gap:8px; align-items:center; flex-wrap:wrap; color:var(--muted); font-size:13px; margin-bottom:8px; }
|
||||
.editor-meta a { color:var(--text); font-weight:800; text-decoration:none; }
|
||||
.editor-title-row { display:flex; gap:8px; align-items:center; flex-wrap:wrap; margin-bottom:8px; }
|
||||
.editor-text { white-space:pre-wrap; max-height:190px; overflow:hidden; line-height:1.55; }
|
||||
.source-compare { margin-top:10px; border-top:1px solid var(--line); padding-top:8px; }
|
||||
.source-compare summary { cursor:pointer; color:var(--muted); font-weight:700; }
|
||||
.compare-grid { display:grid; grid-template-columns:1.3fr .7fr; gap:14px; margin-top:8px; }
|
||||
.editor-actions { display:grid; gap:8px; }
|
||||
.editor-actions form, .editor-actions button { width:100%; }
|
||||
.editor-dialog { width:min(1180px, calc(100vw - 36px)); max-height:calc(100vh - 36px); border:1px solid var(--line); border-radius:8px; padding:0; box-shadow:0 24px 80px rgba(16,24,40,.28); }
|
||||
.editor-dialog::backdrop { background:rgba(15,23,42,.56); backdrop-filter:blur(4px); }
|
||||
.dialog-head { position:sticky; top:0; z-index:2; display:flex; align-items:center; justify-content:space-between; gap:12px; padding:16px 18px; border-bottom:1px solid var(--line); background:#fff; }
|
||||
.dialog-grid { display:grid; grid-template-columns:420px minmax(0,1fr); gap:0; min-height:620px; }
|
||||
.context-pane { border-right:1px solid var(--line); background:var(--panel-soft); padding:16px; overflow:auto; max-height:calc(100vh - 112px); }
|
||||
.context-pane details { border-top:1px solid var(--line); padding-top:10px; margin-top:12px; }
|
||||
.context-pane summary { cursor:pointer; font-weight:800; }
|
||||
.edit-pane { padding:16px; overflow:auto; max-height:calc(100vh - 112px); }
|
||||
.edit-form textarea[name="final_text"] { min-height:360px; resize:vertical; line-height:1.55; }
|
||||
.hashtag-preview { margin:-2px 0 10px; color:var(--muted); font-family:Consolas, monospace; }
|
||||
.dialog-actions { display:flex; gap:8px; justify-content:flex-end; flex-wrap:wrap; }
|
||||
.regen-form { margin-top:16px; border-top:1px solid var(--line); padding-top:14px; }
|
||||
.compact-kv { margin-top:12px; grid-template-columns:90px minmax(0,1fr); }
|
||||
.empty-state { text-align:center; color:var(--muted); padding:34px; }
|
||||
@media (max-width: 900px) { header { padding:8px 12px; align-items:flex-start; } .filter-grid, .sources-filter, .source-add-grid, .hint-grid, .advanced-grid, .tool-row, .tool-row.sort-row, .editor-filter, .page-head, .compare-grid, .dialog-grid, .catalog-layout, .media-upload-form, .category-create { grid-template-columns:1fr; } .page-head { display:grid; } .form-grid { grid-template-columns:1fr; } .detail-grid { grid-template-columns:1fr; } .editor-card { grid-template-columns:1fr; } .editor-actions { grid-template-columns:repeat(3, minmax(0,1fr)); } .context-pane { border-right:0; border-bottom:1px solid var(--line); max-height:none; } .edit-pane { max-height:none; } .side-panel, .filter-sidebar { position:static; max-height:none; } .list-toolbar { display:grid; align-items:start; } .compact-controls { justify-content:start; } main { padding:14px; } }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
{% if user %}
|
||||
<header>
|
||||
<nav>
|
||||
<a href="/sources">Источники</a>
|
||||
<a href="/raw">Raw</a>
|
||||
<a href="/editor">Редакторская</a>
|
||||
<a href="/workers">Воркеры</a>
|
||||
<a href="/users">Пользователи</a>
|
||||
</nav>
|
||||
<form method="post" action="/logout"><button class="btn" type="submit">{{ user.login }} · выйти</button></form>
|
||||
</header>
|
||||
{% endif %}
|
||||
<main>{% block body %}{% endblock %}</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,338 @@
|
||||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
<div class="page-head">
|
||||
<div>
|
||||
<h1>Редакторская</h1>
|
||||
<div class="muted">Посты после AI-рерайта: быстрая проверка, правка и принятие.</div>
|
||||
</div>
|
||||
<div class="status-tabs">
|
||||
{% for st in status_options %}
|
||||
<a class="tab {% if st.value in editorial_statuses %}active{% endif %}" href="/editor?editorial_status={{ st.value }}">
|
||||
{{ st.label }} <span>{{ counts.get(st.value, 0) }}</span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="catalog-layout editor-catalog">
|
||||
<section class="catalog-main">
|
||||
<div class="panel list-toolbar">
|
||||
<div class="muted">Всего: {{ pagination.total }}</div>
|
||||
<form class="row compact-controls" method="get" action="/editor">
|
||||
{% for item in preserved_query %}
|
||||
{% if item.key != "per_page" and item.key != "sort" %}
|
||||
<input type="hidden" name="{{ item.key }}" value="{{ item.value }}">
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
<label><span>Сортировка</span>
|
||||
<select name="sort" data-autosubmit>
|
||||
<option value="rewritten_desc" {% if sort == "rewritten_desc" %}selected{% endif %}>рерайт: новые</option>
|
||||
<option value="rewritten_asc" {% if sort == "rewritten_asc" %}selected{% endif %}>рерайт: старые</option>
|
||||
<option value="posted_desc" {% if sort == "posted_desc" %}selected{% endif %}>VK: новые</option>
|
||||
<option value="posted_asc" {% if sort == "posted_asc" %}selected{% endif %}>VK: старые</option>
|
||||
<option value="score_desc" {% if sort == "score_desc" %}selected{% endif %}>оценка: выше</option>
|
||||
<option value="score_asc" {% if sort == "score_asc" %}selected{% endif %}>оценка: ниже</option>
|
||||
</select>
|
||||
</label>
|
||||
<label><span>На странице</span>
|
||||
<select name="per_page" data-autosubmit>
|
||||
{% for n in [10,25,50,100] %}
|
||||
<option value="{{ n }}" {% if pagination.per_page == n %}selected{% endif %}>{{ n }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</label>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div class="editor-list">
|
||||
{% for p in posts %}
|
||||
<article class="editor-card">
|
||||
<div class="editor-media">
|
||||
<div class="media-strip">
|
||||
{% for m in p.media_items %}
|
||||
{% if m.type == "photo" and m.url %}
|
||||
<button class="media-thumb" type="button" data-gallery-src="{{ m.url }}" data-gallery-title="#{{ p.id }} · фото {{ loop.index }}" title="{{ m.status }}">
|
||||
<img src="{{ m.url }}" alt="photo">
|
||||
</button>
|
||||
{% elif m.url %}
|
||||
<a class="media-thumb media-video" href="{{ m.url }}" target="_blank" title="{{ m.error or m.status }}"><span>▶</span></a>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
<div class="muted small-help">{{ p.media_count or 0 }} медиа</div>
|
||||
</div>
|
||||
|
||||
<div class="editor-main">
|
||||
<div class="editor-meta">
|
||||
<a href="{{ p.original_url }}" target="_blank">{{ p.source_name }}</a>
|
||||
<span>#{{ p.review_source_tag or p.source_tag or "source" }}</span>
|
||||
<span>{{ p.posted_at_fmt or p.created_at_fmt }}</span>
|
||||
</div>
|
||||
<div class="editor-title-row">
|
||||
<span class="pill {% if p.editorial_status == 'accepted' %}ok{% elif p.editorial_status == 'rejected' %}bad{% elif p.editorial_status == 'regenerating' %}warn{% endif %}">
|
||||
{{ p.editorial_status_label }}
|
||||
</span>
|
||||
<span class="pill">{{ p.review_category or "без категории" }}</span>
|
||||
<span class="muted">AI {{ p.qualification_score or "?" }}/10</span>
|
||||
</div>
|
||||
<div class="editor-text">{{ p.review_text }}</div>
|
||||
<details class="source-compare">
|
||||
<summary>Исходник и AI-заметки</summary>
|
||||
<div class="compare-grid">
|
||||
<div>
|
||||
<div class="hint-label">Оригинал</div>
|
||||
<div class="raw-full-text">{{ p.raw_text }}</div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="hint-label">Заметки</div>
|
||||
<div>{{ p.rewrite_notes or p.qualification_reason or "—" }}</div>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
|
||||
<div class="editor-actions">
|
||||
<form method="post" action="/editor/{{ p.id }}/accept">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<button class="btn primary" type="submit">Принять</button>
|
||||
</form>
|
||||
<button class="btn" type="button" data-open-editor="edit-{{ p.id }}">Редактировать</button>
|
||||
<form method="post" action="/editor/{{ p.id }}/reject" onsubmit="return confirm('Отклонить пост #{{ p.id }}?');">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<button class="btn danger" type="submit">Отклонить</button>
|
||||
</form>
|
||||
</div>
|
||||
</article>
|
||||
|
||||
<dialog class="editor-dialog" id="edit-{{ p.id }}">
|
||||
<div class="dialog-head">
|
||||
<div>
|
||||
<h2>Пост #{{ p.id }}</h2>
|
||||
<div class="muted"><a href="{{ p.original_url }}" target="_blank">{{ p.source_name }}</a> · {{ p.posted_at_fmt or p.created_at_fmt }}</div>
|
||||
</div>
|
||||
<button class="icon-btn" type="button" data-close-dialog="edit-{{ p.id }}">×</button>
|
||||
</div>
|
||||
<form method="post" action="/editor/{{ p.id }}/save" class="edit-form dialog-grid" enctype="multipart/form-data">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<aside class="context-pane">
|
||||
<div class="media-strip detail-media">
|
||||
{% for m in p.media_items %}
|
||||
<div class="editable-media" data-media-id="{{ m.id }}">
|
||||
{% if m.type == "photo" and m.url %}
|
||||
<button class="media-thumb detail-thumb" type="button" data-gallery-src="{{ m.url }}" data-gallery-title="#{{ p.id }} · фото {{ loop.index }}" title="{{ m.status }}">
|
||||
<img src="{{ m.url }}" alt="photo">
|
||||
</button>
|
||||
{% elif m.url %}
|
||||
<a class="media-thumb detail-thumb media-video" href="{{ m.url }}" target="_blank"><span>▶</span></a>
|
||||
{% endif %}
|
||||
<button class="media-remove" type="button" data-mark-media-delete="{{ m.id }}" title="Убрать после сохранения">×</button>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
<label class="media-upload-form"><span>Добавить медиа после сохранения</span><input type="file" name="media_files" accept="image/*,video/*,.pdf,.doc,.docx" multiple></label>
|
||||
<div class="pending-media-list muted small-help" data-pending-media-list>Выбранные файлы появятся после сохранения.</div>
|
||||
<details open>
|
||||
<summary>Исходный пост</summary>
|
||||
<div class="raw-full-text">{{ p.raw_text }}</div>
|
||||
</details>
|
||||
<details>
|
||||
<summary>AI-рерайт</summary>
|
||||
<div class="raw-full-text">{{ p.rewritten_text }}</div>
|
||||
</details>
|
||||
<dl class="kv compact-kv">
|
||||
<dt>Оценка</dt><dd>{{ p.qualification_score or "—" }}/10</dd>
|
||||
<dt>Категория</dt><dd>{{ p.rewrite_category or "—" }}</dd>
|
||||
<dt>Модель</dt><dd>{{ p.rewrite_model or "—" }}</dd>
|
||||
</dl>
|
||||
</aside>
|
||||
|
||||
<section class="edit-pane">
|
||||
<label><span>Текст публикации</span>
|
||||
<textarea name="final_text" rows="15">{{ p.review_text }}</textarea>
|
||||
</label>
|
||||
<div class="form-grid">
|
||||
<label><span>Категория</span>
|
||||
<select name="final_category">
|
||||
{% for category in categories %}
|
||||
<option value="{{ category }}" {% if p.review_category == category %}selected{% endif %}>{{ category }}</option>
|
||||
{% endfor %}
|
||||
{% if p.review_category and p.review_category not in categories %}
|
||||
<option value="{{ p.review_category }}" selected>{{ p.review_category }} · старая категория</option>
|
||||
{% endif %}
|
||||
</select>
|
||||
</label>
|
||||
<label><span>Тэг источника</span>
|
||||
<input name="final_source_tag" value="{{ p.review_source_tag or p.source_tag or '' }}">
|
||||
</label>
|
||||
</div>
|
||||
<div class="hashtag-preview" data-hashtag-preview>#{{ p.review_category_tag or p.review_category or "category" }} #{{ p.review_source_tag or p.source_tag or "source" }}</div>
|
||||
<label><span>Заметка редактора</span>
|
||||
<textarea name="editor_notes" rows="3">{{ p.editor_notes or "" }}</textarea>
|
||||
</label>
|
||||
<div class="dialog-actions">
|
||||
<button class="btn" type="submit" name="action" value="save">Сохранить правки</button>
|
||||
<button class="btn primary" type="submit" name="action" value="accept">Принять к публикации</button>
|
||||
</div>
|
||||
</section>
|
||||
</form>
|
||||
</dialog>
|
||||
{% else %}
|
||||
<div class="empty-state panel">В этой очереди пока нет постов.</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
|
||||
{% include "pagination.html" %}
|
||||
</section>
|
||||
|
||||
<aside class="filter-sidebar panel">
|
||||
<form method="get" action="/editor" data-filter-form>
|
||||
<input type="hidden" name="sort" value="{{ sort }}">
|
||||
<div class="filter-head">
|
||||
<h2>Фильтры</h2>
|
||||
<a href="/editor">Сбросить</a>
|
||||
</div>
|
||||
<div class="filter-section">
|
||||
<div class="filter-title">Оценка AI</div>
|
||||
<div class="range-row">
|
||||
<input name="score_min" inputmode="numeric" placeholder="От" value="{{ score_min }}">
|
||||
<input name="score_max" inputmode="numeric" placeholder="До" value="{{ score_max }}">
|
||||
</div>
|
||||
</div>
|
||||
<div class="filter-section">
|
||||
<div class="filter-title">Дата поста</div>
|
||||
<div class="range-row">
|
||||
<input type="date" name="date_from" value="{{ date_from }}">
|
||||
<input type="date" name="date_to" value="{{ date_to }}">
|
||||
</div>
|
||||
</div>
|
||||
<div class="filter-section">
|
||||
<div class="filter-title">Источник</div>
|
||||
<input class="facet-search" type="search" placeholder="Найти источник" data-facet-search="editor-sources">
|
||||
<div class="facet-list" data-facet-list="editor-sources">
|
||||
{% for item in facets.sources %}
|
||||
<label class="facet-option"><input type="checkbox" name="source_id" value="{{ item.id }}" {% if item.id in source_ids %}checked{% endif %}> <span>{{ item.name }}</span><small>{{ item.count }}</small></label>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="filter-section">
|
||||
<div class="filter-title">Категория</div>
|
||||
<input class="facet-search" type="search" placeholder="Найти категорию" data-facet-search="editor-categories">
|
||||
<div class="facet-list" data-facet-list="editor-categories">
|
||||
{% for item in facets.categories %}
|
||||
<label class="facet-option"><input type="checkbox" name="category" value="{{ item.value }}" {% if item.value in categories_selected %}checked{% endif %}> <span>{{ item.value }}</span><small>{{ item.count }}</small></label>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="filter-section">
|
||||
<div class="filter-title">Статус редакции</div>
|
||||
<div class="facet-list compact">
|
||||
{% for st in status_options %}
|
||||
<label class="facet-option"><input type="checkbox" name="editorial_status" value="{{ st.value }}" {% if st.value in editorial_statuses %}checked{% endif %}> <span>{{ st.label }}</span><small>{{ counts.get(st.value, 0) }}</small></label>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
<button class="btn primary filter-apply" type="submit">Показать</button>
|
||||
</form>
|
||||
</aside>
|
||||
</div>
|
||||
|
||||
<dialog class="gallery-modal" id="galleryModal" aria-hidden="true">
|
||||
<button class="gallery-close" type="button" aria-label="Закрыть">×</button>
|
||||
<button class="gallery-nav gallery-prev" type="button" aria-label="Назад">‹</button>
|
||||
<img id="galleryImage" src="" alt="">
|
||||
<button class="gallery-nav gallery-next" type="button" aria-label="Вперёд">›</button>
|
||||
<div class="gallery-title" id="galleryTitle"></div>
|
||||
</dialog>
|
||||
|
||||
<script>
|
||||
(() => {
|
||||
document.querySelectorAll("[data-open-editor]").forEach((button) => {
|
||||
button.addEventListener("click", () => document.getElementById(button.dataset.openEditor)?.showModal());
|
||||
});
|
||||
document.querySelectorAll("[data-close-dialog]").forEach((button) => {
|
||||
button.addEventListener("click", () => document.getElementById(button.dataset.closeDialog)?.close());
|
||||
});
|
||||
document.querySelectorAll(".edit-form").forEach((form) => {
|
||||
const category = form.querySelector('[name="final_category"]');
|
||||
const source = form.querySelector('[name="final_source_tag"]');
|
||||
const preview = form.querySelector("[data-hashtag-preview]");
|
||||
const normalize = (value, fallback) => {
|
||||
const tag = String(value || "").trim().replace(/^#+/, "").toLowerCase()
|
||||
.replace(/\s+/g, "_")
|
||||
.replace(/[^\wа-яё_]+/gi, "_")
|
||||
.replace(/_+/g, "_")
|
||||
.replace(/^_+|_+$/g, "");
|
||||
return tag || fallback;
|
||||
};
|
||||
const update = () => {
|
||||
if (!preview) return;
|
||||
preview.textContent = `#${normalize(category?.value, "category")} #${normalize(source?.value, "source")}`;
|
||||
};
|
||||
category?.addEventListener("change", update);
|
||||
source?.addEventListener("input", update);
|
||||
update();
|
||||
});
|
||||
|
||||
const items = Array.from(document.querySelectorAll("[data-gallery-src]"));
|
||||
const modal = document.getElementById("galleryModal");
|
||||
const image = document.getElementById("galleryImage");
|
||||
const title = document.getElementById("galleryTitle");
|
||||
let index = 0;
|
||||
function show(nextIndex) {
|
||||
if (!items.length) return;
|
||||
index = (nextIndex + items.length) % items.length;
|
||||
const item = items[index];
|
||||
image.src = item.dataset.gallerySrc;
|
||||
title.textContent = item.dataset.galleryTitle || "";
|
||||
modal.classList.add("open");
|
||||
modal.setAttribute("aria-hidden", "false");
|
||||
if (typeof modal.showModal === "function" && !modal.open) modal.showModal();
|
||||
}
|
||||
function close() {
|
||||
modal.classList.remove("open");
|
||||
modal.setAttribute("aria-hidden", "true");
|
||||
image.src = "";
|
||||
if (typeof modal.close === "function" && modal.open) modal.close();
|
||||
}
|
||||
items.forEach((item, i) => item.addEventListener("click", () => show(i)));
|
||||
modal.querySelector(".gallery-close").addEventListener("click", close);
|
||||
modal.querySelector(".gallery-prev").addEventListener("click", () => show(index - 1));
|
||||
modal.querySelector(".gallery-next").addEventListener("click", () => show(index + 1));
|
||||
modal.addEventListener("click", (event) => { if (event.target === modal) close(); });
|
||||
document.querySelectorAll("[data-autosubmit]").forEach((control) => {
|
||||
control.addEventListener("change", () => control.form?.submit());
|
||||
});
|
||||
document.querySelectorAll("[data-facet-search]").forEach((input) => {
|
||||
input.addEventListener("input", () => {
|
||||
const list = document.querySelector(`[data-facet-list="${input.dataset.facetSearch}"]`);
|
||||
const q = input.value.trim().toLowerCase();
|
||||
list?.querySelectorAll(".facet-option").forEach((option) => {
|
||||
option.hidden = q && !option.textContent.toLowerCase().includes(q);
|
||||
});
|
||||
});
|
||||
});
|
||||
document.querySelectorAll("[data-mark-media-delete]").forEach((button) => {
|
||||
button.addEventListener("click", () => {
|
||||
const form = button.closest("form");
|
||||
const media = button.closest("[data-media-id]");
|
||||
const mediaId = button.dataset.markMediaDelete;
|
||||
if (!form || !mediaId || form.querySelector(`input[name="delete_media_ids"][value="${mediaId}"]`)) return;
|
||||
const hidden = document.createElement("input");
|
||||
hidden.type = "hidden";
|
||||
hidden.name = "delete_media_ids";
|
||||
hidden.value = mediaId;
|
||||
form.appendChild(hidden);
|
||||
media?.classList.add("marked-delete");
|
||||
});
|
||||
});
|
||||
document.querySelectorAll('input[type="file"][name="media_files"]').forEach((input) => {
|
||||
input.addEventListener("change", () => {
|
||||
const list = input.closest(".context-pane")?.querySelector("[data-pending-media-list]");
|
||||
if (!list) return;
|
||||
const names = Array.from(input.files || []).map((file) => file.name);
|
||||
list.textContent = names.length ? `Будет добавлено: ${names.join(", ")}` : "Выбранные файлы появятся после сохранения.";
|
||||
});
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,12 @@
|
||||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
<div class="panel" style="max-width:420px;margin:10vh auto 0;">
|
||||
<h1>Вход</h1>
|
||||
{% if error %}<p class="pill bad">{{ error }}</p>{% endif %}
|
||||
<form method="post" action="/login">
|
||||
<label><span>Логин</span><input name="login" autocomplete="username" required></label>
|
||||
<label><span>Пароль</span><input name="password" type="password" autocomplete="current-password" required></label>
|
||||
<button class="btn primary" type="submit">Войти</button>
|
||||
</form>
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,37 @@
|
||||
{% if pagination and pagination.pages > 1 %}
|
||||
<div class="pagination">
|
||||
<form method="get" class="pagination-form">
|
||||
{% if preserved_query is defined %}
|
||||
{% for item in preserved_query %}
|
||||
<input type="hidden" name="{{ item.key }}" value="{{ item.value }}">
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{% if q is defined %}<input type="hidden" name="q" value="{{ q }}">{% endif %}
|
||||
{% if status_filter is defined %}<input type="hidden" name="status_filter" value="{{ status_filter }}">{% endif %}
|
||||
{% if date_from is defined %}<input type="hidden" name="date_from" value="{{ date_from }}">{% endif %}
|
||||
{% if date_to is defined %}<input type="hidden" name="date_to" value="{{ date_to }}">{% endif %}
|
||||
{% if category_filter is defined %}<input type="hidden" name="category_filter" value="{{ category_filter }}">{% endif %}
|
||||
{% if sort is defined %}<input type="hidden" name="sort" value="{{ sort }}">{% endif %}
|
||||
{% if table_filters is defined %}
|
||||
{% for item in table_filters %}
|
||||
<input type="hidden" name="f_field" value="{{ item.field }}">
|
||||
<input type="hidden" name="f_op" value="{{ item.op }}">
|
||||
<input type="hidden" name="f_value" value="{{ item.value }}">
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% if table_sorts is defined %}
|
||||
{% for item in table_sorts %}
|
||||
<input type="hidden" name="sort_field" value="{{ item.field }}">
|
||||
<input type="hidden" name="sort_dir" value="{{ item.dir }}">
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
<input type="hidden" name="per_page" value="{{ pagination.per_page }}">
|
||||
<button class="btn" name="page" value="{{ pagination.page - 1 }}" {% if not pagination.has_prev %}disabled{% endif %}>‹</button>
|
||||
<span class="muted">Страница {{ pagination.page }} из {{ pagination.pages }} · всего {{ pagination.total }}</span>
|
||||
<button class="btn" name="page" value="{{ pagination.page + 1 }}" {% if not pagination.has_next %}disabled{% endif %}>›</button>
|
||||
</form>
|
||||
</div>
|
||||
{% else %}
|
||||
{% if pagination %}<div class="pagination muted">Всего {{ pagination.total }}</div>{% endif %}
|
||||
{% endif %}
|
||||
@@ -0,0 +1,194 @@
|
||||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
<div class="toolbar">
|
||||
<div>
|
||||
<a class="muted" href="/raw">← Raw-посты</a>
|
||||
<h1>Raw #{{ post.id }}</h1>
|
||||
<div class="muted">
|
||||
<a href="{{ post.original_url }}" target="_blank">{{ post.source_name }}</a>
|
||||
· {{ post.source_platform }}
|
||||
· #{{ post.source_tag or "—" }}
|
||||
{% if post.posted_at_fmt %}· пост VK: {{ post.posted_at_fmt }}{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<span class="pill {{ post.stage.class }}" title="{{ post.stage.key }}">{{ post.stage.label }}</span>
|
||||
</div>
|
||||
|
||||
<div class="detail-grid">
|
||||
<section class="panel">
|
||||
<div class="media-strip detail-media">
|
||||
{% for m in post.media_items %}
|
||||
{% if m.type == "photo" and m.url %}
|
||||
<button class="media-thumb detail-thumb" type="button" data-gallery-src="{{ m.url }}" data-gallery-title="#{{ post.id }} · фото {{ loop.index }}" title="{{ m.status }}">
|
||||
<img src="{{ m.url }}" alt="photo">
|
||||
</button>
|
||||
{% elif m.url %}
|
||||
<a class="media-thumb detail-thumb media-video" href="{{ m.url }}" target="_blank" title="{{ m.error or m.status }}">
|
||||
<span>▶</span>
|
||||
</a>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% if post.media_count %}<div class="muted media-count">{{ post.media_count }} медиа</div>{% endif %}
|
||||
<div class="raw-full-text detail-text">{{ post.raw_text }}</div>
|
||||
<div class="post-meta">Загружен парсером: {{ post.created_at_fmt }}</div>
|
||||
</section>
|
||||
|
||||
<aside class="panel side-panel" id="ai">
|
||||
<h2>AI-оценка</h2>
|
||||
<a class="ai-badge detail-ai {{ post.ai_badge.class }}" href="#ai" title="{{ post.ai_badge.reason or post.ai_badge.sublabel }}">
|
||||
<span>{{ post.ai_badge.label }}</span>
|
||||
<small>{{ post.ai_badge.sublabel }}</small>
|
||||
</a>
|
||||
<dl class="kv">
|
||||
<dt>Статус</dt>
|
||||
<dd>{{ post.qualification_status or "pending" }}</dd>
|
||||
<dt>Решение модели</dt>
|
||||
<dd>{{ post.qualification_model_decision or post.qualification_decision or "—" }}</dd>
|
||||
<dt>Итог по порогу</dt>
|
||||
<dd>{{ post.qualification_decision or "—" }}</dd>
|
||||
<dt>Причина</dt>
|
||||
<dd>{{ post.qualification_reason or "—" }}</dd>
|
||||
<dt>Reject tag</dt>
|
||||
<dd>{{ post.qualification_reject_tag or "—" }}</dd>
|
||||
<dt>Модель</dt>
|
||||
<dd>{{ post.qualification_model or post.batch_model or "—" }}</dd>
|
||||
<dt>Проверен</dt>
|
||||
<dd>{{ post.qualified_at_fmt or "—" }}</dd>
|
||||
<dt>Batch</dt>
|
||||
<dd>{% if post.qualification_batch_id %}#{{ post.qualification_batch_id }}{% else %}—{% endif %}</dd>
|
||||
<dt>Токены</dt>
|
||||
<dd>{{ post.batch_total_tokens or "—" }}</dd>
|
||||
</dl>
|
||||
</aside>
|
||||
</div>
|
||||
|
||||
<section class="panel" id="rewrite">
|
||||
<div class="toolbar">
|
||||
<div>
|
||||
<h2>Рерайт</h2>
|
||||
<div class="muted">Готовый текст для будущей редакторской проверки.</div>
|
||||
</div>
|
||||
<span class="pill {% if post.rewrite_status == 'ready' %}ok{% elif post.rewrite_status == 'failed' %}bad{% elif post.rewrite_status == 'processing' %}warn{% endif %}">
|
||||
{{ post.rewrite_status or "pending" }}
|
||||
</span>
|
||||
</div>
|
||||
{% if post.rewritten_text %}
|
||||
<div class="rewrite-text">{{ post.rewritten_text }}</div>
|
||||
{% else %}
|
||||
<div class="muted">Рерайт ещё не готов.</div>
|
||||
{% endif %}
|
||||
<dl class="kv kv-wide rewrite-meta">
|
||||
<dt>Категория</dt><dd>{{ post.rewrite_category or "—" }}</dd>
|
||||
<dt>Хэштеги</dt>
|
||||
<dd>
|
||||
{% if post.rewrite_category_tag or post.rewrite_source_tag %}
|
||||
#{{ post.rewrite_category_tag or "—" }} #{{ post.rewrite_source_tag or "—" }}
|
||||
{% else %}
|
||||
—
|
||||
{% endif %}
|
||||
</dd>
|
||||
<dt>Заметка</dt><dd>{{ post.rewrite_notes or "—" }}</dd>
|
||||
<dt>Модель</dt><dd>{{ post.rewrite_model or post.writer_batch_model or "—" }}</dd>
|
||||
<dt>Готов</dt><dd>{{ post.rewritten_at_fmt or "—" }}</dd>
|
||||
<dt>Batch</dt><dd>{% if post.rewrite_batch_id %}#{{ post.rewrite_batch_id }}{% else %}—{% endif %}</dd>
|
||||
<dt>Токены</dt><dd>{{ post.writer_total_tokens or "—" }}</dd>
|
||||
<dt>Стоимость</dt><dd>{% if post.writer_estimated_cost_usd %}${{ post.writer_estimated_cost_usd }}{% else %}—{% endif %}</dd>
|
||||
</dl>
|
||||
</section>
|
||||
|
||||
{% if post.qualification_batch_id %}
|
||||
<details class="panel">
|
||||
<summary><strong>Технический ответ AI</strong></summary>
|
||||
<dl class="kv kv-wide">
|
||||
<dt>Provider</dt><dd>{{ post.qualification_provider or "—" }}</dd>
|
||||
<dt>Batch status</dt><dd>{{ post.batch_status or "—" }}</dd>
|
||||
<dt>Создан</dt><dd>{{ post.batch_created_at_fmt or "—" }}</dd>
|
||||
<dt>Завершён</dt><dd>{{ post.batch_completed_at_fmt or "—" }}</dd>
|
||||
<dt>Постов в batch</dt><dd>{{ post.batch_posts_count or "—" }}</dd>
|
||||
<dt>Accepted / rejected / maybe</dt>
|
||||
<dd>{{ post.batch_accepted_count or 0 }} / {{ post.batch_rejected_count or 0 }} / {{ post.batch_maybe_count or 0 }}</dd>
|
||||
<dt>Токены</dt>
|
||||
<dd>{{ post.batch_prompt_tokens or 0 }} / {{ post.batch_completion_tokens or 0 }} / {{ post.batch_total_tokens or 0 }}</dd>
|
||||
<dt>Стоимость</dt><dd>{% if post.batch_estimated_cost_usd %}${{ post.batch_estimated_cost_usd }}{% else %}—{% endif %}</dd>
|
||||
<dt>Ошибка</dt><dd>{{ post.batch_error or "—" }}</dd>
|
||||
</dl>
|
||||
{% if post.batch_prompt_text %}
|
||||
<details class="nested-details">
|
||||
<summary>Использованный prompt квалификатора</summary>
|
||||
<pre class="prompt-box">{{ post.batch_prompt_text }}</pre>
|
||||
</details>
|
||||
{% endif %}
|
||||
<pre class="json-box">{{ post.batch_response_pretty or "{}" }}</pre>
|
||||
</details>
|
||||
{% endif %}
|
||||
|
||||
{% if post.rewrite_batch_id %}
|
||||
<details class="panel">
|
||||
<summary><strong>Технический ответ AI-райтера</strong></summary>
|
||||
<dl class="kv kv-wide">
|
||||
<dt>Provider</dt><dd>{{ post.rewrite_provider or "—" }}</dd>
|
||||
<dt>Batch status</dt><dd>{{ post.writer_batch_status or "—" }}</dd>
|
||||
<dt>Создан</dt><dd>{{ post.writer_batch_created_at_fmt or "—" }}</dd>
|
||||
<dt>Завершён</dt><dd>{{ post.writer_batch_completed_at_fmt or "—" }}</dd>
|
||||
<dt>Постов в batch</dt><dd>{{ post.writer_batch_posts_count or "—" }}</dd>
|
||||
<dt>Ready / failed</dt><dd>{{ post.writer_batch_ready_count or 0 }} / {{ post.writer_batch_failed_count or 0 }}</dd>
|
||||
<dt>Токены</dt>
|
||||
<dd>{{ post.writer_prompt_tokens or 0 }} / {{ post.writer_completion_tokens or 0 }} / {{ post.writer_total_tokens or 0 }}</dd>
|
||||
<dt>Стоимость</dt><dd>{% if post.writer_estimated_cost_usd %}${{ post.writer_estimated_cost_usd }}{% else %}—{% endif %}</dd>
|
||||
<dt>Ошибка</dt><dd>{{ post.writer_batch_error or "—" }}</dd>
|
||||
</dl>
|
||||
{% if post.writer_batch_prompt_text %}
|
||||
<details class="nested-details">
|
||||
<summary>Использованный prompt райтера</summary>
|
||||
<pre class="prompt-box">{{ post.writer_batch_prompt_text }}</pre>
|
||||
</details>
|
||||
{% endif %}
|
||||
<pre class="json-box">{{ post.writer_batch_response_pretty or "{}" }}</pre>
|
||||
</details>
|
||||
{% endif %}
|
||||
|
||||
<div class="gallery-modal" id="galleryModal" aria-hidden="true">
|
||||
<button class="gallery-close" type="button" aria-label="Закрыть">×</button>
|
||||
<button class="gallery-nav gallery-prev" type="button" aria-label="Назад">‹</button>
|
||||
<img id="galleryImage" src="" alt="">
|
||||
<button class="gallery-nav gallery-next" type="button" aria-label="Вперёд">›</button>
|
||||
<div class="gallery-title" id="galleryTitle"></div>
|
||||
</div>
|
||||
<script>
|
||||
(() => {
|
||||
const items = Array.from(document.querySelectorAll("[data-gallery-src]"));
|
||||
const modal = document.getElementById("galleryModal");
|
||||
const image = document.getElementById("galleryImage");
|
||||
const title = document.getElementById("galleryTitle");
|
||||
let index = 0;
|
||||
|
||||
function show(nextIndex) {
|
||||
if (!items.length) return;
|
||||
index = (nextIndex + items.length) % items.length;
|
||||
const item = items[index];
|
||||
image.src = item.dataset.gallerySrc;
|
||||
title.textContent = item.dataset.galleryTitle || "";
|
||||
modal.classList.add("open");
|
||||
modal.setAttribute("aria-hidden", "false");
|
||||
}
|
||||
function close() {
|
||||
modal.classList.remove("open");
|
||||
modal.setAttribute("aria-hidden", "true");
|
||||
image.src = "";
|
||||
}
|
||||
|
||||
items.forEach((item, i) => item.addEventListener("click", () => show(i)));
|
||||
modal.querySelector(".gallery-close").addEventListener("click", close);
|
||||
modal.querySelector(".gallery-prev").addEventListener("click", () => show(index - 1));
|
||||
modal.querySelector(".gallery-next").addEventListener("click", () => show(index + 1));
|
||||
modal.addEventListener("click", (event) => { if (event.target === modal) close(); });
|
||||
document.addEventListener("keydown", (event) => {
|
||||
if (!modal.classList.contains("open")) return;
|
||||
if (event.key === "Escape") close();
|
||||
if (event.key === "ArrowLeft") show(index - 1);
|
||||
if (event.key === "ArrowRight") show(index + 1);
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,211 @@
|
||||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
<div class="toolbar">
|
||||
<div>
|
||||
<h1>Raw-посты</h1>
|
||||
<div class="muted">Исходный пост, этап обработки и AI-оценка.</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="catalog-layout">
|
||||
<section class="panel catalog-main">
|
||||
<div class="list-toolbar">
|
||||
<div class="muted">Всего: {{ pagination.total }}</div>
|
||||
</div>
|
||||
<table class="raw-table">
|
||||
<tr>
|
||||
<th><a class="sort-head {% if sort_headers.id.active %}active {{ sort_headers.id.direction }}{% endif %}" href="{{ sort_headers.id.url }}">#</a></th>
|
||||
<th><a class="sort-head {% if sort_headers.source.active %}active {{ sort_headers.source.direction }}{% endif %}" href="{{ sort_headers.source.url }}">Источник</a></th>
|
||||
<th><a class="sort-head {% if sort_headers.stage.active %}active {{ sort_headers.stage.direction }}{% endif %}" href="{{ sort_headers.stage.url }}">Этап</a></th>
|
||||
<th><a class="sort-head {% if sort_headers.post.active %}active {{ sort_headers.post.direction }}{% endif %}" href="{{ sort_headers.post.url }}">Пост</a></th>
|
||||
<th><a class="sort-head {% if sort_headers.ai.active %}active {{ sort_headers.ai.direction }}{% endif %}" href="{{ sort_headers.ai.url }}">AI</a></th>
|
||||
</tr>
|
||||
{% for p in posts %}
|
||||
<tr>
|
||||
<td><a class="raw-id" href="/raw/{{ p.id }}">{{ p.id }}</a></td>
|
||||
<td class="source-cell">
|
||||
<a href="{{ p.original_url }}" target="_blank">{{ p.source_name }}</a>
|
||||
<div class="muted">{{ p.source_platform }} · #{{ p.source_tag or "—" }}</div>
|
||||
{% if p.posted_at_fmt %}<div class="muted">{{ p.posted_at_fmt }}</div>{% endif %}
|
||||
</td>
|
||||
<td class="stage-cell">
|
||||
<span class="pill {{ p.stage.class }}" title="{{ p.stage.key }}">{{ p.stage.label }}</span>
|
||||
{% if p.error_reason %}<div class="muted">{{ p.error_reason }}</div>{% endif %}
|
||||
</td>
|
||||
<td class="raw-content-cell">
|
||||
<div class="media-strip">
|
||||
{% for m in p.media_items %}
|
||||
{% if m.type == "photo" and m.url %}
|
||||
<button class="media-thumb" type="button" data-gallery-src="{{ m.url }}" data-gallery-title="#{{ p.id }} · фото {{ loop.index }}" title="{{ m.status }}">
|
||||
<img src="{{ m.url }}" alt="photo">
|
||||
</button>
|
||||
{% elif m.url %}
|
||||
<a class="media-thumb media-video" href="{{ m.url }}" target="_blank" title="{{ m.error or m.status }}">
|
||||
<span>▶</span>
|
||||
</a>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% if p.media_count %}<div class="muted media-count">{{ p.media_count }} медиа</div>{% endif %}
|
||||
{% if p.raw_text|length > 520 %}
|
||||
<details class="raw-text-details">
|
||||
<summary>{{ p.raw_text[:520] }}...</summary>
|
||||
<div class="raw-full-text">{{ p.raw_text }}</div>
|
||||
</details>
|
||||
{% else %}
|
||||
<div class="raw-full-text">{{ p.raw_text }}</div>
|
||||
{% endif %}
|
||||
<div class="post-meta">Загружен парсером: {{ p.created_at_fmt }}</div>
|
||||
</td>
|
||||
<td class="ai-cell">
|
||||
<a class="ai-badge {{ p.ai_badge.class }}" href="/raw/{{ p.id }}#ai" title="{{ p.ai_badge.reason or p.ai_badge.sublabel }}">
|
||||
<span>{{ p.ai_badge.label }}</span>
|
||||
<small>итог: {{ p.ai_badge.sublabel }}</small>
|
||||
</a>
|
||||
{% if p.qualification_model_decision %}<div class="muted ai-model-decision">модель: {{ p.qualification_model_decision }}</div>{% endif %}
|
||||
{% if p.rewrite_category %}<div class="muted ai-model-decision">{{ p.rewrite_category }}</div>{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
<div class="list-footer">
|
||||
<form class="row per-page-form" method="get" action="/raw">
|
||||
{% for item in preserved_query %}
|
||||
{% if item.key != "per_page" %}
|
||||
<input type="hidden" name="{{ item.key }}" value="{{ item.value }}">
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
<label><span>На странице</span>
|
||||
<select name="per_page" data-autosubmit>
|
||||
{% for n in [25,50,100,200] %}
|
||||
<option value="{{ n }}" {% if pagination.per_page == n %}selected{% endif %}>{{ n }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</label>
|
||||
</form>
|
||||
{% include "pagination.html" %}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<aside class="filter-sidebar panel">
|
||||
<form method="get" action="/raw" data-filter-form>
|
||||
<input type="hidden" name="sort" value="{{ sort }}">
|
||||
<div class="filter-head">
|
||||
<h2>Фильтры</h2>
|
||||
<a href="/raw">Сбросить</a>
|
||||
</div>
|
||||
<div class="filter-section">
|
||||
<div class="filter-title">Оценка AI</div>
|
||||
<div class="range-row">
|
||||
<input name="score_min" inputmode="numeric" placeholder="От" value="{{ score_min }}">
|
||||
<input name="score_max" inputmode="numeric" placeholder="До" value="{{ score_max }}">
|
||||
</div>
|
||||
</div>
|
||||
<div class="filter-section">
|
||||
<div class="filter-title">Дата поста</div>
|
||||
<div class="range-row">
|
||||
<input type="date" name="date_from" value="{{ date_from }}">
|
||||
<input type="date" name="date_to" value="{{ date_to }}">
|
||||
</div>
|
||||
</div>
|
||||
<div class="filter-section">
|
||||
<div class="filter-title">Источник</div>
|
||||
<input class="facet-search" type="search" placeholder="Найти источник" data-facet-search="raw-sources">
|
||||
<div class="facet-list" data-facet-list="raw-sources">
|
||||
{% for item in facets.sources %}
|
||||
<label class="facet-option"><input type="checkbox" name="source_id" value="{{ item.id }}" {% if item.id in source_ids %}checked{% endif %}> <span>{{ item.name }}</span><small>{{ item.count }}</small></label>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="filter-section">
|
||||
<div class="filter-title">Категория</div>
|
||||
<input class="facet-search" type="search" placeholder="Найти категорию" data-facet-search="raw-categories">
|
||||
<div class="facet-list" data-facet-list="raw-categories">
|
||||
{% for item in facets.categories %}
|
||||
<label class="facet-option"><input type="checkbox" name="category" value="{{ item.value }}" {% if item.value in categories_selected %}checked{% endif %}> <span>{{ item.value }}</span><small>{{ item.count }}</small></label>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="filter-section">
|
||||
<div class="filter-title">Raw-статус</div>
|
||||
<div class="facet-list compact">
|
||||
{% for item in facets.statuses %}
|
||||
<label class="facet-option"><input type="checkbox" name="raw_status" value="{{ item.value }}" {% if item.value in raw_statuses %}checked{% endif %}> <span>{{ item.value }}</span><small>{{ item.count }}</small></label>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="filter-section">
|
||||
<div class="filter-title">Квалификация</div>
|
||||
<div class="facet-list compact">
|
||||
{% for item in facets.qualification_statuses %}
|
||||
<label class="facet-option"><input type="checkbox" name="qualification_status" value="{{ item.value }}" {% if item.value in qualification_statuses %}checked{% endif %}> <span>{{ item.value }}</span><small>{{ item.count }}</small></label>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="filter-section">
|
||||
<div class="filter-title">Рерайт</div>
|
||||
<div class="facet-list compact">
|
||||
{% for item in facets.rewrite_statuses %}
|
||||
<label class="facet-option"><input type="checkbox" name="rewrite_status" value="{{ item.value }}" {% if item.value in rewrite_statuses %}checked{% endif %}> <span>{{ item.value }}</span><small>{{ item.count }}</small></label>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
<button class="btn primary filter-apply" type="submit">Показать</button>
|
||||
</form>
|
||||
</aside>
|
||||
</div>
|
||||
<div class="gallery-modal" id="galleryModal" aria-hidden="true">
|
||||
<button class="gallery-close" type="button" aria-label="Закрыть">×</button>
|
||||
<button class="gallery-nav gallery-prev" type="button" aria-label="Назад">‹</button>
|
||||
<img id="galleryImage" src="" alt="">
|
||||
<button class="gallery-nav gallery-next" type="button" aria-label="Вперёд">›</button>
|
||||
<div class="gallery-title" id="galleryTitle"></div>
|
||||
</div>
|
||||
<script>
|
||||
(() => {
|
||||
const items = Array.from(document.querySelectorAll("[data-gallery-src]"));
|
||||
const modal = document.getElementById("galleryModal");
|
||||
const image = document.getElementById("galleryImage");
|
||||
const title = document.getElementById("galleryTitle");
|
||||
let index = 0;
|
||||
|
||||
function show(nextIndex) {
|
||||
if (!items.length) return;
|
||||
index = (nextIndex + items.length) % items.length;
|
||||
const item = items[index];
|
||||
image.src = item.dataset.gallerySrc;
|
||||
title.textContent = item.dataset.galleryTitle || "";
|
||||
modal.classList.add("open");
|
||||
modal.setAttribute("aria-hidden", "false");
|
||||
}
|
||||
function close() {
|
||||
modal.classList.remove("open");
|
||||
modal.setAttribute("aria-hidden", "true");
|
||||
image.src = "";
|
||||
}
|
||||
|
||||
items.forEach((item, i) => item.addEventListener("click", () => show(i)));
|
||||
modal.querySelector(".gallery-close").addEventListener("click", close);
|
||||
modal.querySelector(".gallery-prev").addEventListener("click", () => show(index - 1));
|
||||
modal.querySelector(".gallery-next").addEventListener("click", () => show(index + 1));
|
||||
modal.addEventListener("click", (event) => { if (event.target === modal) close(); });
|
||||
document.addEventListener("keydown", (event) => {
|
||||
if (!modal.classList.contains("open")) return;
|
||||
if (event.key === "Escape") close();
|
||||
if (event.key === "ArrowLeft") show(index - 1);
|
||||
if (event.key === "ArrowRight") show(index + 1);
|
||||
});
|
||||
document.querySelectorAll("[data-autosubmit]").forEach((control) => {
|
||||
control.addEventListener("change", () => control.form?.submit());
|
||||
});
|
||||
document.querySelectorAll("[data-facet-search]").forEach((input) => {
|
||||
input.addEventListener("input", () => {
|
||||
const list = document.querySelector(`[data-facet-list="${input.dataset.facetSearch}"]`);
|
||||
const q = input.value.trim().toLowerCase();
|
||||
list?.querySelectorAll(".facet-option").forEach((option) => {
|
||||
option.hidden = q && !option.textContent.toLowerCase().includes(q);
|
||||
});
|
||||
});
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,25 @@
|
||||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
<div class="toolbar">
|
||||
<h1>{{ title }}</h1>
|
||||
<a class="btn" href="/sources">Назад</a>
|
||||
</div>
|
||||
<div class="panel">
|
||||
<form method="post" action="{{ action }}">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<div class="form-grid">
|
||||
<label><span>Площадка</span>
|
||||
<select name="platform">
|
||||
<option value="vk" {% if not source or source.platform == "vk" %}selected{% endif %}>VK</option>
|
||||
</select>
|
||||
</label>
|
||||
<label><span>Приоритет</span><input name="priority" type="number" value="{{ source.priority if source else 100 }}"></label>
|
||||
<label><span>Название</span><input name="name" value="{{ source.name if source else '' }}" required></label>
|
||||
<label><span>Тэг</span><input name="tag" value="{{ source.tag if source else '' }}" placeholder="academy_gear"></label>
|
||||
<label><span>Ссылка</span><input name="url" value="{{ source.url if source else '' }}" required></label>
|
||||
<label><span>Активен</span><input name="active" type="checkbox" {% if not source or source.active %}checked{% endif %}></label>
|
||||
</div>
|
||||
<button class="btn primary" type="submit">Сохранить</button>
|
||||
</form>
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,123 @@
|
||||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
<div class="toolbar">
|
||||
<div>
|
||||
<h1>Источники</h1>
|
||||
<div class="muted">VK-источники для парсинга. Название идёт в prompt, тэг — в будущие хэштеги.</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<details class="panel" {% if source_preview %}open{% endif %}>
|
||||
<summary><strong>Добавить источники</strong></summary>
|
||||
<div class="source-add-grid">
|
||||
<form method="post" action="/sources/new">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<input type="hidden" name="platform" value="vk">
|
||||
<input type="hidden" name="priority" value="100">
|
||||
<label><span>Название</span><input name="name" placeholder="Можно оставить как в VK"></label>
|
||||
<label><span>Тэг</span><input name="tag" placeholder="academy_gear"></label>
|
||||
<label><span>Ссылка</span><input name="url" placeholder="https://vk.com/academy_gear" required></label>
|
||||
<label class="checkline"><input name="active" type="checkbox" checked> <span>Включить сразу</span></label>
|
||||
<button class="btn primary" type="submit">Добавить</button>
|
||||
</form>
|
||||
|
||||
<form method="post" action="/sources/preview">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<input type="hidden" name="q" value="{{ q }}">
|
||||
<input type="hidden" name="status_filter" value="{{ status_filter }}">
|
||||
<input type="hidden" name="per_page" value="{{ pagination.per_page }}">
|
||||
<label><span>Список источников</span>
|
||||
<textarea name="bulk_text" rows="7" placeholder="Academy Gear academy_gear https://vk.com/academy_gear A2 Technologies a2technologies https://vk.com/a2technologies">{{ bulk_text }}</textarea>
|
||||
</label>
|
||||
<label class="checkline"><input name="bulk_active" type="checkbox" {% if bulk_active %}checked{% endif %}> <span>Включить после добавления</span></label>
|
||||
<button class="btn" type="submit">Проверить список</button>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
{% if source_preview %}
|
||||
<div class="preview-block">
|
||||
<h3>Предпросмотр</h3>
|
||||
<table>
|
||||
<tr><th>#</th><th>Название</th><th>Тэг</th><th>Ссылка</th><th>VK id</th><th>Проверка</th></tr>
|
||||
{% for item in source_preview %}
|
||||
<tr>
|
||||
<td>{{ item.line_no }}</td>
|
||||
<td>{{ item.name or "—" }}</td>
|
||||
<td class="mono">#{{ item.tag or "—" }}</td>
|
||||
<td><a href="{{ item.url }}" target="_blank">{{ item.url }}</a></td>
|
||||
<td class="mono">{{ item.external_id }}<br>{{ item.external_owner_id or "" }}</td>
|
||||
<td>{% if item.ok %}<span class="pill ok">готов</span>{% else %}<span class="pill bad">{{ item.error }}</span>{% endif %}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
<form method="post" action="/sources/bulk" class="row">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<input type="hidden" name="bulk_payload" value='{{ source_preview|tojson }}'>
|
||||
<button class="btn primary" type="submit">Добавить прошедшие проверку</button>
|
||||
<span class="muted">Строки с ошибками и дубли будут пропущены.</span>
|
||||
</form>
|
||||
</div>
|
||||
{% endif %}
|
||||
</details>
|
||||
|
||||
<div class="panel">
|
||||
<form class="filter-grid sources-filter" method="get" action="/sources">
|
||||
<label><span>Поиск</span><input name="q" value="{{ q }}" placeholder="название, ссылка, id"></label>
|
||||
<label><span>Статус</span>
|
||||
<select name="status_filter">
|
||||
<option value="">Все</option>
|
||||
{% for st in ["new","ok","paused","error"] %}
|
||||
<option value="{{ st }}" {% if status_filter == st %}selected{% endif %}>{{ st }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</label>
|
||||
<label><span>На странице</span>
|
||||
<select name="per_page">
|
||||
{% for n in [25,50,100,200] %}
|
||||
<option value="{{ n }}" {% if pagination.per_page == n %}selected{% endif %}>{{ n }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</label>
|
||||
{% include "table_tools.html" %}
|
||||
<button class="btn filter-submit" type="submit">Показать</button>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div class="panel">
|
||||
<table class="compact-table">
|
||||
<tr>
|
||||
<th>#</th><th>Площадка</th><th>Название</th><th>Тэг</th><th>Ссылка</th><th>Статус</th><th>Посты</th><th>Последний парсинг</th><th></th>
|
||||
</tr>
|
||||
{% for s in sources %}
|
||||
<tr>
|
||||
<td>{{ s.id }}</td>
|
||||
<td>{{ s.platform }}</td>
|
||||
<td>
|
||||
<b>{{ s.name }}</b>
|
||||
{% if s.active %}<span class="pill ok source-state">on</span>{% else %}<span class="pill bad source-state">off</span>{% endif %}
|
||||
<div class="muted mono">{{ s.external_id or "" }}</div>
|
||||
</td>
|
||||
<td class="mono">#{{ s.tag or "—" }}</td>
|
||||
<td class="source-url"><a href="{{ s.url }}" target="_blank">{{ s.url }}</a></td>
|
||||
<td><span class="pill {% if s.status == 'ok' %}ok{% elif s.status == 'error' %}bad{% endif %}">{{ s.status }}</span>{% if s.status_msg %}<div class="muted status-msg">{{ s.status_msg }}</div>{% endif %}</td>
|
||||
<td>{{ s.posts_count }} <span class="muted">+{{ s.posts_24h }}/24ч</span></td>
|
||||
<td>{{ s.last_parsed_at_fmt or "нет" }}</td>
|
||||
<td>
|
||||
<div class="icon-actions">
|
||||
<a class="icon-btn" href="/sources/{{ s.id }}/edit" title="Править" aria-label="Править">✎</a>
|
||||
<form method="post" action="/sources/{{ s.id }}/toggle">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<button class="icon-btn" type="submit" title="{% if s.active %}Выключить{% else %}Включить{% endif %}" aria-label="Вкл/выкл">⏻</button>
|
||||
</form>
|
||||
<form method="post" action="/sources/{{ s.id }}/delete" onsubmit="return confirm('Удалить источник {{ s.name }}?');">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<button class="icon-btn danger" type="submit" title="Удалить" aria-label="Удалить">×</button>
|
||||
</form>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
{% include "pagination.html" %}
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,43 @@
|
||||
<details class="advanced-tools" {% if table_filters|selectattr("field")|list or table_sorts|selectattr("field")|list %}open{% endif %}>
|
||||
<summary>Расширенные фильтры и сортировка</summary>
|
||||
<div class="advanced-grid">
|
||||
<div>
|
||||
<div class="tool-title">Фильтры</div>
|
||||
{% for item in table_filters %}
|
||||
<div class="tool-row">
|
||||
<select name="f_field">
|
||||
<option value="">Поле</option>
|
||||
{% for field in field_options %}
|
||||
<option value="{{ field.value }}" {% if item.field == field.value %}selected{% endif %}>{{ field.label }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
<select name="f_op">
|
||||
{% for op in filter_op_options %}
|
||||
<option value="{{ op.value }}" {% if item.op == op.value %}selected{% endif %}>{{ op.label }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
<input name="f_value" value="{{ item.value }}" placeholder="значение">
|
||||
</div>
|
||||
{% endfor %}
|
||||
<div class="muted small-help">Пустые строки игнорируются. Для дат используй формат YYYY-MM-DD, для true/false — true или false.</div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="tool-title">Сортировка</div>
|
||||
{% for item in table_sorts %}
|
||||
<div class="tool-row sort-row">
|
||||
<select name="sort_field">
|
||||
<option value="">Поле</option>
|
||||
{% for field in field_options %}
|
||||
<option value="{{ field.value }}" {% if item.field == field.value %}selected{% endif %}>{{ field.label }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
<select name="sort_dir">
|
||||
<option value="desc" {% if item.dir == "desc" %}selected{% endif %}>по убыванию</option>
|
||||
<option value="asc" {% if item.dir == "asc" %}selected{% endif %}>по возрастанию</option>
|
||||
</select>
|
||||
</div>
|
||||
{% endfor %}
|
||||
<div class="muted small-help">Сортировки применяются сверху вниз. Если сортировка задана здесь, обычная сортировка страницы становится запасной.</div>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
@@ -0,0 +1,35 @@
|
||||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
<h1>Пользователи</h1>
|
||||
<div class="panel">
|
||||
<h2>Пользователи</h2>
|
||||
<table>
|
||||
<tr><th>ID</th><th>Логин</th><th>Роль</th><th>Создан</th><th>Активен</th><th></th></tr>
|
||||
{% for u in users %}
|
||||
<tr>
|
||||
<td>{{ u.id }}</td><td>{{ u.login }}</td><td>{{ u.role }}</td>
|
||||
<td>{{ u.created_at_fmt }}</td>
|
||||
<td>{% if u.is_active %}<span class="pill ok">yes</span>{% else %}<span class="pill bad">no</span>{% endif %}</td>
|
||||
<td>{% if user.role == "admin" and user.id != u.id %}<form method="post" action="/users/{{ u.id }}/toggle"><input type="hidden" name="csrf_token" value="{{ user.csrf_token }}"><button class="btn" type="submit">Вкл/выкл</button></form>{% endif %}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
{% include "pagination.html" %}
|
||||
</div>
|
||||
{% if user.role == "admin" %}
|
||||
<div class="panel">
|
||||
<h2>Добавить</h2>
|
||||
<form method="post" action="/users/create">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<div class="form-grid">
|
||||
<label><span>Логин</span><input name="login" required></label>
|
||||
<label><span>Пароль</span><input name="password" type="password" required minlength="10"></label>
|
||||
<label><span>Роль</span>
|
||||
<select name="role"><option value="editor">editor</option><option value="viewer">viewer</option><option value="admin">admin</option></select>
|
||||
</label>
|
||||
</div>
|
||||
<button class="btn primary" type="submit">Создать</button>
|
||||
</form>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,52 @@
|
||||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
<div class="panel" style="max-width:760px;margin:8vh auto 0;">
|
||||
<h1>VK OAuth callback</h1>
|
||||
{% if code %}
|
||||
<p class="muted">Скопируйте code и code_verifier для обмена на access token.</p>
|
||||
<label>code</label>
|
||||
<textarea readonly style="min-height:110px;" onclick="this.select()">{{ code }}</textarea>
|
||||
<label>code_verifier</label>
|
||||
<textarea readonly style="min-height:110px;" onclick="this.select()">{{ code_verifier }}</textarea>
|
||||
{% if device_id %}
|
||||
<label>device_id</label>
|
||||
<textarea readonly style="min-height:80px;" onclick="this.select()">{{ device_id }}</textarea>
|
||||
{% endif %}
|
||||
{% if state and expected_state and state != expected_state %}
|
||||
<p class="pill bad">state не совпал, такой code лучше не использовать.</p>
|
||||
{% endif %}
|
||||
{% elif error %}
|
||||
<p class="pill bad">{{ error }}</p>
|
||||
<p>{{ error_description }}</p>
|
||||
{% else %}
|
||||
<p class="muted">VK вернул callback без code.</p>
|
||||
{% endif %}
|
||||
<div id="fragment-token" style="display:none;margin-top:24px;">
|
||||
<h2>Access token сообщества</h2>
|
||||
<p class="muted">VK передал токен в fragment URL. Скопируйте значение ниже.</p>
|
||||
<textarea id="fragment-token-value" readonly style="min-height:140px;" onclick="this.select()"></textarea>
|
||||
</div>
|
||||
<div id="fragment-data" style="display:none;margin-top:16px;">
|
||||
<h2>Параметры fragment</h2>
|
||||
<textarea id="fragment-data-value" readonly style="min-height:120px;" onclick="this.select()"></textarea>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
const hash = window.location.hash.replace(/^#/, "");
|
||||
if (hash) {
|
||||
const params = new URLSearchParams(hash);
|
||||
const pairs = Array.from(params.entries());
|
||||
const tokenPair = pairs.find(([key]) => key.startsWith("access_token"));
|
||||
if (tokenPair) {
|
||||
document.getElementById("fragment-token").style.display = "block";
|
||||
document.getElementById("fragment-token-value").value = tokenPair[1];
|
||||
}
|
||||
if (pairs.length) {
|
||||
document.getElementById("fragment-data").style.display = "block";
|
||||
document.getElementById("fragment-data-value").value = pairs
|
||||
.map(([key, value]) => `${key}=${value}`)
|
||||
.join("\n");
|
||||
}
|
||||
}
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,141 @@
|
||||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
<h1>Воркеры</h1>
|
||||
<div class="panel">
|
||||
<table>
|
||||
<tr><th>Имя</th><th>Enabled</th><th>Heartbeat</th><th>Статус</th><th>Job</th><th></th></tr>
|
||||
{% for w in workers %}
|
||||
<tr>
|
||||
<td class="mono">{{ w.name }}</td>
|
||||
<td>{% if w.enabled %}<span class="pill ok">enabled</span>{% else %}<span class="pill bad">disabled</span>{% endif %}</td>
|
||||
<td>{{ w.heartbeat_at or "нет" }}</td>
|
||||
<td>{{ w.status or "" }}</td>
|
||||
<td>{{ w.current_job_id or "" }}</td>
|
||||
<td><form method="post" action="/workers/{{ w.name }}/toggle"><input type="hidden" name="csrf_token" value="{{ user.csrf_token }}"><button class="btn" type="submit">Вкл/выкл</button></form></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<details class="panel" open>
|
||||
<summary><strong>Категории публикаций</strong></summary>
|
||||
<p class="muted">Активные категории попадают в выпадающий список редактора и передаются AI-райтеру. Удаление здесь выключает категорию, но не ломает старые посты.</p>
|
||||
<form class="category-create" method="post" action="/categories/create">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<label><span>Название</span><input name="name" placeholder="например, разгрузка" required></label>
|
||||
<label><span>Тэг</span><input name="tag" placeholder="если пусто, соберётся из названия"></label>
|
||||
<button class="btn primary" type="submit">Добавить</button>
|
||||
</form>
|
||||
<table class="compact-table category-table">
|
||||
<tr><th>Название</th><th>Тэг</th><th>Порядок</th><th>Статус</th><th></th></tr>
|
||||
{% for c in categories %}
|
||||
<tr>
|
||||
<td>
|
||||
<form id="category-update-{{ c.id }}" method="post" action="/categories/{{ c.id }}/update">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<input name="name" value="{{ c.name }}">
|
||||
</form>
|
||||
</td>
|
||||
<td><input form="category-update-{{ c.id }}" name="tag" value="{{ c.tag }}"></td>
|
||||
<td><input form="category-update-{{ c.id }}" name="sort_order" type="number" value="{{ c.sort_order }}"></td>
|
||||
<td>{% if c.is_active %}<span class="pill ok">активна</span>{% else %}<span class="pill">выключена</span>{% endif %}</td>
|
||||
<td class="icon-actions">
|
||||
<button class="icon-btn" form="category-update-{{ c.id }}" type="submit" title="Сохранить">✓</button>
|
||||
<form method="post" action="/categories/{{ c.id }}/toggle">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<button class="icon-btn" type="submit" title="{% if c.is_active %}Выключить{% else %}Включить{% endif %}">{% if c.is_active %}⏸{% else %}▶{% endif %}</button>
|
||||
</form>
|
||||
<form method="post" action="/categories/{{ c.id }}/delete">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<button class="icon-btn danger" type="submit" title="Удалить из активных">×</button>
|
||||
</form>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
</details>
|
||||
|
||||
<h2>Настройки</h2>
|
||||
{% for category, rows in settings|groupby("category") %}
|
||||
<details class="panel" {% if category == "AI Qualifier" %}open{% endif %}>
|
||||
<summary><strong>{{ category_titles.get(category, category) }}</strong></summary>
|
||||
<table>
|
||||
<tr><th>Ключ</th><th>Значение</th><th>Описание</th></tr>
|
||||
{% for s in rows %}
|
||||
<tr>
|
||||
<td><div class="mono">{{ s.key }}</div><div class="muted">{{ s.title }}</div></td>
|
||||
<td>
|
||||
<form class="row" method="post" action="/settings/save">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<input type="hidden" name="key" value="{{ s.key }}">
|
||||
{% if s.value_type == "bool" %}
|
||||
<select name="value" style="width:120px;">
|
||||
<option value="true" {% if s.value_json == true %}selected{% endif %}>true</option>
|
||||
<option value="false" {% if s.value_json == false %}selected{% endif %}>false</option>
|
||||
</select>
|
||||
{% elif s.key in ["ai_qualifier_provider", "ai_writer_provider"] %}
|
||||
<select name="value" style="width:260px;">
|
||||
{% for p in provider_options %}
|
||||
<option value="{{ p.value }}" {% if s.value_json == p.value %}selected{% endif %}>{{ p.label }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
{% elif s.key == "ai_qualifier_model" %}
|
||||
<select name="value" style="width:420px; max-width:100%;">
|
||||
{% for m in ai_model_options %}
|
||||
<option value="{{ m.value }}" {% if s.value_json == m.value %}selected{% endif %}>{{ m.label }}</option>
|
||||
{% endfor %}
|
||||
{% if s.value_json and s.value_json not in ai_model_options|map(attribute="value")|list %}
|
||||
<option value="{{ s.value_json }}">{{ s.value_json }} · текущее значение</option>
|
||||
{% endif %}
|
||||
</select>
|
||||
{% elif s.key == "ai_writer_model" %}
|
||||
<select name="value" style="width:420px; max-width:100%;">
|
||||
{% for m in writer_model_options %}
|
||||
<option value="{{ m.value }}" {% if s.value_json == m.value %}selected{% endif %}>{{ m.label }}</option>
|
||||
{% endfor %}
|
||||
{% if s.value_json and s.value_json not in writer_model_options|map(attribute="value")|list %}
|
||||
<option value="{{ s.value_json }}">{{ s.value_json }} · текущее значение</option>
|
||||
{% endif %}
|
||||
</select>
|
||||
{% elif s.value_type == "secret" %}
|
||||
<div class="secret-editor">
|
||||
<input class="secret-visible" name="value" value="{{ s.value_json }}" autocomplete="off" spellcheck="false" style="width:420px; max-width:100%;">
|
||||
<div class="muted small-help">Ключ показан полностью для удобства. Не открывай эту страницу в демонстрации экрана.</div>
|
||||
</div>
|
||||
{% elif s.value_type == "text" %}
|
||||
<div class="prompt-editor">
|
||||
{% if prompt_hints and s.key in prompt_hints %}
|
||||
<div class="prompt-hint">
|
||||
<strong>{{ prompt_hints[s.key].title }}</strong>
|
||||
<div class="muted">{{ prompt_hints[s.key].body }}</div>
|
||||
<div class="hint-grid">
|
||||
<div>
|
||||
<div class="hint-label">Как безопасно менять</div>
|
||||
<div>{{ prompt_hints[s.key].safe }}</div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="hint-label">Что приходит на вход</div>
|
||||
<div>{{ prompt_hints[s.key].input }}</div>
|
||||
</div>
|
||||
</div>
|
||||
<details class="nested-details">
|
||||
<summary>Базовая форма/пример</summary>
|
||||
<pre>{{ prompt_hints[s.key].contract }}</pre>
|
||||
</details>
|
||||
</div>
|
||||
{% endif %}
|
||||
<textarea name="value" rows="12" style="width:720px; max-width:100%;">{{ s.value_json }}</textarea>
|
||||
</div>
|
||||
{% else %}
|
||||
<input name="value" value="{{ s.value_json }}" style="width:180px;">
|
||||
{% endif %}
|
||||
<button class="btn" type="submit">OK</button>
|
||||
</form>
|
||||
</td>
|
||||
<td>{{ s.description }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
</details>
|
||||
{% endfor %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,28 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def normalize_hash_tag(value: str, fallback: str = "source") -> str:
|
||||
tag = (value or "").strip().lstrip("#").lower()
|
||||
tag = re.sub(r"\s+", "_", tag)
|
||||
tag = re.sub(r"[^\wа-яё_]+", "_", tag, flags=re.IGNORECASE)
|
||||
tag = re.sub(r"_+", "_", tag).strip("_")
|
||||
return tag or fallback
|
||||
|
||||
|
||||
def parse_categories(value: object) -> list[str]:
|
||||
if isinstance(value, list):
|
||||
raw_items = [str(item) for item in value]
|
||||
else:
|
||||
text = str(value or "")
|
||||
raw_items = re.split(r"[\n,|]+", text)
|
||||
categories: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for item in raw_items:
|
||||
category = item.strip().strip("#")
|
||||
key = category.lower()
|
||||
if category and key not in seen:
|
||||
categories.append(category)
|
||||
seen.add(key)
|
||||
return categories
|
||||
@@ -0,0 +1,290 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
from loguru import logger
|
||||
|
||||
from .config import settings
|
||||
|
||||
|
||||
class VKAPIError(RuntimeError):
|
||||
def __init__(self, code: int | None, message: str) -> None:
|
||||
self.code = code
|
||||
super().__init__(f"VK API error {code}: {message}")
|
||||
|
||||
|
||||
class VKRateLimiter:
|
||||
def __init__(self, rps: int = 3) -> None:
|
||||
self.rps = max(1, int(rps))
|
||||
self.interval = 1.0 / self.rps
|
||||
self._last = 0.0
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def acquire(self) -> None:
|
||||
async with self._lock:
|
||||
now = time.monotonic()
|
||||
wait = self.interval - (now - self._last)
|
||||
if wait > 0:
|
||||
await asyncio.sleep(wait)
|
||||
self._last = time.monotonic()
|
||||
|
||||
|
||||
class VKAPIClient:
|
||||
base_url = "https://api.vk.com/method"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
token: str | None = None,
|
||||
version: str | None = None,
|
||||
rps: int = 3,
|
||||
timeout_total_sec: int = 60,
|
||||
timeout_connect_sec: int = 10,
|
||||
rate_limit_sleep_sec: float = 1.0,
|
||||
retry_attempts: int = 3,
|
||||
retry_min_delay_sec: float = 2.0,
|
||||
retry_max_delay_sec: float = 10.0,
|
||||
) -> None:
|
||||
self.token = token or settings.vk_access_token
|
||||
self.version = version or settings.vk_api_version
|
||||
self.limiter = VKRateLimiter(rps)
|
||||
self.timeout_total_sec = max(1, int(timeout_total_sec))
|
||||
self.timeout_connect_sec = max(1, int(timeout_connect_sec))
|
||||
self.rate_limit_sleep_sec = max(0.1, float(rate_limit_sleep_sec))
|
||||
self.retry_attempts = max(1, int(retry_attempts))
|
||||
self.retry_min_delay_sec = max(0.1, float(retry_min_delay_sec))
|
||||
self.retry_max_delay_sec = max(self.retry_min_delay_sec, float(retry_max_delay_sec))
|
||||
self.session: aiohttp.ClientSession | None = None
|
||||
|
||||
async def __aenter__(self) -> "VKAPIClient":
|
||||
self.session = aiohttp.ClientSession(
|
||||
timeout=aiohttp.ClientTimeout(total=self.timeout_total_sec, connect=self.timeout_connect_sec)
|
||||
)
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args) -> None:
|
||||
if self.session:
|
||||
await self.session.close()
|
||||
|
||||
async def call(self, method: str, **params: Any) -> dict | list:
|
||||
if not self.session:
|
||||
raise RuntimeError("VKAPIClient is not initialized")
|
||||
if not self.token:
|
||||
raise RuntimeError("VK_ACCESS_TOKEN is empty")
|
||||
|
||||
payload = dict(params)
|
||||
payload["access_token"] = self.token
|
||||
payload["v"] = self.version
|
||||
|
||||
for attempt in range(1, self.retry_attempts + 1):
|
||||
try:
|
||||
await self.limiter.acquire()
|
||||
async with self.session.post(f"{self.base_url}/{method}", data=payload) as resp:
|
||||
resp.raise_for_status()
|
||||
data = await resp.json(content_type=None)
|
||||
if "error" not in data:
|
||||
return data.get("response", {})
|
||||
|
||||
err = data["error"]
|
||||
code = err.get("error_code")
|
||||
message = err.get("error_msg", "unknown")
|
||||
if code == 6 and attempt < self.retry_attempts:
|
||||
logger.warning("VK rate limit hit, sleeping {}s", self.rate_limit_sleep_sec)
|
||||
await asyncio.sleep(self.rate_limit_sleep_sec)
|
||||
continue
|
||||
raise VKAPIError(code, message)
|
||||
except VKAPIError:
|
||||
raise
|
||||
except Exception:
|
||||
if attempt >= self.retry_attempts:
|
||||
raise
|
||||
delay = min(self.retry_min_delay_sec * (2 ** (attempt - 1)), self.retry_max_delay_sec)
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
raise VKAPIError(None, "retry exhausted")
|
||||
|
||||
async def resolve_group(self, input_value: str) -> tuple[str, int, str]:
|
||||
external_id = normalize_vk_source(input_value)
|
||||
if external_id.lstrip("-").isdigit():
|
||||
group_id = abs(int(external_id))
|
||||
response = await self.call("groups.getById", group_id=str(group_id))
|
||||
else:
|
||||
response = await self.call("groups.getById", group_id=external_id)
|
||||
items = response if isinstance(response, list) else response.get("groups", [])
|
||||
if not items:
|
||||
raise VKAPIError(None, f"cannot resolve VK group: {input_value}")
|
||||
group = items[0]
|
||||
group_id = int(group["id"])
|
||||
screen_name = str(group.get("screen_name") or external_id)
|
||||
name = str(group.get("name") or screen_name)
|
||||
return screen_name, -group_id, name
|
||||
|
||||
async def get_wall_posts(self, owner_id: int, count: int, offset: int = 0) -> dict:
|
||||
response = await self.call("wall.get", owner_id=owner_id, count=count, offset=offset, filter="owner")
|
||||
if not isinstance(response, dict):
|
||||
raise VKAPIError(None, "wall.get returned non-object response")
|
||||
return response
|
||||
|
||||
async def get_wall_upload_server(self, group_id: int) -> str:
|
||||
response = await self.call("photos.getWallUploadServer", group_id=abs(int(group_id)))
|
||||
if not isinstance(response, dict) or not response.get("upload_url"):
|
||||
raise VKAPIError(None, "photos.getWallUploadServer returned no upload_url")
|
||||
return str(response["upload_url"])
|
||||
|
||||
async def upload_wall_photo_bytes(self, group_id: int, data: bytes, filename: str = "photo.jpg") -> dict:
|
||||
if not self.session:
|
||||
raise RuntimeError("VKAPIClient is not initialized")
|
||||
upload_url = await self.get_wall_upload_server(group_id)
|
||||
form = aiohttp.FormData()
|
||||
form.add_field("photo", data, filename=filename, content_type="image/jpeg")
|
||||
async with self.session.post(upload_url, data=form) as resp:
|
||||
uploaded = await resp.json(content_type=None)
|
||||
saved = await self.call(
|
||||
"photos.saveWallPhoto",
|
||||
group_id=abs(int(group_id)),
|
||||
photo=uploaded.get("photo"),
|
||||
server=uploaded.get("server"),
|
||||
hash=uploaded.get("hash"),
|
||||
)
|
||||
if not isinstance(saved, list) or not saved:
|
||||
raise VKAPIError(None, f"photos.saveWallPhoto returned invalid response: {json.dumps(saved)[:300]}")
|
||||
return saved[0]
|
||||
|
||||
async def create_wall_post(
|
||||
self,
|
||||
owner_id: int,
|
||||
message: str,
|
||||
attachments: list[str],
|
||||
from_group: bool = True,
|
||||
) -> int:
|
||||
response = await self.call(
|
||||
"wall.post",
|
||||
owner_id=int(owner_id),
|
||||
from_group=1 if from_group else 0,
|
||||
message=message or "",
|
||||
attachments=",".join(attachments) if attachments else "",
|
||||
)
|
||||
if not isinstance(response, dict) or "post_id" not in response:
|
||||
raise VKAPIError(None, f"wall.post returned invalid response: {response}")
|
||||
return int(response["post_id"])
|
||||
|
||||
|
||||
def normalize_vk_source(value: str) -> str:
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
return ""
|
||||
if "vk.com" in raw:
|
||||
raw = raw.split("vk.com", 1)[1]
|
||||
raw = raw.lstrip("/")
|
||||
raw = raw.split("?", 1)[0].split("#", 1)[0].strip()
|
||||
raw = re.sub(r"^(club|public)", "", raw, flags=re.IGNORECASE)
|
||||
return raw.lower()
|
||||
|
||||
|
||||
def is_repost(post: dict) -> bool:
|
||||
return bool(post.get("copy_history"))
|
||||
|
||||
|
||||
def is_deleted_or_invalid(post: dict) -> bool:
|
||||
if post.get("is_deleted") or post.get("deleted"):
|
||||
return True
|
||||
if not post.get("id") or not post.get("date"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_fatal_source_error(error: Exception | str) -> bool:
|
||||
message = str(error or "").lower()
|
||||
fatal_patterns = (
|
||||
"vk api error 15:",
|
||||
"vk api error 18:",
|
||||
"vk api error 19:",
|
||||
"vk api error 100:",
|
||||
"vk api error 113:",
|
||||
"vk api error 1051:",
|
||||
"vk api error 200:",
|
||||
"vk api error 201:",
|
||||
"vk api error 203:",
|
||||
"[15]",
|
||||
"[18]",
|
||||
"[19]",
|
||||
"[100]",
|
||||
"[113]",
|
||||
"[1051]",
|
||||
"[200]",
|
||||
"[201]",
|
||||
"[203]",
|
||||
"access denied",
|
||||
"private profile",
|
||||
"cannot resolve vk group",
|
||||
"cannot resolve screen_name",
|
||||
"method is unavailable with current profile type",
|
||||
"wall is disabled",
|
||||
)
|
||||
return any(pattern in message for pattern in fatal_patterns)
|
||||
|
||||
|
||||
def post_vk_url(owner_id: int, post_id: int | str) -> str:
|
||||
return f"https://vk.com/wall{int(owner_id)}_{post_id}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExtractedMedia:
|
||||
media_type: str
|
||||
original_url: str | None
|
||||
attachment_id: str
|
||||
width: int | None = None
|
||||
height: int | None = None
|
||||
duration_sec: int | None = None
|
||||
sort_order: int = 0
|
||||
|
||||
|
||||
def extract_media(post: dict) -> list[ExtractedMedia]:
|
||||
out: list[ExtractedMedia] = []
|
||||
for idx, att in enumerate(post.get("attachments", []) or []):
|
||||
att_type = att.get("type")
|
||||
if att_type == "photo":
|
||||
photo = att.get("photo") or {}
|
||||
sizes = sorted(
|
||||
photo.get("sizes", []) or [],
|
||||
key=lambda s: int(s.get("width") or 0) * int(s.get("height") or 0),
|
||||
reverse=True,
|
||||
)
|
||||
best = sizes[0] if sizes else {}
|
||||
owner_id = photo.get("owner_id")
|
||||
media_id = photo.get("id")
|
||||
if owner_id is None or media_id is None:
|
||||
continue
|
||||
out.append(
|
||||
ExtractedMedia(
|
||||
media_type="photo",
|
||||
original_url=best.get("url"),
|
||||
attachment_id=f"photo{owner_id}_{media_id}",
|
||||
width=best.get("width"),
|
||||
height=best.get("height"),
|
||||
sort_order=idx,
|
||||
)
|
||||
)
|
||||
elif att_type == "video":
|
||||
video = att.get("video") or {}
|
||||
owner_id = video.get("owner_id")
|
||||
media_id = video.get("id")
|
||||
if owner_id is None or media_id is None:
|
||||
continue
|
||||
out.append(
|
||||
ExtractedMedia(
|
||||
media_type="video",
|
||||
original_url=f"https://vk.com/video{owner_id}_{media_id}",
|
||||
attachment_id=f"video{owner_id}_{media_id}",
|
||||
width=video.get("width"),
|
||||
height=video.get("height"),
|
||||
duration_sec=video.get("duration"),
|
||||
sort_order=idx,
|
||||
)
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,409 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
import litellm
|
||||
from loguru import logger
|
||||
|
||||
from ..config import settings
|
||||
from ..constants import WORKER_AI_QUALIFIER
|
||||
from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool
|
||||
from ..heartbeat import HeartbeatReporter
|
||||
from ..jobs import is_worker_enabled
|
||||
|
||||
|
||||
def now_utc() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def prompt_hash(prompt: str) -> str:
|
||||
return hashlib.sha256((prompt or "").encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def normalize_prompt(prompt: str) -> str:
|
||||
return (prompt or "").replace("\\r\\n", "\n").replace("\\n", "\n").strip()
|
||||
|
||||
|
||||
DEFAULT_CONTRACT_PROMPT = """
|
||||
OUTPUT SCHEMA (return array matching input order):
|
||||
{"results":[{"id":123,"score":8,"decision":"accepted","reason":"до 10 слов на русском","reject_tag":null}]}
|
||||
|
||||
Rules:
|
||||
- Input is a JSON array of posts.
|
||||
- Return JSON only. No markdown. No text outside JSON.
|
||||
- score must be integer 1..10.
|
||||
- decision must be exactly one of: "accepted", "rejected", "maybe".
|
||||
- reject_tag (rejected/maybe only) must be one of: "meme", "no_product", "politics", "discount_only", "off_topic", "vacancy", "low_content", "wrong_language", "injection", "weapon", null.
|
||||
- Return one result for every input post id.
|
||||
"""
|
||||
|
||||
|
||||
def build_system_prompt(user_prompt: str, contract_prompt: str) -> str:
|
||||
parts = [normalize_prompt(user_prompt), normalize_prompt(contract_prompt or DEFAULT_CONTRACT_PROMPT)]
|
||||
return "\n\n".join(part for part in parts if part).strip()
|
||||
|
||||
|
||||
def normalize_model(provider: str, model: str) -> str:
|
||||
provider = (provider or "").strip().lower()
|
||||
model = (model or "").strip()
|
||||
if not model:
|
||||
return model
|
||||
if "/" in model:
|
||||
return model
|
||||
if provider in {"openrouter", "anthropic", "gemini", "vertex_ai", "bedrock"}:
|
||||
return f"{provider}/{model}"
|
||||
return model
|
||||
|
||||
|
||||
def parse_ai_json(content: str) -> dict:
|
||||
raw = (content or "").strip()
|
||||
if raw.startswith("```"):
|
||||
raw = raw.strip("`")
|
||||
if raw.lower().startswith("json"):
|
||||
raw = raw[4:].strip()
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
def response_usage(response: Any) -> dict[str, Any]:
|
||||
usage = getattr(response, "usage", None)
|
||||
if usage is None and isinstance(response, dict):
|
||||
usage = response.get("usage")
|
||||
|
||||
def get(name: str) -> int | None:
|
||||
if usage is None:
|
||||
return None
|
||||
value = getattr(usage, name, None)
|
||||
if value is None and isinstance(usage, dict):
|
||||
value = usage.get(name)
|
||||
try:
|
||||
return int(value) if value is not None else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
cost = getattr(response, "_hidden_params", None)
|
||||
if isinstance(cost, dict):
|
||||
cost = cost.get("response_cost")
|
||||
else:
|
||||
cost = None
|
||||
try:
|
||||
cost_value = Decimal(str(cost)) if cost is not None else None
|
||||
except Exception:
|
||||
cost_value = None
|
||||
|
||||
return {
|
||||
"prompt_tokens": get("prompt_tokens"),
|
||||
"completion_tokens": get("completion_tokens"),
|
||||
"total_tokens": get("total_tokens"),
|
||||
"estimated_cost_usd": cost_value,
|
||||
}
|
||||
|
||||
|
||||
def validate_results(data: dict, expected_ids: set[int], min_score: int) -> list[dict]:
|
||||
results = data.get("results")
|
||||
if not isinstance(results, list) and isinstance(data.get("data"), dict):
|
||||
results = data["data"].get("results")
|
||||
if not isinstance(results, list):
|
||||
raise ValueError("AI response has no results list")
|
||||
|
||||
out: list[dict] = []
|
||||
seen: set[int] = set()
|
||||
for item in results:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
post_id = int(item.get("id"))
|
||||
if post_id not in expected_ids:
|
||||
raise ValueError(f"AI returned unexpected id={post_id}")
|
||||
score = max(1, min(10, int(item.get("score"))))
|
||||
decision = str(item.get("decision") or "").strip().lower()
|
||||
if decision not in {"accepted", "rejected", "maybe"}:
|
||||
decision = "accepted" if score >= min_score else "rejected"
|
||||
reason = str(item.get("reason") or "").strip()[:1000]
|
||||
reject_tag = item.get("reject_tag")
|
||||
reject_tag = str(reject_tag).strip().lower()[:80] if reject_tag not in {None, ""} else None
|
||||
seen.add(post_id)
|
||||
out.append({"id": post_id, "score": score, "decision": decision, "reason": reason, "reject_tag": reject_tag})
|
||||
|
||||
missing = expected_ids - seen
|
||||
if missing:
|
||||
raise ValueError(f"AI response missing ids: {sorted(missing)[:10]}")
|
||||
return out
|
||||
|
||||
|
||||
class AIQualifierWorker:
|
||||
def __init__(self) -> None:
|
||||
self.pool = None
|
||||
self.heartbeat = HeartbeatReporter(WORKER_AI_QUALIFIER, 30)
|
||||
|
||||
async def init(self) -> None:
|
||||
self.pool = await get_pool()
|
||||
|
||||
async def claim_posts(self, batch_size: int) -> list[dict]:
|
||||
async with self.pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
count = await conn.fetchval(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM raw_posts
|
||||
WHERE status='storage_ready'
|
||||
AND COALESCE(qualification_status, 'pending') IN ('pending', 'failed')
|
||||
"""
|
||||
)
|
||||
if int(count or 0) < batch_size:
|
||||
return []
|
||||
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
WITH cte AS (
|
||||
SELECT id
|
||||
FROM raw_posts
|
||||
WHERE status='storage_ready'
|
||||
AND COALESCE(qualification_status, 'pending') IN ('pending', 'failed')
|
||||
ORDER BY created_at ASC, id ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT $1
|
||||
)
|
||||
UPDATE raw_posts rp
|
||||
SET qualification_status='processing',
|
||||
updated_at=NOW()
|
||||
FROM cte
|
||||
WHERE rp.id=cte.id
|
||||
RETURNING rp.id, rp.raw_text, rp.original_url, rp.storage_post_url, rp.created_at,
|
||||
rp.source_id,
|
||||
(SELECT s.name FROM sources s WHERE s.id=rp.source_id) AS source_name,
|
||||
(SELECT COUNT(*) FROM raw_post_media m WHERE m.raw_post_id=rp.id) AS media_count,
|
||||
(SELECT ARRAY_AGG(DISTINCT m.media_type ORDER BY m.media_type)
|
||||
FROM raw_post_media m WHERE m.raw_post_id=rp.id) AS media_types
|
||||
""",
|
||||
batch_size,
|
||||
)
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
async def create_batch(self, provider: str, model: str, prompt: str, payload: dict) -> int:
|
||||
return int(
|
||||
await self.pool.fetchval(
|
||||
"""
|
||||
INSERT INTO ai_qualification_batches(provider, model, prompt_hash, prompt_text, posts_count, request_json)
|
||||
VALUES($1, $2, $3, $4, $5, $6::jsonb)
|
||||
RETURNING id
|
||||
""",
|
||||
provider,
|
||||
model,
|
||||
prompt_hash(prompt),
|
||||
prompt,
|
||||
len(payload),
|
||||
json.dumps(payload, ensure_ascii=False),
|
||||
)
|
||||
)
|
||||
|
||||
async def finish_batch(
|
||||
self,
|
||||
batch_id: int,
|
||||
status: str,
|
||||
response: Any = None,
|
||||
error: str | None = None,
|
||||
usage: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
counts = {"accepted": 0, "rejected": 0, "maybe": 0}
|
||||
if isinstance(response, dict) and isinstance(response.get("results"), list):
|
||||
for item in response["results"]:
|
||||
decision = str(item.get("decision") or "")
|
||||
if decision in counts:
|
||||
counts[decision] += 1
|
||||
usage = usage or {}
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE ai_qualification_batches
|
||||
SET status=$2,
|
||||
response_json=$3::jsonb,
|
||||
error=$4,
|
||||
accepted_count=$5,
|
||||
rejected_count=$6,
|
||||
maybe_count=$7,
|
||||
prompt_tokens=$8,
|
||||
completion_tokens=$9,
|
||||
total_tokens=$10,
|
||||
estimated_cost_usd=$11,
|
||||
completed_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
batch_id,
|
||||
status,
|
||||
json.dumps(response, ensure_ascii=False) if response is not None else None,
|
||||
error[:2000] if error else None,
|
||||
counts["accepted"],
|
||||
counts["rejected"],
|
||||
counts["maybe"],
|
||||
usage.get("prompt_tokens"),
|
||||
usage.get("completion_tokens"),
|
||||
usage.get("total_tokens"),
|
||||
usage.get("estimated_cost_usd"),
|
||||
)
|
||||
|
||||
async def mark_posts_failed(self, post_ids: list[int], error: str) -> None:
|
||||
if not post_ids:
|
||||
return
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE raw_posts
|
||||
SET qualification_status='failed',
|
||||
qualification_reason=$2,
|
||||
updated_at=NOW()
|
||||
WHERE id=ANY($1::bigint[])
|
||||
""",
|
||||
post_ids,
|
||||
error[:1000],
|
||||
)
|
||||
|
||||
async def apply_results(self, batch_id: int, results: list[dict], model: str, prompt: str, min_score: int) -> None:
|
||||
for item in results:
|
||||
score = int(item["score"])
|
||||
decision = str(item["decision"])
|
||||
model_decision = decision
|
||||
if score >= min_score and decision == "maybe":
|
||||
decision = "accepted"
|
||||
if score < min_score and decision == "accepted":
|
||||
decision = "rejected"
|
||||
status = "accepted" if score >= min_score and decision == "accepted" else "rejected"
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE raw_posts
|
||||
SET qualification_status=$2,
|
||||
qualification_score=$3,
|
||||
qualification_decision=$4,
|
||||
qualification_reason=$5,
|
||||
qualification_model=$6,
|
||||
qualification_prompt_hash=$7,
|
||||
qualification_batch_id=$8,
|
||||
qualification_reject_tag=$9,
|
||||
qualification_model_decision=$10,
|
||||
qualified_at=NOW(),
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
int(item["id"]),
|
||||
status,
|
||||
score,
|
||||
decision,
|
||||
item["reason"],
|
||||
model,
|
||||
prompt_hash(prompt),
|
||||
batch_id,
|
||||
item.get("reject_tag"),
|
||||
model_decision,
|
||||
)
|
||||
|
||||
async def call_ai(
|
||||
self,
|
||||
provider: str,
|
||||
model: str,
|
||||
api_key: str,
|
||||
api_base: str,
|
||||
prompt: str,
|
||||
payload: dict,
|
||||
temperature: float,
|
||||
timeout: int,
|
||||
) -> tuple[dict, dict[str, Any]]:
|
||||
messages = [
|
||||
{"role": "system", "content": prompt},
|
||||
{"role": "user", "content": json.dumps(payload, ensure_ascii=False)},
|
||||
]
|
||||
kwargs: dict[str, Any] = {
|
||||
"model": normalize_model(provider, model),
|
||||
"messages": messages,
|
||||
"temperature": temperature,
|
||||
"timeout": timeout,
|
||||
"response_format": {"type": "json_object"},
|
||||
}
|
||||
if api_key:
|
||||
kwargs["api_key"] = api_key
|
||||
if api_base:
|
||||
kwargs["api_base"] = api_base
|
||||
response = await asyncio.to_thread(litellm.completion, **kwargs)
|
||||
content = response.choices[0].message.content
|
||||
return parse_ai_json(content), response_usage(response)
|
||||
|
||||
async def run_once(self) -> bool:
|
||||
enabled = await is_worker_enabled(self.pool, WORKER_AI_QUALIFIER)
|
||||
setting_enabled = await fetch_bool_setting("ai_qualifier_enabled", False)
|
||||
if not enabled or not setting_enabled:
|
||||
await self.heartbeat.beat(self.pool, status="disabled", force=True)
|
||||
return False
|
||||
|
||||
provider = str(await fetch_setting("ai_qualifier_provider", "openrouter") or "openrouter")
|
||||
model = str(await fetch_setting("ai_qualifier_model", "") or "").strip()
|
||||
api_key = str(await fetch_setting("ai_qualifier_api_key", "") or "").strip()
|
||||
api_base = str(await fetch_setting("ai_qualifier_api_base", "") or "").strip()
|
||||
batch_size = max(1, await fetch_int_setting("ai_qualifier_batch_size", 30))
|
||||
min_score = max(1, min(10, await fetch_int_setting("ai_qualifier_min_score", 7)))
|
||||
max_text_chars = max(100, await fetch_int_setting("ai_qualifier_max_text_chars", 2000))
|
||||
temperature = max(0.0, await fetch_float_setting("ai_qualifier_temperature", 0.0))
|
||||
timeout = max(10, await fetch_int_setting("ai_qualifier_timeout_sec", 120))
|
||||
prompt = build_system_prompt(
|
||||
str(await fetch_setting("ai_qualifier_prompt", "") or ""),
|
||||
str(await fetch_setting("ai_qualifier_contract", DEFAULT_CONTRACT_PROMPT) or DEFAULT_CONTRACT_PROMPT),
|
||||
)
|
||||
|
||||
if not model or not api_key or not prompt:
|
||||
await self.heartbeat.beat(self.pool, status="not_configured", force=True)
|
||||
return False
|
||||
|
||||
posts = await self.claim_posts(batch_size)
|
||||
await self.heartbeat.beat(self.pool, meta={"claimed": len(posts), "batch_size": batch_size})
|
||||
if not posts:
|
||||
return False
|
||||
|
||||
payload = [
|
||||
{
|
||||
"id": int(p["id"]),
|
||||
"source": p.get("source_name") or "",
|
||||
"original_url": p.get("original_url") or "",
|
||||
"media_count": int(p.get("media_count") or 0),
|
||||
"media_types": list(p.get("media_types") or []),
|
||||
"text": str(p.get("raw_text") or "")[:max_text_chars],
|
||||
}
|
||||
for p in posts
|
||||
]
|
||||
post_ids = [int(p["id"]) for p in posts]
|
||||
batch_id = await self.create_batch(provider, model, prompt, payload)
|
||||
try:
|
||||
response, usage = await self.call_ai(provider, model, api_key, api_base, prompt, payload, temperature, timeout)
|
||||
results = validate_results(response, set(post_ids), min_score)
|
||||
await self.apply_results(batch_id, results, normalize_model(provider, model), prompt, min_score)
|
||||
await self.finish_batch(batch_id, "done", {"results": results}, usage=usage)
|
||||
logger.info("AI qualifier batch done: id={} posts={} usage={}", batch_id, len(results), usage)
|
||||
return True
|
||||
except Exception as exc:
|
||||
await self.mark_posts_failed(post_ids, str(exc))
|
||||
await self.finish_batch(batch_id, "failed", error=str(exc))
|
||||
logger.exception("AI qualifier batch failed: id={} error={}", batch_id, exc)
|
||||
return True
|
||||
|
||||
async def run_loop(self) -> None:
|
||||
await self.init()
|
||||
worker_id = f"{WORKER_AI_QUALIFIER}"
|
||||
logger.info("{} started", worker_id)
|
||||
while True:
|
||||
try:
|
||||
had_work = await self.run_once()
|
||||
except Exception as exc:
|
||||
logger.exception("AI qualifier loop error: {}", exc)
|
||||
had_work = False
|
||||
interval = max(10, await fetch_int_setting("ai_qualifier_interval_sec", 60))
|
||||
if not had_work:
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
logger.remove()
|
||||
logger.add(lambda msg: print(msg, end=""), level=settings.log_level)
|
||||
worker = AIQualifierWorker()
|
||||
await worker.run_loop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,456 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
import litellm
|
||||
from loguru import logger
|
||||
|
||||
from ..config import settings
|
||||
from ..constants import WORKER_AI_WRITER
|
||||
from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool
|
||||
from ..heartbeat import HeartbeatReporter
|
||||
from ..jobs import is_worker_enabled
|
||||
from ..text_utils import normalize_hash_tag, parse_categories
|
||||
|
||||
|
||||
def now_utc() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def prompt_hash(prompt: str) -> str:
|
||||
return hashlib.sha256((prompt or "").encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def normalize_prompt(prompt: str) -> str:
|
||||
return (prompt or "").replace("\\r\\n", "\n").replace("\\n", "\n").strip()
|
||||
|
||||
|
||||
DEFAULT_CONTRACT_PROMPT = """
|
||||
OUTPUT SCHEMA (return array matching input order):
|
||||
{"rewrites":[{"id":123,"category":"разгрузка","text":"готовый текст без хэштегов","notes":"короткая заметка для редактора или null"}]}
|
||||
|
||||
Rules:
|
||||
- Input is a JSON object with key "posts" containing accepted posts.
|
||||
- Each post includes producer_name and producer_tag. Use producer_name when it helps, but do not invent facts.
|
||||
- Do not add hashtags to rewritten text. Pick exactly one category from the categories list in input.
|
||||
- Return JSON only. No markdown. No text outside JSON.
|
||||
- Return one rewrite for every input post id.
|
||||
"""
|
||||
|
||||
|
||||
def build_system_prompt(user_prompt: str, contract_prompt: str) -> str:
|
||||
parts = [normalize_prompt(user_prompt), normalize_prompt(contract_prompt or DEFAULT_CONTRACT_PROMPT)]
|
||||
return "\n\n".join(part for part in parts if part).strip()
|
||||
|
||||
|
||||
def normalize_model(provider: str, model: str) -> str:
|
||||
provider = (provider or "").strip().lower()
|
||||
model = (model or "").strip()
|
||||
if not model:
|
||||
return model
|
||||
if "/" in model:
|
||||
return model
|
||||
if provider in {"openrouter", "anthropic", "gemini", "vertex_ai", "bedrock"}:
|
||||
return f"{provider}/{model}"
|
||||
return model
|
||||
|
||||
|
||||
def parse_ai_json(content: str) -> dict:
|
||||
raw = (content or "").strip()
|
||||
if raw.startswith("```"):
|
||||
raw = raw.strip("`")
|
||||
if raw.lower().startswith("json"):
|
||||
raw = raw[4:].strip()
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
def response_usage(response: Any) -> dict[str, Any]:
|
||||
usage = getattr(response, "usage", None)
|
||||
if usage is None and isinstance(response, dict):
|
||||
usage = response.get("usage")
|
||||
|
||||
def get(name: str) -> int | None:
|
||||
if usage is None:
|
||||
return None
|
||||
value = getattr(usage, name, None)
|
||||
if value is None and isinstance(usage, dict):
|
||||
value = usage.get(name)
|
||||
try:
|
||||
return int(value) if value is not None else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
cost = getattr(response, "_hidden_params", None)
|
||||
if isinstance(cost, dict):
|
||||
cost = cost.get("response_cost")
|
||||
else:
|
||||
cost = None
|
||||
try:
|
||||
cost_value = Decimal(str(cost)) if cost is not None else None
|
||||
except Exception:
|
||||
cost_value = None
|
||||
|
||||
return {
|
||||
"prompt_tokens": get("prompt_tokens"),
|
||||
"completion_tokens": get("completion_tokens"),
|
||||
"total_tokens": get("total_tokens"),
|
||||
"estimated_cost_usd": cost_value,
|
||||
}
|
||||
|
||||
|
||||
def validate_rewrites(
|
||||
data: dict,
|
||||
expected_ids: set[int],
|
||||
categories: list[str],
|
||||
source_tags_by_id: dict[int, str],
|
||||
) -> list[dict]:
|
||||
rewrites = data.get("rewrites")
|
||||
if not isinstance(rewrites, list) and isinstance(data.get("data"), dict):
|
||||
rewrites = data["data"].get("rewrites")
|
||||
if not isinstance(rewrites, list):
|
||||
raise ValueError("AI response has no rewrites list")
|
||||
out: list[dict] = []
|
||||
seen: set[int] = set()
|
||||
for item in rewrites:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
post_id = int(item.get("id"))
|
||||
if post_id not in expected_ids:
|
||||
raise ValueError(f"AI returned unexpected id={post_id}")
|
||||
text = str(item.get("text") or "").strip()
|
||||
if len(text) < 40:
|
||||
raise ValueError(f"AI returned too short rewrite for id={post_id}")
|
||||
category = str(item.get("category") or "").strip().strip("#")
|
||||
category_map = {c.lower(): c for c in categories}
|
||||
if category.lower() not in category_map:
|
||||
raise ValueError(f"AI returned unknown category={category!r} for id={post_id}")
|
||||
category = category_map[category.lower()]
|
||||
notes = str(item.get("notes") or "").strip()[:1000]
|
||||
seen.add(post_id)
|
||||
out.append(
|
||||
{
|
||||
"id": post_id,
|
||||
"category": category,
|
||||
"category_tag": normalize_hash_tag(category, "category"),
|
||||
"source_tag": normalize_hash_tag(source_tags_by_id.get(post_id) or "", "source"),
|
||||
"text": text,
|
||||
"notes": notes,
|
||||
}
|
||||
)
|
||||
missing = expected_ids - seen
|
||||
if missing:
|
||||
raise ValueError(f"AI response missing ids: {sorted(missing)[:10]}")
|
||||
return out
|
||||
|
||||
|
||||
async def load_writer_categories(pool) -> list[str]:
|
||||
try:
|
||||
rows = await pool.fetch(
|
||||
"""
|
||||
SELECT name
|
||||
FROM content_categories
|
||||
WHERE is_active=TRUE
|
||||
ORDER BY sort_order, name
|
||||
"""
|
||||
)
|
||||
except Exception:
|
||||
rows = []
|
||||
categories = [str(row["name"]) for row in rows]
|
||||
if categories:
|
||||
return categories
|
||||
legacy = parse_categories(await fetch_setting("ai_writer_categories", []))
|
||||
return legacy or [
|
||||
"защита",
|
||||
"одежда",
|
||||
"разгрузка",
|
||||
"рюкзаки",
|
||||
"airsoft",
|
||||
"патчи",
|
||||
"электроника",
|
||||
"аксессуары",
|
||||
"производство",
|
||||
]
|
||||
|
||||
|
||||
class AIWriterWorker:
|
||||
def __init__(self) -> None:
|
||||
self.pool = None
|
||||
self.heartbeat = HeartbeatReporter(WORKER_AI_WRITER, 30)
|
||||
|
||||
async def init(self) -> None:
|
||||
self.pool = await get_pool()
|
||||
|
||||
async def claim_posts(self, batch_size: int) -> list[dict]:
|
||||
async with self.pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
count = await conn.fetchval(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM raw_posts
|
||||
WHERE status='storage_ready'
|
||||
AND qualification_status='accepted'
|
||||
AND COALESCE(rewrite_status, 'pending') IN ('pending', 'failed')
|
||||
"""
|
||||
)
|
||||
if int(count or 0) < batch_size:
|
||||
return []
|
||||
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
WITH cte AS (
|
||||
SELECT id
|
||||
FROM raw_posts
|
||||
WHERE status='storage_ready'
|
||||
AND qualification_status='accepted'
|
||||
AND COALESCE(rewrite_status, 'pending') IN ('pending', 'failed')
|
||||
ORDER BY qualified_at ASC NULLS LAST, id ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT $1
|
||||
)
|
||||
UPDATE raw_posts rp
|
||||
SET rewrite_status='processing',
|
||||
updated_at=NOW()
|
||||
FROM cte
|
||||
WHERE rp.id=cte.id
|
||||
RETURNING rp.id, rp.raw_text, rp.original_url, rp.created_at, rp.qualification_score,
|
||||
rp.qualification_reason,
|
||||
(SELECT s.name FROM sources s WHERE s.id=rp.source_id) AS source_name,
|
||||
(SELECT s.tag FROM sources s WHERE s.id=rp.source_id) AS source_tag,
|
||||
(SELECT COUNT(*) FROM raw_post_media m WHERE m.raw_post_id=rp.id) AS media_count,
|
||||
(SELECT ARRAY_AGG(DISTINCT m.media_type ORDER BY m.media_type)
|
||||
FROM raw_post_media m WHERE m.raw_post_id=rp.id) AS media_types
|
||||
""",
|
||||
batch_size,
|
||||
)
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
async def create_batch(self, provider: str, model: str, prompt: str, payload: dict) -> int:
|
||||
return int(
|
||||
await self.pool.fetchval(
|
||||
"""
|
||||
INSERT INTO ai_writer_batches(provider, model, prompt_hash, prompt_text, posts_count, request_json)
|
||||
VALUES($1, $2, $3, $4, $5, $6::jsonb)
|
||||
RETURNING id
|
||||
""",
|
||||
provider,
|
||||
model,
|
||||
prompt_hash(prompt),
|
||||
prompt,
|
||||
len(payload["posts"]),
|
||||
json.dumps(payload, ensure_ascii=False),
|
||||
)
|
||||
)
|
||||
|
||||
async def finish_batch(
|
||||
self,
|
||||
batch_id: int,
|
||||
status: str,
|
||||
response: Any = None,
|
||||
error: str | None = None,
|
||||
usage: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
ready_count = 0
|
||||
if isinstance(response, dict) and isinstance(response.get("rewrites"), list):
|
||||
ready_count = len(response["rewrites"])
|
||||
usage = usage or {}
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE ai_writer_batches
|
||||
SET status=$2,
|
||||
response_json=$3::jsonb,
|
||||
error=$4,
|
||||
ready_count=$5,
|
||||
failed_count=$6,
|
||||
prompt_tokens=$7,
|
||||
completion_tokens=$8,
|
||||
total_tokens=$9,
|
||||
estimated_cost_usd=$10,
|
||||
completed_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
batch_id,
|
||||
status,
|
||||
json.dumps(response, ensure_ascii=False) if response is not None else None,
|
||||
error[:2000] if error else None,
|
||||
ready_count if status == "done" else 0,
|
||||
0 if status == "done" else 1,
|
||||
usage.get("prompt_tokens"),
|
||||
usage.get("completion_tokens"),
|
||||
usage.get("total_tokens"),
|
||||
usage.get("estimated_cost_usd"),
|
||||
)
|
||||
|
||||
async def mark_posts_failed(self, post_ids: list[int], error: str) -> None:
|
||||
if not post_ids:
|
||||
return
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE raw_posts
|
||||
SET rewrite_status='failed',
|
||||
rewrite_notes=$2,
|
||||
updated_at=NOW()
|
||||
WHERE id=ANY($1::bigint[])
|
||||
""",
|
||||
post_ids,
|
||||
error[:1000],
|
||||
)
|
||||
|
||||
async def apply_rewrites(self, batch_id: int, rewrites: list[dict], model: str, prompt: str) -> None:
|
||||
for item in rewrites:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE raw_posts
|
||||
SET rewrite_status='ready',
|
||||
rewritten_text=$2,
|
||||
rewrite_notes=$3,
|
||||
rewrite_model=$4,
|
||||
rewrite_prompt_hash=$5,
|
||||
rewrite_batch_id=$6,
|
||||
rewrite_category=$7,
|
||||
rewrite_category_tag=$8,
|
||||
rewrite_source_tag=$9,
|
||||
editorial_status='review',
|
||||
final_text=$2,
|
||||
final_category=$7,
|
||||
final_category_tag=$8,
|
||||
final_source_tag=$9,
|
||||
rewritten_at=NOW(),
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
int(item["id"]),
|
||||
item["text"],
|
||||
item["notes"],
|
||||
model,
|
||||
prompt_hash(prompt),
|
||||
batch_id,
|
||||
item["category"],
|
||||
item["category_tag"],
|
||||
item["source_tag"],
|
||||
)
|
||||
|
||||
async def call_ai(
|
||||
self,
|
||||
provider: str,
|
||||
model: str,
|
||||
api_key: str,
|
||||
api_base: str,
|
||||
prompt: str,
|
||||
payload: dict,
|
||||
temperature: float,
|
||||
timeout: int,
|
||||
) -> tuple[dict, dict[str, Any]]:
|
||||
messages = [
|
||||
{"role": "system", "content": prompt},
|
||||
{"role": "user", "content": json.dumps(payload, ensure_ascii=False)},
|
||||
]
|
||||
kwargs: dict[str, Any] = {
|
||||
"model": normalize_model(provider, model),
|
||||
"messages": messages,
|
||||
"temperature": temperature,
|
||||
"timeout": timeout,
|
||||
"response_format": {"type": "json_object"},
|
||||
}
|
||||
if api_key:
|
||||
kwargs["api_key"] = api_key
|
||||
if api_base:
|
||||
kwargs["api_base"] = api_base
|
||||
response = await asyncio.to_thread(litellm.completion, **kwargs)
|
||||
content = response.choices[0].message.content
|
||||
return parse_ai_json(content), response_usage(response)
|
||||
|
||||
async def run_once(self) -> bool:
|
||||
enabled = await is_worker_enabled(self.pool, WORKER_AI_WRITER)
|
||||
setting_enabled = await fetch_bool_setting("ai_writer_enabled", False)
|
||||
if not enabled or not setting_enabled:
|
||||
await self.heartbeat.beat(self.pool, status="disabled", force=True)
|
||||
return False
|
||||
|
||||
provider = str(await fetch_setting("ai_writer_provider", "anthropic") or "anthropic")
|
||||
model = str(await fetch_setting("ai_writer_model", "") or "").strip()
|
||||
api_key = str(await fetch_setting("ai_writer_api_key", "") or "").strip()
|
||||
api_base = str(await fetch_setting("ai_writer_api_base", "") or "").strip()
|
||||
batch_size = max(1, await fetch_int_setting("ai_writer_batch_size", 1))
|
||||
max_text_chars = max(200, await fetch_int_setting("ai_writer_max_text_chars", 3500))
|
||||
temperature = max(0.0, await fetch_float_setting("ai_writer_temperature", 0.4))
|
||||
timeout = max(10, await fetch_int_setting("ai_writer_timeout_sec", 180))
|
||||
prompt = build_system_prompt(
|
||||
str(await fetch_setting("ai_writer_prompt", "") or ""),
|
||||
str(await fetch_setting("ai_writer_contract", DEFAULT_CONTRACT_PROMPT) or DEFAULT_CONTRACT_PROMPT),
|
||||
)
|
||||
categories = await load_writer_categories(self.pool)
|
||||
|
||||
if not model or not api_key or not prompt or not categories:
|
||||
await self.heartbeat.beat(self.pool, status="not_configured", force=True)
|
||||
return False
|
||||
|
||||
posts = await self.claim_posts(batch_size)
|
||||
await self.heartbeat.beat(self.pool, meta={"claimed": len(posts), "batch_size": batch_size})
|
||||
if not posts:
|
||||
return False
|
||||
|
||||
payload = {
|
||||
"task": "rewrite_accepted_posts",
|
||||
"categories": categories,
|
||||
"posts": [
|
||||
{
|
||||
"id": int(p["id"]),
|
||||
"producer_name": p.get("source_name") or "",
|
||||
"producer_tag": normalize_hash_tag(p.get("source_tag") or p.get("source_name") or "", "source"),
|
||||
"original_url": p.get("original_url") or "",
|
||||
"qualification_score": p.get("qualification_score"),
|
||||
"qualification_reason": p.get("qualification_reason") or "",
|
||||
"media_count": int(p.get("media_count") or 0),
|
||||
"media_types": list(p.get("media_types") or []),
|
||||
"text": str(p.get("raw_text") or "")[:max_text_chars],
|
||||
}
|
||||
for p in posts
|
||||
],
|
||||
}
|
||||
post_ids = [int(p["id"]) for p in posts]
|
||||
source_tags_by_id = {
|
||||
int(p["id"]): normalize_hash_tag(p.get("source_tag") or p.get("source_name") or "", "source")
|
||||
for p in posts
|
||||
}
|
||||
batch_id = await self.create_batch(provider, model, prompt, payload)
|
||||
try:
|
||||
response, usage = await self.call_ai(provider, model, api_key, api_base, prompt, payload, temperature, timeout)
|
||||
rewrites = validate_rewrites(response, set(post_ids), categories, source_tags_by_id)
|
||||
await self.apply_rewrites(batch_id, rewrites, normalize_model(provider, model), prompt)
|
||||
await self.finish_batch(batch_id, "done", {"rewrites": rewrites}, usage=usage)
|
||||
logger.info("AI writer batch done: id={} posts={} usage={}", batch_id, len(rewrites), usage)
|
||||
return True
|
||||
except Exception as exc:
|
||||
await self.mark_posts_failed(post_ids, str(exc))
|
||||
await self.finish_batch(batch_id, "failed", error=str(exc))
|
||||
logger.exception("AI writer batch failed: id={} error={}", batch_id, exc)
|
||||
return True
|
||||
|
||||
async def run_loop(self) -> None:
|
||||
await self.init()
|
||||
logger.info("{} started", WORKER_AI_WRITER)
|
||||
while True:
|
||||
try:
|
||||
had_work = await self.run_once()
|
||||
except Exception as exc:
|
||||
logger.exception("AI writer loop error: {}", exc)
|
||||
had_work = False
|
||||
interval = max(10, await fetch_int_setting("ai_writer_interval_sec", 60))
|
||||
if not had_work:
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
logger.remove()
|
||||
logger.add(lambda msg: print(msg, end=""), level=settings.log_level)
|
||||
worker = AIWriterWorker()
|
||||
await worker.run_loop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,567 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import html
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import aiohttp
|
||||
from aiogram import Bot
|
||||
from aiogram.client.session.aiohttp import AiohttpSession
|
||||
from aiogram.client.telegram import TelegramAPIServer
|
||||
from aiogram.exceptions import TelegramRetryAfter
|
||||
from aiogram.types import BufferedInputFile, FSInputFile, InputMediaPhoto, InputMediaVideo
|
||||
from loguru import logger
|
||||
|
||||
from ..config import settings
|
||||
from ..constants import (
|
||||
JOB_TYPE_VK_STORAGE_COPY,
|
||||
MEDIA_STATUS_FAILED,
|
||||
MEDIA_STATUS_LINK_ONLY,
|
||||
MEDIA_STATUS_UPLOADED,
|
||||
POST_STATUS_FAILED,
|
||||
POST_STATUS_STORAGE_READY,
|
||||
WORKER_STORAGE_UPLOADER,
|
||||
)
|
||||
from ..db import fetch_float_setting, fetch_int_setting, fetch_setting, get_pool
|
||||
from ..heartbeat import HeartbeatReporter
|
||||
from ..jobs import ack_done, ack_retry, claim_job, is_worker_enabled, recover_stale_jobs
|
||||
|
||||
TMP_DIR = Path("/tmp")
|
||||
TMP_PREFIX = "vkparser_tg_media_"
|
||||
MAX_MEDIA_GROUP = 10
|
||||
|
||||
|
||||
def parse_topic(value: str) -> tuple[int, int | None]:
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
raise RuntimeError("TG_MEDIA_CHANNEL_ID is empty")
|
||||
if ":" in raw:
|
||||
chat_id, thread_id = raw.split(":", 1)
|
||||
return int(chat_id), int(thread_id)
|
||||
return int(raw), None
|
||||
|
||||
|
||||
def parse_vk_video_url(vk_url: str) -> tuple[int, int] | None:
|
||||
match = re.search(r"video(-?\d+)_(\d+)", vk_url or "")
|
||||
if not match:
|
||||
return None
|
||||
return int(match.group(1)), int(match.group(2))
|
||||
|
||||
|
||||
def split_message_chunks(text: str, limit: int) -> list[str]:
|
||||
text = str(text or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
chunks: list[str] = []
|
||||
while len(text) > limit:
|
||||
split_at = text.rfind("\n", 0, limit)
|
||||
if split_at < limit // 2:
|
||||
split_at = text.rfind(" ", 0, limit)
|
||||
if split_at < limit // 2:
|
||||
split_at = limit
|
||||
chunks.append(text[:split_at].strip())
|
||||
text = text[split_at:].strip()
|
||||
if text:
|
||||
chunks.append(text)
|
||||
return chunks
|
||||
|
||||
|
||||
def original_link(post: dict) -> str:
|
||||
url = str(post.get("original_url") or "").strip()
|
||||
return f'<a href="{url}">#{int(post["id"])}</a>' if url else f'#{int(post["id"])}'
|
||||
|
||||
|
||||
class TelegramStorageUploader:
|
||||
def __init__(self) -> None:
|
||||
self.pool = None
|
||||
self.bot: Bot | None = None
|
||||
self.storage_chat_id: int | None = None
|
||||
self.storage_thread_id: int | None = None
|
||||
self.heartbeat = HeartbeatReporter(WORKER_STORAGE_UPLOADER, 30)
|
||||
self.media_group_max_items = MAX_MEDIA_GROUP
|
||||
self.media_upload_delay_sec = 1.0
|
||||
self.tg_retry_max_attempts = 4
|
||||
self.tg_retry_backoff_max_sec = 15
|
||||
self.caption_limit = 1024
|
||||
self.message_limit = 4096
|
||||
self.text_overflow_marker = "Продолжение следующим сообщением."
|
||||
self.download_timeout_sec = 45
|
||||
self.video_max_size_mb = 49
|
||||
self.video_max_duration_sec = 300
|
||||
self.yt_dlp_timeout_sec = 300
|
||||
self.max_media_attempts = 3
|
||||
self.post_job_pause_sec = 0.2
|
||||
|
||||
async def init(self) -> None:
|
||||
self.pool = await get_pool()
|
||||
recovered = await recover_stale_jobs(self.pool, JOB_TYPE_VK_STORAGE_COPY, stale_minutes=20)
|
||||
if recovered:
|
||||
logger.warning("Recovered stale storage jobs: {}", recovered)
|
||||
|
||||
if not settings.tg_bot_token:
|
||||
raise RuntimeError("TG_BOT_TOKEN is empty")
|
||||
media_channel = str(await fetch_setting("tg_media_channel_id", settings.tg_media_channel_id) or "").strip()
|
||||
self.storage_chat_id, self.storage_thread_id = parse_topic(media_channel)
|
||||
|
||||
local_bot_api_url = str(await fetch_setting("local_bot_api_url", settings.local_bot_api_url) or "").strip()
|
||||
if local_bot_api_url:
|
||||
session = AiohttpSession(
|
||||
api=TelegramAPIServer.from_base(local_bot_api_url.rstrip("/"), is_local=True)
|
||||
)
|
||||
self.bot = Bot(token=settings.tg_bot_token, session=session)
|
||||
logger.info("Using local Telegram Bot API: {}", local_bot_api_url)
|
||||
else:
|
||||
self.bot = Bot(token=settings.tg_bot_token)
|
||||
|
||||
self.media_group_max_items = max(1, min(MAX_MEDIA_GROUP, await fetch_int_setting("telegram_media_group_max_items", 10)))
|
||||
self.media_upload_delay_sec = max(0.0, await fetch_float_setting("telegram_media_upload_delay_sec", 1.0))
|
||||
self.tg_retry_max_attempts = max(1, await fetch_int_setting("telegram_retry_max_attempts", 4))
|
||||
self.tg_retry_backoff_max_sec = max(1, await fetch_int_setting("telegram_retry_backoff_max_sec", 15))
|
||||
self.caption_limit = max(128, await fetch_int_setting("telegram_caption_limit", 1024))
|
||||
self.message_limit = max(512, await fetch_int_setting("telegram_message_limit", 4096))
|
||||
self.text_overflow_marker = str(await fetch_setting("telegram_text_overflow_marker", self.text_overflow_marker))
|
||||
self.download_timeout_sec = max(5, await fetch_int_setting("uploader_download_timeout_sec", 45))
|
||||
self.video_max_size_mb = max(1, await fetch_int_setting("video_max_size_mb", 49))
|
||||
self.video_max_duration_sec = max(1, await fetch_int_setting("video_max_duration_sec", 300))
|
||||
self.yt_dlp_timeout_sec = max(30, await fetch_int_setting("uploader_yt_dlp_timeout_sec", 300))
|
||||
self.max_media_attempts = max(1, await fetch_int_setting("uploader_max_media_attempts", 3))
|
||||
self.post_job_pause_sec = max(0.0, await fetch_float_setting("media_post_job_pause_sec", 0.2))
|
||||
|
||||
logger.info(
|
||||
"Telegram storage config: media_delay={}s, media_group_max_items={}, tg_retry={}, tg_backoff_max={}s, dl_timeout={}s, ytdlp_timeout={}s, post_pause={}s",
|
||||
self.media_upload_delay_sec,
|
||||
self.media_group_max_items,
|
||||
self.tg_retry_max_attempts,
|
||||
self.tg_retry_backoff_max_sec,
|
||||
self.download_timeout_sec,
|
||||
self.yt_dlp_timeout_sec,
|
||||
self.post_job_pause_sec,
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.bot:
|
||||
await self.bot.session.close()
|
||||
|
||||
def chat_kwargs(self) -> dict:
|
||||
kwargs = {"chat_id": self.storage_chat_id}
|
||||
if self.storage_thread_id:
|
||||
kwargs["message_thread_id"] = self.storage_thread_id
|
||||
return kwargs
|
||||
|
||||
async def tg_retry(self, fn):
|
||||
for attempt in range(1, self.tg_retry_max_attempts + 1):
|
||||
try:
|
||||
return await fn()
|
||||
except TelegramRetryAfter as exc:
|
||||
delay = float(exc.retry_after) + 0.5
|
||||
logger.warning("Telegram flood control, sleep {}s", delay)
|
||||
await asyncio.sleep(delay)
|
||||
except Exception as exc:
|
||||
if attempt >= self.tg_retry_max_attempts:
|
||||
raise
|
||||
delay = min(2**attempt, self.tg_retry_backoff_max_sec)
|
||||
logger.warning("Telegram send error: {}. retry in {}s", exc, delay)
|
||||
await asyncio.sleep(delay)
|
||||
raise RuntimeError("telegram retry exhausted")
|
||||
|
||||
async def load_raw_post(self, raw_post_id: int) -> dict | None:
|
||||
row = await self.pool.fetchrow(
|
||||
"""
|
||||
SELECT rp.*, s.name AS source_name
|
||||
FROM raw_posts rp
|
||||
JOIN sources s ON s.id=rp.source_id
|
||||
WHERE rp.id=$1
|
||||
""",
|
||||
raw_post_id,
|
||||
)
|
||||
return dict(row) if row else None
|
||||
|
||||
async def load_media(self, raw_post_id: int) -> list[dict]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT *
|
||||
FROM raw_post_media
|
||||
WHERE raw_post_id=$1
|
||||
ORDER BY sort_order ASC, id ASC
|
||||
""",
|
||||
raw_post_id,
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
async def download_bytes(self, session: aiohttp.ClientSession, url: str) -> bytes | None:
|
||||
try:
|
||||
async with session.get(url, timeout=aiohttp.ClientTimeout(total=self.download_timeout_sec)) as response:
|
||||
if response.status != 200:
|
||||
return None
|
||||
return await response.read()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
async def mark_media_uploaded(self, media_id: int, file_id: str, unique_id: str | None) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE raw_post_media
|
||||
SET status=$2,
|
||||
storage_attachment_id=$3,
|
||||
storage_url=$3,
|
||||
tg_file_id=$3,
|
||||
tg_file_unique_id=$4,
|
||||
error=NULL,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
media_id,
|
||||
MEDIA_STATUS_UPLOADED,
|
||||
file_id,
|
||||
unique_id,
|
||||
)
|
||||
|
||||
async def mark_media_link_only(self, media_id: int, error: str | None = None) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE raw_post_media
|
||||
SET status=$2,
|
||||
error=COALESCE($3, error),
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
media_id,
|
||||
MEDIA_STATUS_LINK_ONLY,
|
||||
error,
|
||||
)
|
||||
|
||||
async def mark_media_failed_attempt(self, media_id: int, error: str) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE raw_post_media
|
||||
SET attempts=attempts+1,
|
||||
status=CASE WHEN attempts + 1 >= $3 THEN $2 ELSE status END,
|
||||
error=$4,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
media_id,
|
||||
MEDIA_STATUS_FAILED,
|
||||
self.max_media_attempts,
|
||||
error[:1000],
|
||||
)
|
||||
|
||||
async def download_video(self, vk_url: str, output_path: str) -> dict | None:
|
||||
parsed = parse_vk_video_url(vk_url)
|
||||
if not parsed:
|
||||
return None
|
||||
owner_id, video_id = parsed
|
||||
max_size = self.video_max_size_mb * 1024 * 1024
|
||||
netrc_path = f"{output_path}.netrc"
|
||||
fd = os.open(netrc_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
||||
fh.write(f"machine vk.com login vk_token password {settings.vk_access_token}\n")
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"yt_dlp",
|
||||
"--netrc-location",
|
||||
netrc_path,
|
||||
f"https://vk.com/video{owner_id}_{video_id}",
|
||||
"-o",
|
||||
output_path,
|
||||
"--no-playlist",
|
||||
"-f",
|
||||
f"best[filesize<{max_size}]/bestvideo[filesize<{max_size}]+bestaudio/best",
|
||||
"--quiet",
|
||||
"--no-warnings",
|
||||
]
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
_, stderr = await asyncio.wait_for(proc.communicate(), timeout=self.yt_dlp_timeout_sec)
|
||||
except asyncio.TimeoutError:
|
||||
proc.kill()
|
||||
await proc.communicate()
|
||||
return {"error": "video download timeout", "permanent": False}
|
||||
finally:
|
||||
try:
|
||||
os.unlink(netrc_path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
if proc.returncode != 0 or not os.path.exists(output_path):
|
||||
err = (stderr or b"").decode("utf-8", errors="ignore").lower()
|
||||
permanent = any(marker in err for marker in ("removed", "unavailable", "private", "access denied"))
|
||||
return {"error": "video unavailable or download failed", "permanent": permanent}
|
||||
size = os.path.getsize(output_path)
|
||||
if size > max_size:
|
||||
return {"error": "video too large", "permanent": True}
|
||||
return {"path": output_path, "size_bytes": size}
|
||||
|
||||
async def prepare_media(self, raw_post_id: int, media: list[dict]) -> list[dict]:
|
||||
prepared: list[dict] = []
|
||||
async with aiohttp.ClientSession() as session:
|
||||
for item in media:
|
||||
media_id = int(item["id"])
|
||||
media_type = str(item["media_type"])
|
||||
url = str(item.get("original_url") or "")
|
||||
|
||||
if item.get("tg_file_id"):
|
||||
prepared.append({"media_id": media_id, "media_type": media_type, "media": item["tg_file_id"]})
|
||||
continue
|
||||
if int(item.get("attempts") or 0) >= self.max_media_attempts:
|
||||
continue
|
||||
|
||||
if media_type == "photo":
|
||||
data = await self.download_bytes(session, url)
|
||||
if not data:
|
||||
await self.mark_media_failed_attempt(media_id, "photo download failed")
|
||||
continue
|
||||
prepared.append(
|
||||
{
|
||||
"media_id": media_id,
|
||||
"media_type": "photo",
|
||||
"media": BufferedInputFile(data, filename=f"photo_{media_id}.jpg"),
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
if media_type == "video":
|
||||
duration = item.get("duration_sec")
|
||||
if duration and int(duration) > self.video_max_duration_sec:
|
||||
await self.mark_media_link_only(media_id, "video too long")
|
||||
continue
|
||||
temp_path = str(TMP_DIR / f"{TMP_PREFIX}{raw_post_id}_{media_id}.mp4")
|
||||
info = await self.download_video(url, temp_path)
|
||||
if not info:
|
||||
await self.mark_media_failed_attempt(media_id, "video download failed")
|
||||
continue
|
||||
if info.get("error"):
|
||||
if info.get("permanent"):
|
||||
await self.mark_media_link_only(media_id, str(info["error"]))
|
||||
else:
|
||||
await self.mark_media_failed_attempt(media_id, str(info["error"]))
|
||||
continue
|
||||
prepared.append(
|
||||
{
|
||||
"media_id": media_id,
|
||||
"media_type": "video",
|
||||
"media": FSInputFile(temp_path, filename="video.mp4"),
|
||||
"tmp_path": temp_path,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
await self.mark_media_link_only(media_id, "unsupported media type")
|
||||
return prepared
|
||||
|
||||
async def send_text_chunk(self, text: str) -> int:
|
||||
message = await self.tg_retry(
|
||||
lambda: self.bot.send_message(
|
||||
text=text,
|
||||
parse_mode="HTML",
|
||||
disable_web_page_preview=True,
|
||||
**self.chat_kwargs(),
|
||||
)
|
||||
)
|
||||
return int(message.message_id)
|
||||
|
||||
def build_text_parts(self, post: dict, has_media: bool, link_only_media: list[dict] | None = None) -> tuple[str | None, list[str]]:
|
||||
link = original_link(post)
|
||||
raw_text = html.escape(str(post.get("raw_text") or "").strip())
|
||||
link_only_media = link_only_media or []
|
||||
media_lines = []
|
||||
for item in link_only_media:
|
||||
url = str(item.get("original_url") or "").strip()
|
||||
if not url:
|
||||
continue
|
||||
media_type = html.escape(str(item.get("media_type") or "media"))
|
||||
reason = html.escape(str(item.get("error") or "link only"))
|
||||
media_lines.append(f"- {media_type}: <a href=\"{html.escape(url, quote=True)}\">ссылка</a> ({reason})")
|
||||
media_note = "\n\nМедиа по ссылке:\n" + "\n".join(media_lines) if media_lines else ""
|
||||
suffix = f"\n\n{link}"
|
||||
body = raw_text + media_note
|
||||
if has_media:
|
||||
if len(body + suffix) <= self.caption_limit:
|
||||
return body + suffix if body else link, []
|
||||
chunks = split_message_chunks(body + suffix, self.message_limit)
|
||||
return self.text_overflow_marker[: self.caption_limit], chunks
|
||||
chunks = split_message_chunks(body + suffix if body else link, self.message_limit)
|
||||
return None, chunks
|
||||
|
||||
async def save_sent_media_ids(self, prepared: list[dict], messages: list) -> None:
|
||||
for item, message in zip(prepared, messages):
|
||||
if item["media_type"] == "photo" and message.photo:
|
||||
photo = message.photo[-1]
|
||||
await self.mark_media_uploaded(int(item["media_id"]), photo.file_id, photo.file_unique_id)
|
||||
elif item["media_type"] == "video" and message.video:
|
||||
video = message.video
|
||||
await self.mark_media_uploaded(int(item["media_id"]), video.file_id, video.file_unique_id)
|
||||
|
||||
async def send_storage_post(self, post: dict, prepared: list[dict], media: list[dict]) -> tuple[list[int], int | None]:
|
||||
uploaded_media_ids = {int(item["media_id"]) for item in prepared}
|
||||
link_only_media = [
|
||||
item
|
||||
for item in media
|
||||
if int(item["id"]) not in uploaded_media_ids and str(item.get("status") or "") == MEDIA_STATUS_LINK_ONLY
|
||||
]
|
||||
caption, text_chunks = self.build_text_parts(post, bool(prepared), link_only_media)
|
||||
message_ids: list[int] = []
|
||||
|
||||
if prepared:
|
||||
first = prepared[: self.media_group_max_items]
|
||||
group = []
|
||||
for idx, item in enumerate(first):
|
||||
cap = caption if idx == 0 else None
|
||||
if item["media_type"] == "photo":
|
||||
group.append(InputMediaPhoto(media=item["media"], caption=cap, parse_mode="HTML"))
|
||||
elif item["media_type"] == "video":
|
||||
group.append(InputMediaVideo(media=item["media"], caption=cap, parse_mode="HTML"))
|
||||
if group:
|
||||
messages = await self.tg_retry(lambda: self.bot.send_media_group(media=group, **self.chat_kwargs()))
|
||||
await self.save_sent_media_ids(first, messages)
|
||||
message_ids.extend(int(msg.message_id) for msg in messages)
|
||||
await asyncio.sleep(self.media_upload_delay_sec * len(first))
|
||||
|
||||
rest = prepared[self.media_group_max_items :]
|
||||
for start in range(0, len(rest), self.media_group_max_items):
|
||||
chunk = rest[start : start + self.media_group_max_items]
|
||||
group = [
|
||||
InputMediaPhoto(media=item["media"])
|
||||
if item["media_type"] == "photo"
|
||||
else InputMediaVideo(media=item["media"])
|
||||
for item in chunk
|
||||
if item["media_type"] in ("photo", "video")
|
||||
]
|
||||
if group:
|
||||
messages = await self.tg_retry(lambda g=group: self.bot.send_media_group(media=g, **self.chat_kwargs()))
|
||||
await self.save_sent_media_ids(chunk, messages)
|
||||
message_ids.extend(int(msg.message_id) for msg in messages)
|
||||
await asyncio.sleep(self.media_upload_delay_sec * len(chunk))
|
||||
for chunk in text_chunks:
|
||||
message_ids.append(await self.send_text_chunk(chunk))
|
||||
return message_ids, None
|
||||
|
||||
async def mark_post_ready(self, raw_post_id: int, message_ids: list[int], meta_message_id: int | None) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE raw_posts
|
||||
SET status=$2,
|
||||
tg_storage_chat_id=$3,
|
||||
tg_storage_thread_id=$4,
|
||||
tg_storage_message_ids=$5,
|
||||
tg_storage_meta_message_id=$6,
|
||||
storage_post_url=$7,
|
||||
copied_at=NOW(),
|
||||
error_reason=NULL,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
raw_post_id,
|
||||
POST_STATUS_STORAGE_READY,
|
||||
self.storage_chat_id,
|
||||
self.storage_thread_id,
|
||||
message_ids,
|
||||
meta_message_id,
|
||||
f"tg://resolve?domain=c/{str(abs(int(self.storage_chat_id or 0))).removeprefix('100')}/{message_ids[0]}" if message_ids else None,
|
||||
)
|
||||
|
||||
async def mark_post_failed(self, raw_post_id: int, error: str) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE raw_posts
|
||||
SET status=$2,
|
||||
error_reason=$3,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
raw_post_id,
|
||||
POST_STATUS_FAILED,
|
||||
error[:1000],
|
||||
)
|
||||
|
||||
async def process_job(self, job: dict, worker_id: str) -> None:
|
||||
job_id = int(job["id"])
|
||||
raw_post_id = int(job["entity_id"])
|
||||
await self.heartbeat.beat(self.pool, current_job_id=job_id, force=True)
|
||||
|
||||
post = await self.load_raw_post(raw_post_id)
|
||||
if not post:
|
||||
await ack_retry(self.pool, job, f"raw_post {raw_post_id} not found", delay_sec=60)
|
||||
return
|
||||
if post.get("tg_storage_message_ids"):
|
||||
await ack_done(self.pool, job_id)
|
||||
return
|
||||
|
||||
media = await self.load_media(raw_post_id)
|
||||
prepared = await self.prepare_media(raw_post_id, media)
|
||||
media = await self.load_media(raw_post_id)
|
||||
try:
|
||||
message_ids, meta_message_id = await self.send_storage_post(post, prepared, media)
|
||||
finally:
|
||||
for item in prepared:
|
||||
tmp_path = item.get("tmp_path")
|
||||
if tmp_path:
|
||||
try:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
except Exception:
|
||||
pass
|
||||
await self.mark_post_ready(raw_post_id, message_ids, meta_message_id)
|
||||
await ack_done(self.pool, job_id)
|
||||
logger.info("Telegram storage done: raw_post={} messages={}", raw_post_id, message_ids)
|
||||
|
||||
async def run_once(self, worker_id: str) -> bool:
|
||||
enabled = await is_worker_enabled(self.pool, WORKER_STORAGE_UPLOADER)
|
||||
if not enabled:
|
||||
await self.heartbeat.beat(self.pool, status="disabled", force=True)
|
||||
return False
|
||||
|
||||
job = await claim_job(self.pool, JOB_TYPE_VK_STORAGE_COPY, worker_id)
|
||||
if not job:
|
||||
await self.heartbeat.beat(self.pool, status="idle")
|
||||
return False
|
||||
try:
|
||||
await self.process_job(job, worker_id)
|
||||
except Exception as exc:
|
||||
await self.mark_post_failed(int(job["entity_id"]), str(exc))
|
||||
await ack_retry(self.pool, job, str(exc), delay_sec=120)
|
||||
logger.exception("Telegram storage job {} failed: {}", job["id"], exc)
|
||||
return True
|
||||
|
||||
async def cleanup_tmp_files(self) -> None:
|
||||
now = time.time()
|
||||
for path in TMP_DIR.glob(f"{TMP_PREFIX}*"):
|
||||
try:
|
||||
if path.is_file() and path.stat().st_mtime < now - 3600:
|
||||
path.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def run_loop(self) -> None:
|
||||
await self.init()
|
||||
worker_id = f"{WORKER_STORAGE_UPLOADER}:{os.getpid()}"
|
||||
logger.info("{} started", worker_id)
|
||||
try:
|
||||
while True:
|
||||
await self.cleanup_tmp_files()
|
||||
had_job = await self.run_once(worker_id)
|
||||
if not had_job:
|
||||
await asyncio.sleep(max(1, await fetch_int_setting("uploader_interval_sec", 5)))
|
||||
else:
|
||||
await asyncio.sleep(self.post_job_pause_sec)
|
||||
finally:
|
||||
await self.close()
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
logger.remove()
|
||||
logger.add(sys.stdout, level=settings.log_level)
|
||||
worker = TelegramStorageUploader()
|
||||
await worker.run_loop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user