From 59c899dc9c48f372753b653370d7107218709894 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 15 Jun 2026 21:14:07 +0500 Subject: [PATCH] Initial parser admin implementation --- .env.example | 23 + .gitattributes | 5 + .gitignore | 15 + README.md | 199 ++ db/migrations/001_initial.sql | 194 ++ db/migrations/002_telegram_storage.sql | 27 + .../003_parser_hardening_settings.sql | 16 + db/migrations/004_junk_post_policy.sql | 28 + db/migrations/005_ai_qualifier.sql | 50 + .../006_russian_settings_and_ai_catalog.sql | 47 + db/migrations/007_ai_writer.sql | 57 + db/migrations/008_ai_prompt_text.sql | 5 + .../009_qualification_reject_tag.sql | 6 + .../010_source_tags_and_rewrite_category.sql | 53 + db/migrations/011_safer_ai_settings.sql | 39 + .../012_qualification_model_decision.sql | 15 + db/migrations/013_editorial_review.sql | 32 + ...14_prompt_contracts_and_admin_security.sql | 43 + db/migrations/015_editor_media_flags.sql | 9 + ..._remove_regeneration_and_writer_prompt.sql | 56 + ...7_content_categories_and_media_cleanup.sql | 24 + requirements.txt | 11 + scripts/apply_migrations.py | 16 + scripts/vk_storage_spike.py | 37 + src/vk_parser_app/__init__.py | 3 + src/vk_parser_app/admin.py | 2494 +++++++++++++++++ src/vk_parser_app/config.py | 45 + src/vk_parser_app/constants.py | 30 + src/vk_parser_app/db.py | 99 + src/vk_parser_app/heartbeat.py | 45 + src/vk_parser_app/jobs.py | 98 + src/vk_parser_app/main.py | 3 + src/vk_parser_app/security.py | 37 + src/vk_parser_app/templates/base.html | 229 ++ src/vk_parser_app/templates/editor.html | 338 +++ src/vk_parser_app/templates/login.html | 12 + src/vk_parser_app/templates/pagination.html | 37 + .../templates/raw_post_detail.html | 194 ++ src/vk_parser_app/templates/raw_posts.html | 211 ++ src/vk_parser_app/templates/source_form.html | 25 + src/vk_parser_app/templates/sources.html | 123 + src/vk_parser_app/templates/table_tools.html | 43 + src/vk_parser_app/templates/users.html | 35 + .../templates/vk_oauth_callback.html | 52 + src/vk_parser_app/templates/workers.html | 141 + src/vk_parser_app/text_utils.py | 28 + src/vk_parser_app/vk_api.py | 290 ++ src/vk_parser_app/workers/__init__.py | 1 + src/vk_parser_app/workers/ai_qualifier.py | 409 +++ src/vk_parser_app/workers/ai_writer.py | 456 +++ .../workers/vk_storage_uploader.py | 567 ++++ 51 files changed, 7052 insertions(+) create mode 100644 .env.example create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 README.md create mode 100644 db/migrations/001_initial.sql create mode 100644 db/migrations/002_telegram_storage.sql create mode 100644 db/migrations/003_parser_hardening_settings.sql create mode 100644 db/migrations/004_junk_post_policy.sql create mode 100644 db/migrations/005_ai_qualifier.sql create mode 100644 db/migrations/006_russian_settings_and_ai_catalog.sql create mode 100644 db/migrations/007_ai_writer.sql create mode 100644 db/migrations/008_ai_prompt_text.sql create mode 100644 db/migrations/009_qualification_reject_tag.sql create mode 100644 db/migrations/010_source_tags_and_rewrite_category.sql create mode 100644 db/migrations/011_safer_ai_settings.sql create mode 100644 db/migrations/012_qualification_model_decision.sql create mode 100644 db/migrations/013_editorial_review.sql create mode 100644 db/migrations/014_prompt_contracts_and_admin_security.sql create mode 100644 db/migrations/015_editor_media_flags.sql create mode 100644 db/migrations/016_remove_regeneration_and_writer_prompt.sql create mode 100644 db/migrations/017_content_categories_and_media_cleanup.sql create mode 100644 requirements.txt create mode 100644 scripts/apply_migrations.py create mode 100644 scripts/vk_storage_spike.py create mode 100644 src/vk_parser_app/__init__.py create mode 100644 src/vk_parser_app/admin.py create mode 100644 src/vk_parser_app/config.py create mode 100644 src/vk_parser_app/constants.py create mode 100644 src/vk_parser_app/db.py create mode 100644 src/vk_parser_app/heartbeat.py create mode 100644 src/vk_parser_app/jobs.py create mode 100644 src/vk_parser_app/main.py create mode 100644 src/vk_parser_app/security.py create mode 100644 src/vk_parser_app/templates/base.html create mode 100644 src/vk_parser_app/templates/editor.html create mode 100644 src/vk_parser_app/templates/login.html create mode 100644 src/vk_parser_app/templates/pagination.html create mode 100644 src/vk_parser_app/templates/raw_post_detail.html create mode 100644 src/vk_parser_app/templates/raw_posts.html create mode 100644 src/vk_parser_app/templates/source_form.html create mode 100644 src/vk_parser_app/templates/sources.html create mode 100644 src/vk_parser_app/templates/table_tools.html create mode 100644 src/vk_parser_app/templates/users.html create mode 100644 src/vk_parser_app/templates/vk_oauth_callback.html create mode 100644 src/vk_parser_app/templates/workers.html create mode 100644 src/vk_parser_app/text_utils.py create mode 100644 src/vk_parser_app/vk_api.py create mode 100644 src/vk_parser_app/workers/__init__.py create mode 100644 src/vk_parser_app/workers/ai_qualifier.py create mode 100644 src/vk_parser_app/workers/ai_writer.py create mode 100644 src/vk_parser_app/workers/vk_storage_uploader.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d88b3fc --- /dev/null +++ b/.env.example @@ -0,0 +1,23 @@ +APP_ENV=production +APP_SECRET_KEY=change-me +ADMIN_BOOTSTRAP_LOGIN=admin +ADMIN_BOOTSTRAP_PASSWORD=change-me-long-password + +DB_HOST=localhost +DB_PORT=5432 +DB_NAME=vk_parser +DB_USER=vk_parser_user +DB_PASSWORD=change-me + +VK_ACCESS_TOKEN= +VK_GROUP_ACCESS_TOKEN= +VK_API_VERSION=5.199 +VK_STORAGE_GROUP_ID=239548476 + +TG_BOT_TOKEN= +TG_MEDIA_CHANNEL_ID= +LOCAL_BOT_API_URL= + +ADMIN_HOST=0.0.0.0 +ADMIN_PORT=8080 +LOG_LEVEL=INFO diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..fea410f --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +* text=auto eol=lf + +*.bat text eol=crlf +*.cmd text eol=crlf +*.ps1 text eol=crlf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..28f65d1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +.env +.env.* +!.env.example +__pycache__/ +*.py[cod] +.venv/ +logs/ +uploads/ +original_project/ +ai_worker.py +media_worker.py +parser.py +raw_feed_worker.py +vkdev_main.js +sample_photo.jpg diff --git a/README.md b/README.md new file mode 100644 index 0000000..874cb65 --- /dev/null +++ b/README.md @@ -0,0 +1,199 @@ +# VK Parser New Core + +Первый чистый слой проекта: + +- админка с источниками, пользователями, воркерами и raw-лентой; +- VK parser worker; +- VK storage uploader worker; +- PostgreSQL job queue. + +Старый проект лежит в `original_project/` как справочник. Новый код живёт в `src/vk_parser_app/`. + +## Что нужно на сервере + +Ubuntu 22.04/24.04: + +```bash +sudo apt update +sudo apt install -y python3 python3-venv python3-pip postgresql postgresql-contrib nginx git curl +``` + +Для будущей работы с видео: + +```bash +sudo apt install -y ffmpeg +``` + +## БД + +Пример: + +```bash +sudo -u postgres psql +``` + +```sql +CREATE DATABASE vk_parser; +CREATE USER vk_parser_user WITH PASSWORD 'CHANGE_ME'; +GRANT ALL PRIVILEGES ON DATABASE vk_parser TO vk_parser_user; +\c vk_parser +GRANT ALL ON SCHEMA public TO vk_parser_user; +``` + +## Установка проекта + +```bash +cd /opt +sudo git clone vk-parser +sudo chown -R $USER:$USER /opt/vk-parser +cd /opt/vk-parser + +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +cp .env.example .env +``` + +Заполнить `.env`: + +```env +APP_SECRET_KEY=long-random-string +ADMIN_BOOTSTRAP_LOGIN=admin +ADMIN_BOOTSTRAP_PASSWORD=long-admin-password + +DB_HOST=localhost +DB_PORT=5432 +DB_NAME=vk_parser +DB_USER=vk_parser_user +DB_PASSWORD=... + +VK_ACCESS_TOKEN=... +VK_STORAGE_GROUP_ID=239548476 +``` + +Важно: для `wall.post`, `wall.edit`, `photos.saveWallPhoto` нужен VK user token пользователя с правами в storage-группе. Community token для этого контура не подходит. + +## Миграции + +```bash +PYTHONPATH=src .venv/bin/python scripts/apply_migrations.py +``` + +Первый админ создаётся из `ADMIN_BOOTSTRAP_LOGIN` / `ADMIN_BOOTSTRAP_PASSWORD` при старте админки. + +## Проверка VK storage + +Без фото: + +```bash +PYTHONPATH=src .venv/bin/python scripts/vk_storage_spike.py +``` + +С фото: + +```bash +cp /path/to/photo.jpg sample_photo.jpg +PYTHONPATH=src .venv/bin/python scripts/vk_storage_spike.py +``` + +Скрипт должен вывести ссылку вида: + +```text +https://vk.com/wall-239548476_123 +``` + +После этого надо руками проверить: + +- пост создан от имени storage-группы; +- админ/редактор группы может открыть и отредактировать пост; +- вложенное фото видно в браузере. + +## Запуск локально + +Админка: + +```bash +PYTHONPATH=src .venv/bin/uvicorn vk_parser_app.admin:app --host 0.0.0.0 --port 8080 +``` + +Парсер: + +```bash +PYTHONPATH=src .venv/bin/python -m vk_parser_app.workers.parser +``` + +VK storage uploader: + +```bash +PYTHONPATH=src .venv/bin/python -m vk_parser_app.workers.vk_storage_uploader +``` + +## systemd units + +Минимальный `vk-admin.service`: + +```ini +[Unit] +Description=VK Parser Admin +After=network.target postgresql.service + +[Service] +WorkingDirectory=/opt/vk-parser +Environment=PYTHONPATH=/opt/vk-parser/src +ExecStart=/opt/vk-parser/.venv/bin/uvicorn vk_parser_app.admin:app --host 0.0.0.0 --port 8080 +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target +``` + +`vk-parser.service`: + +```ini +[Unit] +Description=VK Parser Worker +After=network.target postgresql.service + +[Service] +WorkingDirectory=/opt/vk-parser +Environment=PYTHONPATH=/opt/vk-parser/src +ExecStart=/opt/vk-parser/.venv/bin/python -m vk_parser_app.workers.parser +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target +``` + +`vk-storage-uploader.service`: + +```ini +[Unit] +Description=VK Storage Uploader Worker +After=network.target postgresql.service + +[Service] +WorkingDirectory=/opt/vk-parser +Environment=PYTHONPATH=/opt/vk-parser/src +ExecStart=/opt/vk-parser/.venv/bin/python -m vk_parser_app.workers.vk_storage_uploader +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target +``` + +## Текущий pipeline + +```text +sources(vk) + -> vk-parser + -> raw_posts + raw_post_media + -> jobs(type='vk.storage.copy') + -> vk-storage-uploader + -> wall.post в storage-группе + -> raw_posts.storage_post_url +``` + +Фото копируются в storage-группу. Видео в первой версии сохраняются как `link_only`/исходное VK-вложение, чтобы сначала стабилизировать основной контур. diff --git a/db/migrations/001_initial.sql b/db/migrations/001_initial.sql new file mode 100644 index 0000000..8e02d1b --- /dev/null +++ b/db/migrations/001_initial.sql @@ -0,0 +1,194 @@ +CREATE TABLE IF NOT EXISTS admin_users ( + id BIGSERIAL PRIMARY KEY, + login TEXT NOT NULL UNIQUE, + password_hash TEXT NOT NULL, + role TEXT NOT NULL DEFAULT 'admin' CHECK (role IN ('admin','editor','viewer')), + is_active BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS admin_sessions ( + token_hash TEXT PRIMARY KEY, + user_id BIGINT NOT NULL REFERENCES admin_users(id) ON DELETE CASCADE, + csrf_token TEXT NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_admin_sessions_user_id ON admin_sessions(user_id); +CREATE INDEX IF NOT EXISTS idx_admin_sessions_expires_at ON admin_sessions(expires_at); + +CREATE TABLE IF NOT EXISTS sources ( + id BIGSERIAL PRIMARY KEY, + platform TEXT NOT NULL DEFAULT 'vk', + name TEXT NOT NULL, + url TEXT NOT NULL, + external_id TEXT, + external_owner_id BIGINT, + active BOOLEAN NOT NULL DEFAULT TRUE, + status TEXT NOT NULL DEFAULT 'new', + status_msg TEXT, + parse_from TIMESTAMPTZ, + last_checked_at TIMESTAMPTZ, + last_parsed_at TIMESTAMPTZ, + priority INTEGER NOT NULL DEFAULT 100, + tags TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[], + settings_json JSONB NOT NULL DEFAULT '{}'::jsonb, + created_by BIGINT REFERENCES admin_users(id) ON DELETE SET NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + archived_at TIMESTAMPTZ +); + +CREATE UNIQUE INDEX IF NOT EXISTS ux_sources_platform_url_active +ON sources(platform, lower(url)) +WHERE archived_at IS NULL; + +CREATE INDEX IF NOT EXISTS idx_sources_active_platform ON sources(active, platform, priority, id) +WHERE archived_at IS NULL; + +CREATE TABLE IF NOT EXISTS raw_posts ( + id BIGSERIAL PRIMARY KEY, + source_id BIGINT NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + platform TEXT NOT NULL, + external_post_id TEXT NOT NULL, + external_owner_id BIGINT, + original_url TEXT NOT NULL, + storage_owner_id BIGINT, + storage_post_id BIGINT, + storage_post_url TEXT, + raw_text TEXT NOT NULL DEFAULT '', + raw_json JSONB NOT NULL DEFAULT '{}'::jsonb, + text_hash TEXT NOT NULL, + content_hash TEXT NOT NULL, + posted_at TIMESTAMPTZ, + copied_at TIMESTAMPTZ, + status TEXT NOT NULL DEFAULT 'raw_saved', + skip_reason TEXT, + error_reason TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE(source_id, external_post_id) +); + +CREATE INDEX IF NOT EXISTS idx_raw_posts_status_created ON raw_posts(status, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_raw_posts_source_posted ON raw_posts(source_id, posted_at DESC); +CREATE INDEX IF NOT EXISTS idx_raw_posts_content_hash ON raw_posts(content_hash); + +CREATE TABLE IF NOT EXISTS raw_post_media ( + id BIGSERIAL PRIMARY KEY, + raw_post_id BIGINT NOT NULL REFERENCES raw_posts(id) ON DELETE CASCADE, + platform TEXT NOT NULL, + media_type TEXT NOT NULL CHECK (media_type IN ('photo','video','link','doc','unknown')), + original_url TEXT, + original_attachment_id TEXT, + storage_attachment_id TEXT, + storage_url TEXT, + preview_url TEXT, + width INTEGER, + height INTEGER, + duration_sec INTEGER, + sort_order INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'pending', + attempts INTEGER NOT NULL DEFAULT 0, + error TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_raw_post_media_post ON raw_post_media(raw_post_id, sort_order, id); +CREATE INDEX IF NOT EXISTS idx_raw_post_media_status ON raw_post_media(status); + +CREATE TABLE IF NOT EXISTS jobs ( + id BIGSERIAL PRIMARY KEY, + type TEXT NOT NULL, + entity_type TEXT NOT NULL, + entity_id BIGINT NOT NULL, + payload_json JSONB NOT NULL DEFAULT '{}'::jsonb, + status TEXT NOT NULL DEFAULT 'pending', + attempts INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 5, + next_run_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + locked_by TEXT, + locked_at TIMESTAMPTZ, + last_error TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE UNIQUE INDEX IF NOT EXISTS ux_jobs_open_entity_type +ON jobs(type, entity_type, entity_id) +WHERE status IN ('pending','retry','in_progress'); + +CREATE INDEX IF NOT EXISTS idx_jobs_claim ON jobs(type, status, next_run_at, id); + +CREATE TABLE IF NOT EXISTS worker_controls ( + name TEXT PRIMARY KEY, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + settings_json JSONB NOT NULL DEFAULT '{}'::jsonb, + updated_by BIGINT REFERENCES admin_users(id) ON DELETE SET NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS worker_heartbeats ( + name TEXT PRIMARY KEY, + heartbeat_at TIMESTAMPTZ NOT NULL, + status TEXT NOT NULL DEFAULT 'running', + current_job_id BIGINT, + meta_json JSONB NOT NULL DEFAULT '{}'::jsonb, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS app_settings ( + key TEXT PRIMARY KEY, + value_json JSONB NOT NULL, + value_type TEXT NOT NULL DEFAULT 'str', + title TEXT NOT NULL DEFAULT '', + description TEXT NOT NULL DEFAULT '', + category TEXT NOT NULL DEFAULT 'General', + updated_by BIGINT REFERENCES admin_users(id) ON DELETE SET NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS audit_log ( + id BIGSERIAL PRIMARY KEY, + actor_id BIGINT REFERENCES admin_users(id) ON DELETE SET NULL, + action TEXT NOT NULL, + entity_type TEXT NOT NULL, + entity_id BIGINT, + before_json JSONB, + after_json JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +INSERT INTO worker_controls(name, enabled, settings_json) +VALUES + ('vk-parser', TRUE, '{}'::jsonb), + ('vk-storage-uploader', TRUE, '{}'::jsonb) +ON CONFLICT (name) DO NOTHING; + +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('parser_interval_sec', '300'::jsonb, 'int', 'Parser interval, sec', 'Pause between parser cycles.', 'Parser'), + ('parser_new_source_lookback_days', '14'::jsonb, 'int', 'New source lookback, days', 'How far back to read a new source.', 'Parser'), + ('parser_reparse_overlap_minutes', '120'::jsonb, 'int', 'Reparse overlap, min', 'Overlap window for known sources.', 'Parser'), + ('parser_min_text_length', '50'::jsonb, 'int', 'Minimum text length', 'Posts with shorter text are treated as junk.', 'Parser'), + ('parser_skip_reposts', 'true'::jsonb, 'bool', 'Skip reposts', 'Do not save reposts from VK copy_history.', 'Parser'), + ('parser_skip_empty_text', 'true'::jsonb, 'bool', 'Skip empty text posts', 'Treat posts without text as junk.', 'Parser'), + ('parser_skip_no_media', 'true'::jsonb, 'bool', 'Skip posts without media', 'Treat posts without media as junk.', 'Parser'), + ('parser_skip_text_too_short', 'true'::jsonb, 'bool', 'Skip short text posts', 'Treat posts shorter than parser_min_text_length as junk.', 'Parser'), + ('parser_store_skipped_posts', 'false'::jsonb, 'bool', 'Store skipped posts', 'Keep junk posts in raw_posts instead of dropping them.', 'Parser'), + ('parser_dedupe_content_hash', 'true'::jsonb, 'bool', 'Dedupe by content hash', 'Skip posts whose text and media hash already exists in raw_posts.', 'Parser'), + ('parser_source_pause_sec', '0'::jsonb, 'float', 'Source pause, sec', 'Pause between VK sources inside one parser cycle.', 'Parser'), + ('vk_requests_per_second', '3'::jsonb, 'int', 'VK requests per second', 'Global VK API rate limit per worker.', 'VK'), + ('vk_wall_page_size', '50'::jsonb, 'int', 'VK wall page size', 'wall.get count per page.', 'VK'), + ('vk_api_timeout_total_sec', '15'::jsonb, 'int', 'VK total timeout, sec', 'Total timeout for one VK API request.', 'VK'), + ('vk_api_timeout_connect_sec', '5'::jsonb, 'int', 'VK connect timeout, sec', 'Connection timeout for VK API.', 'VK'), + ('vk_rate_limit_sleep_sec', '1'::jsonb, 'float', 'VK rate limit sleep, sec', 'Sleep after VK API rate limit error code 6.', 'VK'), + ('vk_api_retry_attempts', '3'::jsonb, 'int', 'VK retry attempts', 'Attempts for temporary VK API/network errors.', 'VK'), + ('vk_api_retry_min_delay_sec', '2'::jsonb, 'float', 'VK retry min delay, sec', 'Initial retry delay for temporary VK errors.', 'VK'), + ('vk_api_retry_max_delay_sec', '10'::jsonb, 'float', 'VK retry max delay, sec', 'Maximum retry delay for temporary VK errors.', 'VK'), + ('uploader_interval_sec', '5'::jsonb, 'int', 'Uploader idle interval, sec', 'Pause when no jobs are available.', 'Uploader'), + ('uploader_download_timeout_sec', '45'::jsonb, 'int', 'Download timeout, sec', 'Timeout for source media download.', 'Uploader') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/002_telegram_storage.sql b/db/migrations/002_telegram_storage.sql new file mode 100644 index 0000000..13d5fdb --- /dev/null +++ b/db/migrations/002_telegram_storage.sql @@ -0,0 +1,27 @@ +ALTER TABLE raw_posts + ADD COLUMN IF NOT EXISTS tg_storage_chat_id BIGINT, + ADD COLUMN IF NOT EXISTS tg_storage_thread_id BIGINT, + ADD COLUMN IF NOT EXISTS tg_storage_message_ids BIGINT[] NOT NULL DEFAULT ARRAY[]::BIGINT[], + ADD COLUMN IF NOT EXISTS tg_storage_meta_message_id BIGINT; + +ALTER TABLE raw_post_media + ADD COLUMN IF NOT EXISTS tg_file_id TEXT, + ADD COLUMN IF NOT EXISTS tg_file_unique_id TEXT; + +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('tg_media_channel_id', '""'::jsonb, 'str', 'Telegram media storage chat', 'Chat id or chat_id:topic_id for raw media storage.', 'Telegram'), + ('local_bot_api_url', '""'::jsonb, 'str', 'Local Telegram Bot API URL', 'Optional local Bot API server URL.', 'Telegram'), + ('telegram_media_group_max_items', '10'::jsonb, 'int', 'Telegram media group max items', 'Maximum media items per Telegram media group.', 'Telegram'), + ('telegram_media_upload_delay_sec', '1'::jsonb, 'float', 'Telegram media upload delay, sec', 'Pause between Telegram upload operations.', 'Telegram'), + ('telegram_retry_max_attempts', '4'::jsonb, 'int', 'Telegram retry attempts', 'Retry attempts for Telegram calls.', 'Telegram'), + ('telegram_retry_backoff_max_sec', '15'::jsonb, 'int', 'Telegram retry max backoff, sec', 'Maximum retry backoff for Telegram calls.', 'Telegram'), + ('telegram_caption_limit', '1024'::jsonb, 'int', 'Telegram caption limit', 'Caption character limit for media messages.', 'Telegram'), + ('telegram_message_limit', '4096'::jsonb, 'int', 'Telegram message limit', 'Text message character limit.', 'Telegram'), + ('telegram_text_overflow_marker', '"Продолжение следующим сообщением."'::jsonb, 'str', 'Telegram overflow marker', 'Caption marker when text does not fit.', 'Telegram'), + ('media_post_job_pause_sec', '0.2'::jsonb, 'float', 'Post job pause, sec', 'Pause after each processed media storage job.', 'Uploader'), + ('video_max_size_mb', '49'::jsonb, 'int', 'Video max size, MB', 'Maximum video size to upload to Telegram.', 'Uploader'), + ('video_max_duration_sec', '300'::jsonb, 'int', 'Video max duration, sec', 'Maximum video duration to upload to Telegram.', 'Uploader'), + ('uploader_yt_dlp_timeout_sec', '300'::jsonb, 'int', 'yt-dlp timeout, sec', 'Timeout for VK video downloads.', 'Uploader'), + ('uploader_max_media_attempts', '3'::jsonb, 'int', 'Media max attempts', 'Maximum attempts per media item.', 'Uploader') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/003_parser_hardening_settings.sql b/db/migrations/003_parser_hardening_settings.sql new file mode 100644 index 0000000..c407a91 --- /dev/null +++ b/db/migrations/003_parser_hardening_settings.sql @@ -0,0 +1,16 @@ +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('parser_skip_reposts', 'true'::jsonb, 'bool', 'Skip reposts', 'Do not save reposts from VK copy_history.', 'Parser'), + ('parser_skip_empty_text', 'true'::jsonb, 'bool', 'Skip empty text posts', 'Treat posts without text as junk.', 'Parser'), + ('parser_skip_no_media', 'true'::jsonb, 'bool', 'Skip posts without media', 'Treat posts without media as junk.', 'Parser'), + ('parser_skip_text_too_short', 'true'::jsonb, 'bool', 'Skip short text posts', 'Treat posts shorter than parser_min_text_length as junk.', 'Parser'), + ('parser_store_skipped_posts', 'false'::jsonb, 'bool', 'Store skipped posts', 'Keep junk posts in raw_posts instead of dropping them.', 'Parser'), + ('parser_dedupe_content_hash', 'true'::jsonb, 'bool', 'Dedupe by content hash', 'Skip posts whose text and media hash already exists in raw_posts.', 'Parser'), + ('parser_source_pause_sec', '0'::jsonb, 'float', 'Source pause, sec', 'Pause between VK sources inside one parser cycle.', 'Parser'), + ('vk_api_timeout_total_sec', '15'::jsonb, 'int', 'VK total timeout, sec', 'Total timeout for one VK API request.', 'VK'), + ('vk_api_timeout_connect_sec', '5'::jsonb, 'int', 'VK connect timeout, sec', 'Connection timeout for VK API.', 'VK'), + ('vk_rate_limit_sleep_sec', '1'::jsonb, 'float', 'VK rate limit sleep, sec', 'Sleep after VK API rate limit error code 6.', 'VK'), + ('vk_api_retry_attempts', '3'::jsonb, 'int', 'VK retry attempts', 'Attempts for temporary VK API/network errors.', 'VK'), + ('vk_api_retry_min_delay_sec', '2'::jsonb, 'float', 'VK retry min delay, sec', 'Initial retry delay for temporary VK errors.', 'VK'), + ('vk_api_retry_max_delay_sec', '10'::jsonb, 'float', 'VK retry max delay, sec', 'Maximum retry delay for temporary VK errors.', 'VK') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/004_junk_post_policy.sql b/db/migrations/004_junk_post_policy.sql new file mode 100644 index 0000000..792a3a8 --- /dev/null +++ b/db/migrations/004_junk_post_policy.sql @@ -0,0 +1,28 @@ +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('parser_skip_empty_text', 'true'::jsonb, 'bool', 'Skip empty text posts', 'Treat posts without text as junk.', 'Parser') +ON CONFLICT (key) DO NOTHING; + +UPDATE app_settings +SET value_json='50'::jsonb, + description='Posts with shorter text are treated as junk.', + updated_at=NOW() +WHERE key='parser_min_text_length' + AND value_json='0'::jsonb; + +UPDATE app_settings +SET value_json='false'::jsonb, + description='Keep junk posts in raw_posts instead of dropping them.', + updated_at=NOW() +WHERE key='parser_store_skipped_posts' + AND value_json='true'::jsonb; + +UPDATE app_settings +SET description='Treat posts without media as junk.', + updated_at=NOW() +WHERE key='parser_skip_no_media'; + +UPDATE app_settings +SET description='Treat posts shorter than parser_min_text_length as junk.', + updated_at=NOW() +WHERE key='parser_skip_text_too_short'; diff --git a/db/migrations/005_ai_qualifier.sql b/db/migrations/005_ai_qualifier.sql new file mode 100644 index 0000000..2acab71 --- /dev/null +++ b/db/migrations/005_ai_qualifier.sql @@ -0,0 +1,50 @@ +ALTER TABLE raw_posts + ADD COLUMN IF NOT EXISTS qualification_status TEXT, + ADD COLUMN IF NOT EXISTS qualification_score INTEGER, + ADD COLUMN IF NOT EXISTS qualification_decision TEXT, + ADD COLUMN IF NOT EXISTS qualification_reason TEXT, + ADD COLUMN IF NOT EXISTS qualification_model TEXT, + ADD COLUMN IF NOT EXISTS qualification_prompt_hash TEXT, + ADD COLUMN IF NOT EXISTS qualification_batch_id BIGINT, + ADD COLUMN IF NOT EXISTS qualified_at TIMESTAMPTZ; + +CREATE INDEX IF NOT EXISTS idx_raw_posts_qualification_claim +ON raw_posts(status, qualification_status, created_at) +WHERE status='storage_ready'; + +CREATE TABLE IF NOT EXISTS ai_qualification_batches ( + id BIGSERIAL PRIMARY KEY, + status TEXT NOT NULL DEFAULT 'processing', + provider TEXT NOT NULL, + model TEXT NOT NULL, + prompt_hash TEXT NOT NULL, + posts_count INTEGER NOT NULL DEFAULT 0, + accepted_count INTEGER NOT NULL DEFAULT 0, + rejected_count INTEGER NOT NULL DEFAULT 0, + maybe_count INTEGER NOT NULL DEFAULT 0, + request_json JSONB NOT NULL DEFAULT '{}'::jsonb, + response_json JSONB, + error TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + completed_at TIMESTAMPTZ +); + +INSERT INTO worker_controls(name, enabled, settings_json) +VALUES ('ai-qualifier', FALSE, '{}'::jsonb) +ON CONFLICT (name) DO NOTHING; + +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('ai_qualifier_enabled', 'false'::jsonb, 'bool', 'AI qualifier enabled', 'Enable qualification requests to AI API.', 'AI Qualifier'), + ('ai_qualifier_provider', '"openrouter"'::jsonb, 'str', 'Provider', 'LiteLLM provider: openrouter, openai, anthropic, gemini, openai_compatible.', 'AI Qualifier'), + ('ai_qualifier_model', '"openrouter/openai/gpt-4.1-mini"'::jsonb, 'str', 'Model', 'Model name in LiteLLM format.', 'AI Qualifier'), + ('ai_qualifier_api_key', '""'::jsonb, 'secret', 'API key', 'AI provider API key.', 'AI Qualifier'), + ('ai_qualifier_api_base', '""'::jsonb, 'str', 'API base', 'Optional base URL for OpenAI-compatible providers.', 'AI Qualifier'), + ('ai_qualifier_batch_size', '30'::jsonb, 'int', 'Batch size', 'How many raw posts to classify in one request.', 'AI Qualifier'), + ('ai_qualifier_min_score', '7'::jsonb, 'int', 'Minimum accepted score', 'Posts with score >= this value are accepted.', 'AI Qualifier'), + ('ai_qualifier_interval_sec', '60'::jsonb, 'int', 'Idle interval, sec', 'Pause between qualifier checks.', 'AI Qualifier'), + ('ai_qualifier_max_text_chars', '2000'::jsonb, 'int', 'Max text chars per post', 'Trim raw text before sending it to AI.', 'AI Qualifier'), + ('ai_qualifier_temperature', '0'::jsonb, 'float', 'Temperature', 'AI sampling temperature for stable classification.', 'AI Qualifier'), + ('ai_qualifier_timeout_sec', '120'::jsonb, 'int', 'Timeout, sec', 'AI request timeout.', 'AI Qualifier'), + ('ai_qualifier_prompt', '"Ты квалификатор постов для проекта, который собирает посты производителей и магазинов милитари, тактической и airsoft-снаряги.\\n\\nОцени каждый raw-пост по пригодности для дальнейшего рерайта и публикации.\\n\\nПодходит: товарные посты, новинки, обзоры, производство, экипировка, тактическая одежда, броня, подсумки, рюкзаки, разгрузки, ремни, патчи, аксессуары, airsoft/страйкбольная снаряга, полезные анонсы производителя.\\n\\nМусор: мемы, поздравления, вакансии, политика, бытовуха, пустые посты, посты без конкретного товара/снаряжения, розыгрыши без предметного контента, чистая реклама скидки без понятного товара, нерелевантные новости.\\n\\nВерни только JSON-объект по схеме: {\\\"results\\\":[{\\\"id\\\":123,\\\"score\\\":8,\\\"decision\\\":\\\"accepted\\\",\\\"reason\\\":\\\"короткая причина\\\"}]}. score — целое число от 1 до 10. decision: accepted, rejected или maybe. Не добавляй markdown и пояснения вне JSON."'::jsonb, 'text', 'Prompt', 'Main qualification prompt.', 'AI Qualifier') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/006_russian_settings_and_ai_catalog.sql b/db/migrations/006_russian_settings_and_ai_catalog.sql new file mode 100644 index 0000000..21cdc73 --- /dev/null +++ b/db/migrations/006_russian_settings_and_ai_catalog.sql @@ -0,0 +1,47 @@ +UPDATE app_settings SET title='Интервал парсера, сек', description='Пауза между полными циклами парсинга источников.' WHERE key='parser_interval_sec'; +UPDATE app_settings SET title='Глубина нового источника, дней', description='Сколько дней назад читать посты при первом запуске источника.' WHERE key='parser_new_source_lookback_days'; +UPDATE app_settings SET title='Перекрытие повторного чтения, мин', description='Насколько назад перечитывать источник, чтобы не пропустить посты.' WHERE key='parser_reparse_overlap_minutes'; +UPDATE app_settings SET title='Минимальная длина текста', description='Посты с текстом короче этого значения считаются мусором.' WHERE key='parser_min_text_length'; +UPDATE app_settings SET title='Пропускать репосты', description='Не брать VK-посты с copy_history.' WHERE key='parser_skip_reposts'; +UPDATE app_settings SET title='Пропускать пустой текст', description='Не брать посты без текста.' WHERE key='parser_skip_empty_text'; +UPDATE app_settings SET title='Пропускать без медиа', description='Не брать посты без фото/видео.' WHERE key='parser_skip_no_media'; +UPDATE app_settings SET title='Пропускать короткий текст', description='Не брать посты короче минимальной длины текста.' WHERE key='parser_skip_text_too_short'; +UPDATE app_settings SET title='Сохранять мусорные посты', description='Если включено, мусорные посты попадут в raw_posts со статусом skipped. Обычно выключено.' WHERE key='parser_store_skipped_posts'; +UPDATE app_settings SET title='Дедупликация по контенту', description='Не брать пост, если такой текст и набор медиа уже есть в raw_posts.' WHERE key='parser_dedupe_content_hash'; +UPDATE app_settings SET title='Пауза между источниками, сек', description='Задержка между чтением разных VK-источников внутри одного цикла.' WHERE key='parser_source_pause_sec'; + +UPDATE app_settings SET title='VK запросов в секунду', description='Ограничение частоты запросов к VK API.' WHERE key='vk_requests_per_second'; +UPDATE app_settings SET title='Размер страницы VK wall.get', description='Сколько постов запрашивать за один вызов wall.get.' WHERE key='vk_wall_page_size'; +UPDATE app_settings SET title='VK общий таймаут, сек', description='Общий таймаут одного запроса к VK API.' WHERE key='vk_api_timeout_total_sec'; +UPDATE app_settings SET title='VK таймаут соединения, сек', description='Таймаут установки соединения с VK API.' WHERE key='vk_api_timeout_connect_sec'; +UPDATE app_settings SET title='Пауза при VK rate limit, сек', description='Сколько ждать после ошибки VK API error_code=6.' WHERE key='vk_rate_limit_sleep_sec'; +UPDATE app_settings SET title='VK retry попытки', description='Сколько раз повторять временно упавший VK-запрос.' WHERE key='vk_api_retry_attempts'; +UPDATE app_settings SET title='VK retry минимум, сек', description='Начальная задержка между повторными VK-запросами.' WHERE key='vk_api_retry_min_delay_sec'; +UPDATE app_settings SET title='VK retry максимум, сек', description='Максимальная задержка между повторными VK-запросами.' WHERE key='vk_api_retry_max_delay_sec'; + +UPDATE app_settings SET title='Интервал аплоадера, сек', description='Пауза, когда нет задач на загрузку raw-постов в Telegram.' WHERE key='uploader_interval_sec'; +UPDATE app_settings SET title='Таймаут скачивания, сек', description='Таймаут скачивания исходного медиа.' WHERE key='uploader_download_timeout_sec'; +UPDATE app_settings SET title='Telegram канал raw', description='ID Telegram-канала/топика для хранения raw-постов.' WHERE key='tg_media_channel_id'; +UPDATE app_settings SET title='Локальный Bot API URL', description='Опциональный URL локального Telegram Bot API сервера.' WHERE key='local_bot_api_url'; +UPDATE app_settings SET title='Задержка на медиа, сек', description='Пауза после отправки медиа, умножается на количество файлов.' WHERE key='media_upload_delay_sec'; +UPDATE app_settings SET title='Пауза после поста, сек', description='Короткая пауза после полной обработки одного raw-поста.' WHERE key='media_post_job_pause_sec'; +UPDATE app_settings SET title='Максимум медиа в группе', description='Максимум файлов в одном Telegram media group.' WHERE key='media_group_max_items'; +UPDATE app_settings SET title='Максимум попыток медиа', description='Сколько раз пытаться скачать/загрузить отдельное медиа.' WHERE key='max_media_attempts'; +UPDATE app_settings SET title='Telegram retry попытки', description='Сколько раз повторять временно упавший Telegram-запрос.' WHERE key='tg_retry_attempts'; +UPDATE app_settings SET title='Telegram backoff максимум, сек', description='Максимальная задержка между retry Telegram-запросов.' WHERE key='tg_retry_backoff_max_sec'; +UPDATE app_settings SET title='Максимум видео, МБ', description='Максимальный размер видео для загрузки в Telegram.' WHERE key='video_max_size_mb'; +UPDATE app_settings SET title='Максимум видео, сек', description='Видео длиннее этого значения помечается как link_only.' WHERE key='video_max_duration_sec'; +UPDATE app_settings SET title='yt-dlp таймаут, сек', description='Сколько ждать скачивания VK-видео через yt-dlp.' WHERE key='uploader_yt_dlp_timeout_sec'; + +UPDATE app_settings SET title='Включить AI-квалификатор', description='Разрешить воркеру отправлять raw-посты в AI API.' WHERE key='ai_qualifier_enabled'; +UPDATE app_settings SET title='Провайдер LLM', description='Провайдер для LiteLLM. Для OpenRouter список моделей подгружается live.' WHERE key='ai_qualifier_provider'; +UPDATE app_settings SET title='Модель LLM', description='Модель для квалификации. Можно выбрать из списка или вписать вручную.' WHERE key='ai_qualifier_model'; +UPDATE app_settings SET title='API токен', description='Токен выбранного LLM-провайдера. Сохраняется в настройках.' WHERE key='ai_qualifier_api_key'; +UPDATE app_settings SET title='API base URL', description='Опциональный base URL для OpenAI-compatible провайдера.' WHERE key='ai_qualifier_api_base'; +UPDATE app_settings SET title='Размер пачки', description='Сколько raw-постов отправлять в один AI-запрос.' WHERE key='ai_qualifier_batch_size'; +UPDATE app_settings SET title='Минимальная оценка', description='Пост принимается, если score больше или равен этому значению.' WHERE key='ai_qualifier_min_score'; +UPDATE app_settings SET title='Интервал проверки, сек', description='Пауза между проверками новых raw-постов.' WHERE key='ai_qualifier_interval_sec'; +UPDATE app_settings SET title='Максимум символов текста', description='Сколько символов raw_text отправлять в AI на один пост.' WHERE key='ai_qualifier_max_text_chars'; +UPDATE app_settings SET title='Температура', description='Температура генерации. Для классификации обычно 0.' WHERE key='ai_qualifier_temperature'; +UPDATE app_settings SET title='AI таймаут, сек', description='Таймаут одного AI-запроса.' WHERE key='ai_qualifier_timeout_sec'; +UPDATE app_settings SET title='Промпт квалификатора', description='Основная инструкция для оценки пригодности raw-постов.' WHERE key='ai_qualifier_prompt'; diff --git a/db/migrations/007_ai_writer.sql b/db/migrations/007_ai_writer.sql new file mode 100644 index 0000000..0c2053d --- /dev/null +++ b/db/migrations/007_ai_writer.sql @@ -0,0 +1,57 @@ +ALTER TABLE ai_qualification_batches + ADD COLUMN IF NOT EXISTS prompt_tokens INTEGER, + ADD COLUMN IF NOT EXISTS completion_tokens INTEGER, + ADD COLUMN IF NOT EXISTS total_tokens INTEGER, + ADD COLUMN IF NOT EXISTS estimated_cost_usd NUMERIC(12, 6); + +ALTER TABLE raw_posts + ADD COLUMN IF NOT EXISTS rewrite_status TEXT, + ADD COLUMN IF NOT EXISTS rewritten_text TEXT, + ADD COLUMN IF NOT EXISTS rewrite_notes TEXT, + ADD COLUMN IF NOT EXISTS rewrite_model TEXT, + ADD COLUMN IF NOT EXISTS rewrite_prompt_hash TEXT, + ADD COLUMN IF NOT EXISTS rewrite_batch_id BIGINT, + ADD COLUMN IF NOT EXISTS rewritten_at TIMESTAMPTZ; + +CREATE INDEX IF NOT EXISTS idx_raw_posts_writer_claim +ON raw_posts(qualification_status, rewrite_status, qualified_at, id) +WHERE qualification_status='accepted'; + +CREATE TABLE IF NOT EXISTS ai_writer_batches ( + id BIGSERIAL PRIMARY KEY, + status TEXT NOT NULL DEFAULT 'processing', + provider TEXT NOT NULL, + model TEXT NOT NULL, + prompt_hash TEXT NOT NULL, + posts_count INTEGER NOT NULL DEFAULT 0, + ready_count INTEGER NOT NULL DEFAULT 0, + failed_count INTEGER NOT NULL DEFAULT 0, + request_json JSONB NOT NULL DEFAULT '{}'::jsonb, + response_json JSONB, + error TEXT, + prompt_tokens INTEGER, + completion_tokens INTEGER, + total_tokens INTEGER, + estimated_cost_usd NUMERIC(12, 6), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + completed_at TIMESTAMPTZ +); + +INSERT INTO worker_controls(name, enabled, settings_json) +VALUES ('ai-writer', FALSE, '{}'::jsonb) +ON CONFLICT (name) DO NOTHING; + +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('ai_writer_enabled', 'false'::jsonb, 'bool', 'AI-райтер включён', 'Разрешает отправлять прошедшие квалификацию посты на рерайт.', 'AI Writer'), + ('ai_writer_provider', '"anthropic"'::jsonb, 'str', 'Провайдер LLM', 'Через какого провайдера LiteLLM отправлять запросы райтера.', 'AI Writer'), + ('ai_writer_model', '"claude-haiku-4-5-20251001"'::jsonb, 'str', 'Модель', 'Модель для рерайта. Для отладки используем дешёвую Haiku.', 'AI Writer'), + ('ai_writer_api_key', '""'::jsonb, 'secret', 'API-ключ', 'Ключ провайдера для AI-райтера. Можно использовать тот же Anthropic-ключ, что и у квалификатора.', 'AI Writer'), + ('ai_writer_api_base', '""'::jsonb, 'str', 'API base URL', 'Необязательный base URL для OpenAI-compatible провайдеров.', 'AI Writer'), + ('ai_writer_prompt', '"Ты редактор Telegram/VK-канала про милитари, тактическую и airsoft-снарягу. Перепиши исходный пост производителя в готовый к публикации пост на русском языке.\\n\\nСтиль: живой, понятный, без канцелярита, без чрезмерной рекламы, без выдуманных фактов. Сохраняй конкретику: модель, материал, цвета, размеры, назначение, цену и условия, если они есть в исходнике. Не добавляй то, чего нет в исходном посте.\\n\\nФормат результата: 1 короткий заголовок первой строкой, затем 2-5 абзацев или аккуратный список, если исходник товарный. В конце не добавляй ссылки и хэштеги.\\n\\nВерни только JSON-объект по схеме: {\\\"rewrites\\\":[{\\\"id\\\":123,\\\"text\\\":\\\"готовый текст\\\",\\\"notes\\\":\\\"короткая заметка для редактора\\\"}]}. Без markdown вне JSON."'::jsonb, 'text', 'Промпт райтера', 'Главный промпт рерайта. Поддерживает обычные переносы строк.', 'AI Writer'), + ('ai_writer_batch_size', '1'::jsonb, 'int', 'Размер batch', 'Сколько постов переписывать за один запрос. Для отладки лучше 1.', 'AI Writer'), + ('ai_writer_max_text_chars', '3500'::jsonb, 'int', 'Максимум символов исходника', 'Обрезает исходный raw-текст перед отправкой в LLM.', 'AI Writer'), + ('ai_writer_temperature', '0.4'::jsonb, 'float', 'Температура генерации', 'Ниже — стабильнее и суше, выше — свободнее и разнообразнее.', 'AI Writer'), + ('ai_writer_timeout_sec', '180'::jsonb, 'int', 'Таймаут, сек', 'Сколько ждать ответ модели.', 'AI Writer'), + ('ai_writer_interval_sec', '60'::jsonb, 'int', 'Интервал ожидания, сек', 'Пауза между проверками очереди writer, когда работы нет.', 'AI Writer') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/008_ai_prompt_text.sql b/db/migrations/008_ai_prompt_text.sql new file mode 100644 index 0000000..6d247ba --- /dev/null +++ b/db/migrations/008_ai_prompt_text.sql @@ -0,0 +1,5 @@ +ALTER TABLE ai_qualification_batches + ADD COLUMN IF NOT EXISTS prompt_text TEXT; + +ALTER TABLE ai_writer_batches + ADD COLUMN IF NOT EXISTS prompt_text TEXT; diff --git a/db/migrations/009_qualification_reject_tag.sql b/db/migrations/009_qualification_reject_tag.sql new file mode 100644 index 0000000..80baf63 --- /dev/null +++ b/db/migrations/009_qualification_reject_tag.sql @@ -0,0 +1,6 @@ +ALTER TABLE raw_posts + ADD COLUMN IF NOT EXISTS qualification_reject_tag TEXT; + +CREATE INDEX IF NOT EXISTS idx_raw_posts_qualification_reject_tag +ON raw_posts(qualification_reject_tag) +WHERE qualification_reject_tag IS NOT NULL; diff --git a/db/migrations/010_source_tags_and_rewrite_category.sql b/db/migrations/010_source_tags_and_rewrite_category.sql new file mode 100644 index 0000000..e9b7490 --- /dev/null +++ b/db/migrations/010_source_tags_and_rewrite_category.sql @@ -0,0 +1,53 @@ +ALTER TABLE sources + ADD COLUMN IF NOT EXISTS tag TEXT; + +UPDATE sources +SET tag = lower(regexp_replace(COALESCE(NULLIF(external_id, ''), name), '[^[:alnum:]_]+', '_', 'g')) +WHERE tag IS NULL OR btrim(tag) = ''; + +ALTER TABLE raw_posts + ADD COLUMN IF NOT EXISTS rewrite_category TEXT, + ADD COLUMN IF NOT EXISTS rewrite_category_tag TEXT, + ADD COLUMN IF NOT EXISTS rewrite_source_tag TEXT; + +CREATE INDEX IF NOT EXISTS idx_sources_tag ON sources(lower(tag)) +WHERE archived_at IS NULL; + +CREATE INDEX IF NOT EXISTS idx_raw_posts_rewrite_category ON raw_posts(rewrite_category) +WHERE rewrite_category IS NOT NULL; + +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('ai_writer_categories', '["защита","одежда","разгрузка","рюкзаки","airsoft","патчи","электроника","аксессуары","производство"]'::jsonb, 'text', 'Категории райтера', 'Список категорий, из которых AI-райтер выбирает одну. Можно писать через запятую или с новой строки.', 'AI Writer') +ON CONFLICT (key) DO NOTHING; + +UPDATE app_settings +SET value_json = to_jsonb($prompt$You are an editor of a Russian Telegram channel about military, tactical and airsoft gear. +Write on behalf of the channel — as an expert observer, not the manufacturer. +Source posts come from manufacturers/shops; your job is to reframe them as channel news. + +VOICE: concise, expert, matter-of-fact. Like telling a knowledgeable friend what dropped. +Example opener framing: "У [Producer] вышел...", "Wartech показали...", "[Producer] добавили в линейку..." + +RULES: +- Use only facts from source text. Never invent specs, properties or claims. +- If source is thin — write a short post, do not pad it. +- No hype, no exclamation spam, no manufacturer PR tone. +- Post length: 3-6 lines. Short is better than padded. +- No bullet lists unless source has 4+ distinct specs worth listing. +- Do not add hashtags. The system will add category and producer hashtags separately. + +CATEGORIES: pick exactly one from the categories field in input. + +INPUT fields used: id, producer_name, producer_tag, text. +qualification_score and media_count are metadata; ignore them for rewrite unless they help understand context. + +OUTPUT: JSON only, no markdown outside JSON.$prompt$::text), + description = 'Главный промпт рерайта. Модель выбирает category, но не пишет хэштеги.', + updated_at = NOW() +WHERE key = 'ai_writer_prompt' + AND ( + value_json::text ILIKE '%хэштег%' + OR value_json::text ILIKE '%hashtag%' + OR value_json::text ILIKE '%#CATEGORY%' + ); diff --git a/db/migrations/011_safer_ai_settings.sql b/db/migrations/011_safer_ai_settings.sql new file mode 100644 index 0000000..22150b2 --- /dev/null +++ b/db/migrations/011_safer_ai_settings.sql @@ -0,0 +1,39 @@ +UPDATE app_settings +SET title='Инструкция квалификатора', + description='Редактируемая человеческая инструкция: критерии оценки и мусора. JSON-контракт ответа добавляется кодом автоматически.' +WHERE key='ai_qualifier_prompt'; + +UPDATE app_settings +SET title='Инструкция райтера', + description='Редактируемая человеческая инструкция: стиль, факты, длина и голос канала. Категорию и JSON-контракт ответа контролирует код.' +WHERE key='ai_writer_prompt'; + +UPDATE app_settings +SET description='Модель для квалификации. Список берётся live из API провайдера, если доступен ключ; fallback помечается отдельно.' +WHERE key='ai_qualifier_model'; + +UPDATE app_settings +SET description='Модель для рерайта. Список берётся live из API провайдера, если доступен ключ; fallback помечается отдельно.' +WHERE key='ai_writer_model'; + +UPDATE app_settings +SET description='Токен выбранного LLM-провайдера. В админке показывается полностью, поэтому доступ к настройкам должен быть только у доверенных пользователей.' +WHERE key IN ('ai_qualifier_api_key', 'ai_writer_api_key'); + +UPDATE app_settings m +SET value_json = to_jsonb('anthropic/' || trim(both '"' from m.value_json::text)) +FROM app_settings p +WHERE p.key = replace(m.key, '_model', '_provider') + AND p.value_json = '"anthropic"'::jsonb + AND m.key IN ('ai_qualifier_model', 'ai_writer_model') + AND m.value_json::text NOT LIKE '"anthropic/%' + AND m.value_json::text NOT LIKE '"%/%"'; + +UPDATE app_settings m +SET value_json = to_jsonb('gemini/' || trim(both '"' from m.value_json::text)) +FROM app_settings p +WHERE p.key = replace(m.key, '_model', '_provider') + AND p.value_json = '"gemini"'::jsonb + AND m.key IN ('ai_qualifier_model', 'ai_writer_model') + AND m.value_json::text NOT LIKE '"gemini/%' + AND m.value_json::text NOT LIKE '"%/%"'; diff --git a/db/migrations/012_qualification_model_decision.sql b/db/migrations/012_qualification_model_decision.sql new file mode 100644 index 0000000..8346bb6 --- /dev/null +++ b/db/migrations/012_qualification_model_decision.sql @@ -0,0 +1,15 @@ +ALTER TABLE raw_posts + ADD COLUMN IF NOT EXISTS qualification_model_decision TEXT; + +UPDATE raw_posts rp +SET qualification_model_decision = item->>'decision' +FROM ai_qualification_batches b, + jsonb_array_elements(COALESCE(b.response_json->'results', '[]'::jsonb)) AS item +WHERE rp.qualification_batch_id = b.id + AND (item->>'id')::bigint = rp.id + AND rp.qualification_model_decision IS NULL + AND item ? 'decision'; + +CREATE INDEX IF NOT EXISTS idx_raw_posts_qualification_model_decision +ON raw_posts(qualification_model_decision) +WHERE qualification_model_decision IS NOT NULL; diff --git a/db/migrations/013_editorial_review.sql b/db/migrations/013_editorial_review.sql new file mode 100644 index 0000000..fb4fd45 --- /dev/null +++ b/db/migrations/013_editorial_review.sql @@ -0,0 +1,32 @@ +ALTER TABLE raw_posts + ADD COLUMN IF NOT EXISTS editorial_status TEXT, + ADD COLUMN IF NOT EXISTS final_text TEXT, + ADD COLUMN IF NOT EXISTS final_category TEXT, + ADD COLUMN IF NOT EXISTS final_category_tag TEXT, + ADD COLUMN IF NOT EXISTS final_source_tag TEXT, + ADD COLUMN IF NOT EXISTS editor_notes TEXT, + ADD COLUMN IF NOT EXISTS regeneration_prompt TEXT, + ADD COLUMN IF NOT EXISTS reviewed_by BIGINT REFERENCES admin_users(id) ON DELETE SET NULL, + ADD COLUMN IF NOT EXISTS reviewed_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS edited_at TIMESTAMPTZ; + +UPDATE raw_posts +SET editorial_status = 'review' +WHERE editorial_status IS NULL + AND rewrite_status = 'ready'; + +UPDATE raw_posts +SET final_text = rewritten_text, + final_category = rewrite_category, + final_category_tag = rewrite_category_tag, + final_source_tag = rewrite_source_tag +WHERE rewrite_status = 'ready' + AND final_text IS NULL; + +CREATE INDEX IF NOT EXISTS idx_raw_posts_editorial_status +ON raw_posts(editorial_status, rewritten_at DESC, id) +WHERE rewrite_status = 'ready'; + +CREATE INDEX IF NOT EXISTS idx_raw_posts_final_category +ON raw_posts(final_category) +WHERE final_category IS NOT NULL; diff --git a/db/migrations/014_prompt_contracts_and_admin_security.sql b/db/migrations/014_prompt_contracts_and_admin_security.sql new file mode 100644 index 0000000..f537cac --- /dev/null +++ b/db/migrations/014_prompt_contracts_and_admin_security.sql @@ -0,0 +1,43 @@ +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ( + 'ai_qualifier_contract', + '"OUTPUT SCHEMA (return array matching input order):\n{\"results\":[{\"id\":123,\"score\":8,\"decision\":\"accepted\",\"reason\":\"до 10 слов на русском\",\"reject_tag\":null}]}\n\nRules:\n- Input is a JSON array of posts.\n- Return JSON only. No markdown. No text outside JSON.\n- score must be integer 1..10.\n- decision must be exactly one of: \"accepted\", \"rejected\", \"maybe\".\n- reject_tag (rejected/maybe only) must be one of: \"meme\", \"no_product\", \"politics\", \"discount_only\", \"off_topic\", \"vacancy\", \"low_content\", \"wrong_language\", \"injection\", \"weapon\", null.\n- Return one result for every input post id."'::jsonb, + 'text', + 'Технический контракт квалификатора', + 'JSON-схема и строгие правила ответа. Обычно не меняется при правке смыслового промпта.', + 'AI Qualifier' + ), + ( + 'ai_writer_contract', + '"OUTPUT SCHEMA (return array matching input order):\n{\"rewrites\":[{\"id\":123,\"category\":\"разгрузка\",\"text\":\"готовый текст без хэштегов\",\"notes\":\"короткая заметка для редактора или null\"}]}\n\nRules:\n- Input is a JSON object with key \"posts\" containing accepted posts.\n- Each post includes producer_name and producer_tag. Use producer_name when it helps, but do not invent facts.\n- If post includes editor_regeneration_prompt, follow it as an additional editor instruction while still obeying facts and output schema.\n- Do not add hashtags to rewritten text. Pick exactly one category from the categories list in input.\n- Return JSON only. No markdown. No text outside JSON.\n- Return one rewrite for every input post id."'::jsonb, + 'text', + 'Технический контракт райтера', + 'JSON-схема, выбор категории и запрет хэштегов в тексте. Обычно не меняется при правке стиля.', + 'AI Writer' + ) +ON CONFLICT (key) DO NOTHING; + +UPDATE app_settings +SET title='Свободная инструкция квалификатора', + description='Роль, тон, критерии и примеры оценки. Технический JSON-контракт вынесен отдельно.' +WHERE key='ai_qualifier_prompt'; + +UPDATE app_settings +SET title='Свободная инструкция райтера', + description='Стиль, голос канала, правила фактов и примеры текста. Технический JSON-контракт вынесен отдельно.' +WHERE key='ai_writer_prompt'; + +CREATE TABLE IF NOT EXISTS admin_login_attempts ( + id BIGSERIAL PRIMARY KEY, + ip TEXT NOT NULL, + login TEXT NOT NULL DEFAULT '', + success BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_admin_login_attempts_ip_time +ON admin_login_attempts(ip, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_admin_login_attempts_login_time +ON admin_login_attempts(login, created_at DESC); diff --git a/db/migrations/015_editor_media_flags.sql b/db/migrations/015_editor_media_flags.sql new file mode 100644 index 0000000..39654e8 --- /dev/null +++ b/db/migrations/015_editor_media_flags.sql @@ -0,0 +1,9 @@ +ALTER TABLE raw_post_media + ADD COLUMN IF NOT EXISTS editor_hidden BOOLEAN NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS editor_added BOOLEAN NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS editor_uploaded_by BIGINT REFERENCES admin_users(id) ON DELETE SET NULL, + ADD COLUMN IF NOT EXISTS editor_uploaded_at TIMESTAMPTZ; + +CREATE INDEX IF NOT EXISTS idx_raw_post_media_editor_visible +ON raw_post_media(raw_post_id, sort_order, id) +WHERE editor_hidden = FALSE; diff --git a/db/migrations/016_remove_regeneration_and_writer_prompt.sql b/db/migrations/016_remove_regeneration_and_writer_prompt.sql new file mode 100644 index 0000000..e60d43d --- /dev/null +++ b/db/migrations/016_remove_regeneration_and_writer_prompt.sql @@ -0,0 +1,56 @@ +UPDATE raw_posts +SET rewrite_status='ready', + editorial_status='review', + regeneration_prompt=NULL, + updated_at=NOW() +WHERE COALESCE(editorial_status, '')='regenerating'; + +UPDATE app_settings +SET value_json = to_jsonb($prompt$ +Ты редактор русского Telegram-канала про тактическую, милитари и airsoft-экипировку. +Пиши от лица канала: как экспертный наблюдатель, а не как производитель или магазин. + +Задача: превратить исходный пост в короткую новостную публикацию для канала. + +Стиль: +- коротко, уверенно, по делу; +- без рекламного восторга и без выдуманных характеристик; +- только факты из исходного текста; +- если фактов мало, делай короткий пост, не растягивай; +- текст должен быть структурированным, не простынёй. + +Формат текста: +- 3-6 коротких строк или 2-4 компактных абзаца; +- можно использовать 1-3 тематических эмоджи как маркеры структуры, например: 🧵 материал/конструкция, 🎒 переноска, 🛡 защита, ⚙️ детали, 📦 комплект/наличие, 🎯 назначение; +- эмоджи должны помогать читать текст, а не украшать каждую строку; +- не добавляй ссылки и хэштеги в text. + +Важное правило про источник: +- producer_name — это источник поста: производитель, магазин или площадка. +- Не утверждай, что producer_name «выпустил», «сделал», «представил» товар, если из текста не ясно, что это именно производитель этого товара. +- Если источник выглядит как магазин/площадка или в тексте упомянут другой бренд, формулируй нейтрально: «у [producer_name] появился...», «[producer_name] показали/добавили...», «в продаже появился...». +- Если из текста явно видно, что источник сам производит товар, можно писать: «[producer_name] выпустили...». + +Категория: +- выбери ровно одну category из списка categories во входных данных; +- если сомневаешься, выбирай ближайшую по назначению товара. + +OUTPUT: JSON only. +$prompt$::text), + updated_at = NOW() +WHERE key='ai_writer_prompt'; + +UPDATE app_settings +SET value_json = to_jsonb($contract$ +OUTPUT SCHEMA (return array matching input order): +{"rewrites":[{"id":123,"category":"разгрузка","text":"готовый текст без хэштегов","notes":"короткая заметка для редактора или null"}]} + +Rules: +- Input is a JSON object with key "posts" containing accepted posts. +- Each post includes producer_name and producer_tag. Use producer_name carefully: it can be a manufacturer, shop, or publishing source. +- Do not add hashtags to rewritten text. Pick exactly one category from the categories list in input. +- Return JSON only. No markdown. No text outside JSON. +- Return one rewrite for every input post id. +$contract$::text), + updated_at = NOW() +WHERE key='ai_writer_contract'; diff --git a/db/migrations/017_content_categories_and_media_cleanup.sql b/db/migrations/017_content_categories_and_media_cleanup.sql new file mode 100644 index 0000000..a213b81 --- /dev/null +++ b/db/migrations/017_content_categories_and_media_cleanup.sql @@ -0,0 +1,24 @@ +CREATE TABLE IF NOT EXISTS content_categories ( + id BIGSERIAL PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + tag TEXT NOT NULL, + is_active BOOLEAN NOT NULL DEFAULT TRUE, + sort_order INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_content_categories_active_order +ON content_categories(is_active, sort_order, name); + +INSERT INTO content_categories(name, tag, sort_order) +SELECT value, regexp_replace(lower(value), '[^[:alnum:]_а-яё]+', '_', 'g'), ordinality::int +FROM app_settings s +CROSS JOIN LATERAL jsonb_array_elements_text( + CASE WHEN jsonb_typeof(s.value_json) = 'array' THEN s.value_json ELSE '[]'::jsonb END +) WITH ORDINALITY AS items(value, ordinality) +WHERE s.key = 'ai_writer_categories' +ON CONFLICT (name) DO UPDATE +SET tag = EXCLUDED.tag, + sort_order = LEAST(content_categories.sort_order, EXCLUDED.sort_order), + updated_at = NOW(); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..72a1627 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +aiohttp==3.12.13 +aiogram==3.21.0 +asyncpg==0.30.0 +fastapi==0.115.14 +jinja2==3.1.6 +loguru==0.7.3 +litellm==1.76.0 +pydantic-settings==2.10.1 +python-multipart==0.0.20 +uvicorn[standard]==0.35.0 +yt-dlp==2026.6.9 diff --git a/scripts/apply_migrations.py b/scripts/apply_migrations.py new file mode 100644 index 0000000..1579e94 --- /dev/null +++ b/scripts/apply_migrations.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from vk_parser_app.db import apply_migrations, close_pool + + +async def main() -> None: + root = Path(__file__).resolve().parents[1] + await apply_migrations(root / "db" / "migrations") + await close_pool() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/vk_storage_spike.py b/scripts/vk_storage_spike.py new file mode 100644 index 0000000..a7ea48a --- /dev/null +++ b/scripts/vk_storage_spike.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from vk_parser_app.config import settings +from vk_parser_app.vk_api import VKAPIClient, post_vk_url + + +async def main() -> None: + if not settings.vk_access_token: + raise RuntimeError("VK_ACCESS_TOKEN is empty") + if not settings.vk_storage_group_id: + raise RuntimeError("VK_STORAGE_GROUP_ID is empty") + + async with VKAPIClient() as client: + attachments: list[str] = [] + sample = Path("sample_photo.jpg") + if sample.exists(): + saved = await client.upload_wall_photo_bytes( + abs(settings.vk_storage_group_id), + sample.read_bytes(), + filename="sample_photo.jpg", + ) + attachments.append(f"photo{saved['owner_id']}_{saved['id']}") + + post_id = await client.create_wall_post( + owner_id=settings.vk_storage_owner_id, + message="VK storage spike: test raw copy post", + attachments=attachments, + from_group=True, + ) + print(post_vk_url(settings.vk_storage_owner_id, post_id)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/vk_parser_app/__init__.py b/src/vk_parser_app/__init__.py new file mode 100644 index 0000000..a05eb9a --- /dev/null +++ b/src/vk_parser_app/__init__.py @@ -0,0 +1,3 @@ +__all__ = ["__version__"] + +__version__ = "0.1.0" diff --git a/src/vk_parser_app/admin.py b/src/vk_parser_app/admin.py new file mode 100644 index 0000000..477b834 --- /dev/null +++ b/src/vk_parser_app/admin.py @@ -0,0 +1,2494 @@ +from __future__ import annotations + +import json +import base64 +import hashlib +import mimetypes +import re +import secrets +import time +from datetime import date, datetime, timedelta, timezone +from pathlib import Path +from typing import Any +from urllib.parse import urlencode +from zoneinfo import ZoneInfo + +import aiohttp +from fastapi import FastAPI, File, Form, Request, UploadFile, status +from fastapi.responses import HTMLResponse, RedirectResponse +from fastapi.staticfiles import StaticFiles +from fastapi.templating import Jinja2Templates +from loguru import logger + +from .config import settings +from .constants import PLATFORM_VK +from .db import fetch_setting, get_pool +from .security import hash_password, new_token, token_hash, verify_password +from .text_utils import normalize_hash_tag, parse_categories +from .vk_api import VKAPIClient, normalize_vk_source + +COOKIE_NAME = "vk_parser_admin" +VK_OAUTH_VERIFIER_COOKIE = "vk_oauth_verifier" +VK_OAUTH_STATE_COOKIE = "vk_oauth_state" +BASE_DIR = Path(__file__).resolve().parent +templates = Jinja2Templates(directory=str(BASE_DIR / "templates")) +app = FastAPI(title="VK Parser Admin") +LOCAL_TZ = ZoneInfo("Asia/Yekaterinburg") +MODEL_CACHE: dict[str, Any] = {"key": "", "at": 0.0, "models": []} +UPLOAD_ROOT = Path("uploads").resolve() +EDITOR_MEDIA_DIR = UPLOAD_ROOT / "editor_media" +EDITOR_MEDIA_DIR.mkdir(parents=True, exist_ok=True) +app.mount("/uploads", StaticFiles(directory=str(UPLOAD_ROOT)), name="uploads") + +PROVIDER_OPTIONS = [ + {"value": "openrouter", "label": "OpenRouter"}, + {"value": "openai", "label": "OpenAI"}, + {"value": "anthropic", "label": "Anthropic"}, + {"value": "gemini", "label": "Google Gemini"}, + {"value": "openai_compatible", "label": "OpenAI-compatible API"}, +] + +PROMPT_HINTS = { + "ai_qualifier_prompt": { + "title": "Свободная инструкция квалификатора", + "body": ( + "Здесь можно спокойно менять смысл: роль, тон, критерии пригодности, примеры хороших и плохих постов, " + "логику оценки по шкале 1-10." + ), + "safe": ( + "Можно менять: критерии, примеры, тон причины, пороговые объяснения. " + "Не добавляй сюда JSON-схему: для неё есть отдельный технический контракт ниже." + ), + "input": "На вход уходит JSON-массив постов: id, source, original_url, media_count, media_types, text.", + }, + "ai_qualifier_contract": { + "title": "Технический контракт квалификатора", + "body": ( + "Эта часть защищает парсер от сломанного ответа. Меняй её только если осознанно меняешь формат ответа " + "и одновременно готов править валидатор в коде." + ), + "safe": "Обычно не трогаем. Здесь живут JSON-схема, допустимые decision/reject_tag и правило вернуть результат на каждый id.", + "input": "Контракт склеивается после свободной инструкции и отправляется как system prompt.", + "contract": '{"results":[{"id":123,"score":8,"decision":"accepted","reason":"до 10 слов на русском","reject_tag":null}]}', + }, + "ai_writer_prompt": { + "title": "Свободная инструкция райтера", + "body": ( + "Здесь можно менять редакторскую часть: стиль, длину, голос канала, запреты на выдумки, примеры хорошего текста." + ), + "safe": ( + "Можно менять: tone of voice, правила фактов, длину, примеры фраз. " + "Хэштеги руками писать не надо: модель выбирает category, а код сам соберёт #category и #producer_tag." + ), + "input": "На вход уходит JSON-объект: categories и posts. В каждом post есть id, producer_name, producer_tag, original_url, qualification_score, media_count, media_types, text.", + }, + "ai_writer_contract": { + "title": "Технический контракт райтера", + "body": ( + "Эта часть фиксирует формат JSON, запрет хэштегов в тексте и выбор категории. " + "Менять её стоит только вместе с валидатором райтера." + ), + "safe": "Обычно не трогаем. Если промпт райтера переписывается, свободную часть меняем выше, контракт оставляем стабильным.", + "input": "Контракт склеивается после свободной инструкции и отправляется как system prompt.", + "contract": '{"rewrites":[{"id":123,"category":"разгрузка","text":"готовый текст без хэштегов","notes":"короткая заметка для редактора"}]}', + }, +} + +MODEL_FALLBACKS = { + "openrouter": [ + "openrouter/openai/gpt-4.1-mini", + "openrouter/anthropic/claude-3.5-sonnet", + "openrouter/google/gemini-2.5-flash", + ], + "openai": ["gpt-4.1-mini", "gpt-4o-mini", "o4-mini"], + "anthropic": ["anthropic/claude-haiku-4-5-20251001", "anthropic/claude-sonnet-4-20250514"], + "gemini": ["gemini/gemini-2.5-flash", "gemini/gemini-2.5-pro"], + "openai_compatible": [], +} + +PREFERRED_MODELS = { + "openrouter": [ + "openrouter/anthropic/claude-sonnet-4", + "openrouter/anthropic/claude-3.5-sonnet", + "openrouter/openai/gpt-4.1-mini", + "openrouter/google/gemini-2.5-flash", + ], + "anthropic": [ + "anthropic/claude-haiku-4-5-20251001", + "anthropic/claude-4-sonnet-20250514", + "anthropic/claude-sonnet-4-20250514", + "anthropic/claude-3-7-sonnet-20250219", + "anthropic/claude-3-5-sonnet-20241022", + "anthropic/claude-3-5-haiku-20241022", + ], + "openai": ["gpt-4.1-mini", "gpt-4o-mini", "o4-mini"], + "gemini": ["gemini/gemini-2.5-flash", "gemini/gemini-2.5-pro"], +} + +CATEGORY_TITLES = { + "AI Qualifier": "AI-квалификатор", + "AI Writer": "AI-райтер", + "Parser": "Парсер", + "Uploader": "Аплоадер", + "VK": "VK API", + "General": "Общие", +} + +CATEGORY_ORDER = { + "AI Qualifier": 10, + "AI Writer": 20, + "Parser": 30, + "VK": 40, + "Uploader": 50, + "General": 100, +} + +SETTING_ORDER = { + "AI Qualifier": [ + "ai_qualifier_enabled", + "ai_qualifier_provider", + "ai_qualifier_model", + "ai_qualifier_api_key", + "ai_qualifier_api_base", + "ai_qualifier_prompt", + "ai_qualifier_contract", + "ai_qualifier_batch_size", + "ai_qualifier_min_score", + "ai_qualifier_max_text_chars", + "ai_qualifier_temperature", + "ai_qualifier_timeout_sec", + "ai_qualifier_interval_sec", + ], + "AI Writer": [ + "ai_writer_enabled", + "ai_writer_provider", + "ai_writer_model", + "ai_writer_api_key", + "ai_writer_api_base", + "ai_writer_prompt", + "ai_writer_contract", + "ai_writer_categories", + "ai_writer_batch_size", + "ai_writer_max_text_chars", + "ai_writer_temperature", + "ai_writer_timeout_sec", + "ai_writer_interval_sec", + ], + "Parser": [ + "parser_interval_sec", + "parser_new_source_lookback_days", + "parser_reparse_overlap_minutes", + "parser_min_text_length", + "parser_skip_empty_text", + "parser_skip_no_media", + "parser_skip_text_too_short", + "parser_skip_reposts", + "parser_store_skipped_posts", + "parser_dedupe_content_hash", + "parser_source_pause_sec", + ], + "VK": [ + "vk_requests_per_second", + "vk_wall_page_size", + "vk_api_timeout_total_sec", + "vk_api_timeout_connect_sec", + "vk_rate_limit_sleep_sec", + "vk_api_retry_attempts", + "vk_api_retry_min_delay_sec", + "vk_api_retry_max_delay_sec", + ], + "Uploader": [ + "tg_media_channel_id", + "local_bot_api_url", + "uploader_interval_sec", + "uploader_download_timeout_sec", + "media_group_max_items", + "media_upload_delay_sec", + "media_post_job_pause_sec", + "max_media_attempts", + "tg_retry_attempts", + "tg_retry_backoff_max_sec", + "video_max_size_mb", + "video_max_duration_sec", + "uploader_yt_dlp_timeout_sec", + ], +} + + +def now_utc() -> datetime: + return datetime.now(timezone.utc) + + +def redirect(path: str) -> RedirectResponse: + return RedirectResponse(path, status_code=status.HTTP_303_SEE_OTHER) + + +@app.middleware("http") +async def add_security_headers(request: Request, call_next): + response = await call_next(request) + response.headers.setdefault("X-Frame-Options", "DENY") + response.headers.setdefault("X-Content-Type-Options", "nosniff") + response.headers.setdefault("Referrer-Policy", "same-origin") + response.headers.setdefault("Permissions-Policy", "geolocation=(), microphone=(), camera=()") + return response + + +def pkce_challenge(verifier: str) -> str: + digest = hashlib.sha256(verifier.encode("ascii")).digest() + return base64.urlsafe_b64encode(digest).decode("ascii").rstrip("=") + + +async def bootstrap_admin() -> None: + if not settings.admin_bootstrap_login or not settings.admin_bootstrap_password: + return + pool = await get_pool() + exists = await pool.fetchval("SELECT 1 FROM admin_users WHERE login=$1", settings.admin_bootstrap_login) + if exists: + return + await pool.execute( + """ + INSERT INTO admin_users(login, password_hash, role, is_active) + VALUES($1, $2, 'admin', TRUE) + """, + settings.admin_bootstrap_login.strip().lower(), + hash_password(settings.admin_bootstrap_password), + ) + logger.warning("Bootstrap admin user created: {}", settings.admin_bootstrap_login) + + +async def get_current_user(request: Request) -> dict | None: + token = request.cookies.get(COOKIE_NAME) + if not token: + return None + pool = await get_pool() + row = await pool.fetchrow( + """ + SELECT s.csrf_token, u.id, u.login, u.role, u.is_active + FROM admin_sessions s + JOIN admin_users u ON u.id=s.user_id + WHERE s.token_hash=$1 + AND s.expires_at > NOW() + AND u.is_active=TRUE + """, + token_hash(token, settings.app_secret_key), + ) + return dict(row) if row else None + + +def require_csrf(user: dict, csrf_token: str) -> None: + if not user or csrf_token != user["csrf_token"]: + raise PermissionError("bad csrf") + + +def base_context(request: Request, user: dict | None, **extra: Any) -> dict[str, Any]: + ctx = {"request": request, "user": user, "app_env": settings.app_env} + ctx.update(extra) + return ctx + + +def client_ip(request: Request) -> str: + forwarded = request.headers.get("x-forwarded-for", "") + if forwarded: + return forwarded.split(",", 1)[0].strip()[:80] + real_ip = request.headers.get("x-real-ip", "") + if real_ip: + return real_ip.strip()[:80] + return (request.client.host if request.client else "unknown")[:80] + + +def preserved_query(request: Request, exclude: set[str] | None = None) -> list[dict[str, str]]: + exclude = exclude or {"page"} + items = [] + for key, value in request.query_params.multi_items(): + if key not in exclude: + items.append({"key": key, "value": value}) + return items + + +def query_path(request: Request, **replace: str) -> str: + values: list[tuple[str, str]] = [] + excluded = set(replace) | {"page"} + for key, value in request.query_params.multi_items(): + if key not in excluded: + values.append((key, value)) + for key, value in replace.items(): + if value != "": + values.append((key, value)) + encoded = urlencode(values) + return f"{request.url.path}?{encoded}" if encoded else request.url.path + + +def raw_sort_headers(request: Request, current_sort: str) -> dict[str, dict[str, str | bool]]: + columns = { + "id": ("id_asc", "id_desc"), + "source": ("source_asc", "source_desc"), + "stage": ("stage_asc", "stage_desc"), + "post": ("text_asc", "text_desc"), + "ai": ("score_asc", "score_desc"), + } + headers: dict[str, dict[str, str | bool]] = {} + for key, (asc, desc) in columns.items(): + next_sort = desc if current_sort == asc else asc + headers[key] = { + "url": query_path(request, sort=next_sort), + "active": current_sort in {asc, desc}, + "direction": "asc" if current_sort == asc else "desc" if current_sort == desc else "", + } + return headers + + +def parse_date_filter(value: str) -> date | None: + value = (value or "").strip() + if not value: + return None + try: + return datetime.strptime(value, "%Y-%m-%d").date() + except ValueError: + return None + + +def format_dt(value: datetime | None) -> str: + if not value: + return "" + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(LOCAL_TZ).strftime("%d.%m.%y %H:%M") + + +def clamp_int(value: int, min_value: int, max_value: int) -> int: + return max(min_value, min(max_value, int(value))) + + +def pagination(page: int, per_page: int, total: int) -> dict[str, int | bool]: + page = clamp_int(page, 1, 1_000_000) + per_page = clamp_int(per_page, 10, 200) + pages = max(1, (int(total) + per_page - 1) // per_page) + page = min(page, pages) + return { + "page": page, + "per_page": per_page, + "total": int(total), + "pages": pages, + "offset": (page - 1) * per_page, + "has_prev": page > 1, + "has_next": page < pages, + } + + +FILTER_OPS = { + "text": [ + {"value": "contains", "label": "содержит"}, + {"value": "eq", "label": "равно"}, + {"value": "neq", "label": "не равно"}, + ], + "enum": [ + {"value": "eq", "label": "равно"}, + {"value": "neq", "label": "не равно"}, + ], + "number": [ + {"value": "eq", "label": "="}, + {"value": "gte", "label": ">="}, + {"value": "lte", "label": "<="}, + ], + "date": [ + {"value": "gte", "label": "от"}, + {"value": "lte", "label": "до"}, + {"value": "eq", "label": "день"}, + ], +} +FILTER_OP_OPTIONS = [ + {"value": "contains", "label": "содержит"}, + {"value": "eq", "label": "равно / день"}, + {"value": "neq", "label": "не равно"}, + {"value": "gte", "label": ">= / от"}, + {"value": "lte", "label": "<= / до"}, +] + +SOURCE_FIELD_SPECS = { + "id": {"label": "ID", "expr": "s.id", "type": "number"}, + "platform": {"label": "Площадка", "expr": "s.platform", "type": "enum"}, + "name": {"label": "Название", "expr": "s.name", "type": "text"}, + "tag": {"label": "Тэг", "expr": "COALESCE(s.tag,'')", "type": "text"}, + "url": {"label": "Ссылка", "expr": "s.url", "type": "text"}, + "external_id": {"label": "VK id", "expr": "COALESCE(s.external_id,'')", "type": "text"}, + "active": {"label": "Включён", "expr": "s.active", "type": "enum"}, + "status": {"label": "Статус", "expr": "s.status", "type": "enum"}, + "posts_count": { + "label": "Постов", + "expr": "(SELECT COUNT(*) FROM raw_posts rp2 WHERE rp2.source_id=s.id)", + "type": "number", + }, + "posts_24h": { + "label": "Постов за 24ч", + "expr": "(SELECT COUNT(*) FROM raw_posts rp3 WHERE rp3.source_id=s.id AND rp3.created_at > NOW() - INTERVAL '24 hours')", + "type": "number", + }, + "last_parsed_at": {"label": "Последний парсинг", "expr": "timezone('Asia/Yekaterinburg', s.last_parsed_at)::date", "sort_expr": "s.last_parsed_at", "type": "date"}, + "created_at": {"label": "Создан", "expr": "timezone('Asia/Yekaterinburg', s.created_at)::date", "sort_expr": "s.created_at", "type": "date"}, +} + +RAW_FIELD_SPECS = { + "id": {"label": "ID", "expr": "rp.id", "type": "number"}, + "source_name": {"label": "Источник", "expr": "s.name", "type": "text"}, + "source_tag": {"label": "Тэг источника", "expr": "COALESCE(s.tag,'')", "type": "text"}, + "platform": {"label": "Площадка", "expr": "rp.platform", "type": "enum"}, + "status": {"label": "Raw статус", "expr": "rp.status", "type": "enum"}, + "original_url": {"label": "Оригинал", "expr": "rp.original_url", "type": "text"}, + "text": {"label": "Текст", "expr": "rp.raw_text", "type": "text"}, + "media_count": { + "label": "Медиа", + "expr": "(SELECT COUNT(*) FROM raw_post_media rmf WHERE rmf.raw_post_id=rp.id)", + "type": "number", + }, + "posted_at": {"label": "Дата VK", "expr": "timezone('Asia/Yekaterinburg', rp.posted_at)::date", "sort_expr": "rp.posted_at", "type": "date"}, + "created_at": {"label": "Дата загрузки", "expr": "timezone('Asia/Yekaterinburg', rp.created_at)::date", "sort_expr": "rp.created_at", "type": "date"}, + "qualification_status": {"label": "Итог AI", "expr": "COALESCE(rp.qualification_status,'pending')", "type": "enum"}, + "qualification_model_decision": {"label": "Решение модели", "expr": "COALESCE(rp.qualification_model_decision, rp.qualification_decision, '')", "type": "enum"}, + "qualification_score": {"label": "Оценка", "expr": "rp.qualification_score", "type": "number"}, + "qualification_reject_tag": {"label": "Reject tag", "expr": "COALESCE(rp.qualification_reject_tag,'')", "type": "enum"}, + "rewrite_status": {"label": "Статус рерайта", "expr": "COALESCE(rp.rewrite_status,'pending')", "type": "enum"}, + "rewrite_category": {"label": "Категория", "expr": "COALESCE(rp.rewrite_category,'')", "type": "enum"}, + "rewrite_source_tag": {"label": "Хэштег источника", "expr": "COALESCE(rp.rewrite_source_tag,'')", "type": "text"}, +} + +EDITOR_STATUS_OPTIONS = [ + {"value": "review", "label": "На проверке"}, + {"value": "edited", "label": "Отредактировано"}, + {"value": "accepted", "label": "Принято"}, + {"value": "rejected", "label": "Отклонено"}, +] + + +def field_options(specs: dict[str, dict[str, str]]) -> list[dict[str, str]]: + return [{"value": key, "label": spec["label"], "type": spec["type"]} for key, spec in specs.items()] + + +def parse_table_filters(request: Request, specs: dict[str, dict[str, str]], slots: int = 4) -> list[dict[str, str]]: + qp = request.query_params + fields = qp.getlist("f_field") + ops = qp.getlist("f_op") + values = qp.getlist("f_value") + rows: list[dict[str, str]] = [] + for i in range(max(slots, len(fields))): + field = fields[i] if i < len(fields) else "" + op = ops[i] if i < len(ops) else "" + value = values[i] if i < len(values) else "" + if field not in specs: + field = "" + field_type = specs[field]["type"] if field else "text" + allowed_ops = {item["value"] for item in FILTER_OPS[field_type]} + if op not in allowed_ops: + op = "contains" if field_type == "text" else "eq" + rows.append({"field": field, "op": op, "value": value}) + return rows + + +def parse_table_sorts(request: Request, specs: dict[str, dict[str, str]], slots: int = 3) -> list[dict[str, str]]: + qp = request.query_params + fields = qp.getlist("sort_field") + dirs = qp.getlist("sort_dir") + rows: list[dict[str, str]] = [] + for i in range(max(slots, len(fields))): + field = fields[i] if i < len(fields) else "" + direction = dirs[i] if i < len(dirs) else "desc" + if field not in specs: + field = "" + if direction not in {"asc", "desc"}: + direction = "desc" + rows.append({"field": field, "dir": direction}) + return rows + + +def build_filter_sql(filters: list[dict[str, str]], specs: dict[str, dict[str, str]], start_index: int = 1) -> tuple[list[str], list[Any]]: + clauses: list[str] = [] + args: list[Any] = [] + index = start_index + for item in filters: + field = item.get("field") or "" + value = str(item.get("value") or "").strip() + if not field or field not in specs or value == "": + continue + spec = specs[field] + expr = spec["expr"] + field_type = spec["type"] + op = item.get("op") or ("contains" if field_type == "text" else "eq") + if field_type == "text": + if op == "eq": + clauses.append(f"{expr} = ${index}") + args.append(value) + elif op == "neq": + clauses.append(f"{expr} <> ${index}") + args.append(value) + else: + clauses.append(f"{expr} ILIKE '%' || ${index} || '%'") + args.append(value) + elif field_type == "enum": + if op == "neq": + clauses.append(f"{expr} <> ${index}") + else: + clauses.append(f"{expr} = ${index}") + if value.lower() in {"true", "false"}: + args.append(value.lower() == "true") + else: + args.append(value) + elif field_type == "number": + try: + number_value = int(value) + except ValueError: + continue + sign = {"gte": ">=", "lte": "<=", "neq": "<>", "eq": "="}.get(op, "=") + clauses.append(f"{expr} {sign} ${index}") + args.append(number_value) + elif field_type == "date": + sign = {"gte": ">=", "lte": "<=", "eq": "="}.get(op, "=") + clauses.append(f"{expr} {sign} ${index}::date") + args.append(value) + index += 1 + return clauses, args + + +def build_sort_sql(sorts: list[dict[str, str]], specs: dict[str, dict[str, str]], default_sql: str) -> str: + parts: list[str] = [] + seen: set[str] = set() + for item in sorts: + field = item.get("field") or "" + if not field or field not in specs or field in seen: + continue + spec = specs[field] + expr = spec.get("sort_expr") or spec["expr"] + direction = "ASC" if item.get("dir") == "asc" else "DESC" + nulls = " NULLS LAST" if direction == "DESC" else " NULLS FIRST" + parts.append(f"{expr} {direction}{nulls}") + seen.add(field) + if parts: + if "id" not in seen and "id" in specs: + parts.append(f"{specs['id'].get('sort_expr') or specs['id']['expr']} DESC") + return ", ".join(part for part in parts if part) + return default_sql + + +def selected_values(request: Request, name: str) -> list[str]: + return [str(value).strip() for value in request.query_params.getlist(name) if str(value).strip()] + + +def selected_int_values(request: Request, name: str) -> list[int]: + values: list[int] = [] + for value in selected_values(request, name): + try: + values.append(int(value)) + except ValueError: + continue + return values + + +def add_where(clauses: list[str], args: list[Any], sql_template: str, value: Any) -> None: + args.append(value) + clauses.append(sql_template.format(i=len(args))) + + +async def raw_filter_facets(pool) -> dict[str, Any]: + source_rows = await pool.fetch( + """ + SELECT s.id, s.name, COALESCE(s.tag, '') AS tag, COUNT(rp.id) AS count + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + GROUP BY s.id, s.name, s.tag + ORDER BY COUNT(rp.id) DESC, s.name ASC + """ + ) + category_rows = await pool.fetch( + """ + SELECT COALESCE(final_category, rewrite_category, '') AS value, COUNT(*) AS count + FROM raw_posts + WHERE COALESCE(final_category, rewrite_category, '') <> '' + GROUP BY COALESCE(final_category, rewrite_category, '') + ORDER BY COUNT(*) DESC, value ASC + """ + ) + status_rows = await pool.fetch("SELECT status AS value, COUNT(*) AS count FROM raw_posts GROUP BY status ORDER BY count DESC, value") + qualification_rows = await pool.fetch( + """ + SELECT COALESCE(qualification_status, 'pending') AS value, COUNT(*) AS count + FROM raw_posts + GROUP BY COALESCE(qualification_status, 'pending') + ORDER BY count DESC, value + """ + ) + rewrite_rows = await pool.fetch( + """ + SELECT COALESCE(rewrite_status, 'pending') AS value, COUNT(*) AS count + FROM raw_posts + GROUP BY COALESCE(rewrite_status, 'pending') + ORDER BY count DESC, value + """ + ) + return { + "sources": [dict(row) for row in source_rows], + "categories": [dict(row) for row in category_rows], + "statuses": [dict(row) for row in status_rows], + "qualification_statuses": [dict(row) for row in qualification_rows], + "rewrite_statuses": [dict(row) for row in rewrite_rows], + } + + +async def editor_filter_facets(pool) -> dict[str, Any]: + source_rows = await pool.fetch( + """ + SELECT s.id, s.name, COALESCE(s.tag, '') AS tag, COUNT(rp.id) AS count + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + WHERE rp.rewrite_status='ready' + GROUP BY s.id, s.name, s.tag + ORDER BY COUNT(rp.id) DESC, s.name ASC + """ + ) + category_rows = await pool.fetch( + """ + SELECT COALESCE(final_category, rewrite_category, '') AS value, COUNT(*) AS count + FROM raw_posts + WHERE rewrite_status='ready' + AND COALESCE(final_category, rewrite_category, '') <> '' + GROUP BY COALESCE(final_category, rewrite_category, '') + ORDER BY COUNT(*) DESC, value ASC + """ + ) + status_rows = await pool.fetch( + """ + SELECT COALESCE(editorial_status, 'review') AS value, COUNT(*) AS count + FROM raw_posts + WHERE rewrite_status='ready' + GROUP BY COALESCE(editorial_status, 'review') + ORDER BY count DESC, value + """ + ) + return { + "sources": [dict(row) for row in source_rows], + "categories": [dict(row) for row in category_rows], + "statuses": [dict(row) for row in status_rows], + } + + +def parse_source_line(line: str) -> dict[str, str]: + raw = (line or "").strip() + if not raw: + return {"name": "", "tag": "", "url": "", "error": "Пустая строка"} + parts = re.split(r"\s+", raw) + url_index = next((i for i, part in enumerate(parts) if "vk.com/" in part or part.startswith("club")), -1) + if url_index < 0: + if len(parts) == 1: + value = parts[0].strip() + url = value if value.startswith("http") else f"https://vk.com/{value}" + return {"name": "", "tag": "", "url": url, "error": ""} + return {"name": "", "tag": "", "url": "", "error": "Не нашёл ссылку VK"} + url = parts[url_index].strip() + before_url = parts[:url_index] + if len(before_url) >= 2: + tag = before_url[-1].strip() + name = " ".join(before_url[:-1]).strip() + elif len(before_url) == 1: + name = before_url[0].strip() + tag = before_url[0].strip() + else: + name = "" + tag = "" + if url.startswith("vk.com/"): + url = f"https://{url}" + return {"name": name, "tag": normalize_hash_tag(tag, ""), "url": url, "error": ""} + + +async def build_sources_preview(lines_text: str, default_active: bool = True) -> list[dict[str, Any]]: + lines = [line for line in (lines_text or "").splitlines() if line.strip()] + parsed = [parse_source_line(line) for line in lines] + normalized_values = [normalize_vk_source(item["url"]) for item in parsed if item.get("url")] + pool = await get_pool() + existing_rows = await pool.fetch( + """ + SELECT lower(external_id) AS external_id, lower(url) AS url, lower(COALESCE(tag,'')) AS tag + FROM sources + WHERE archived_at IS NULL + AND (lower(external_id)=ANY($1::text[]) OR lower(url)=ANY($2::text[]) OR lower(COALESCE(tag,''))=ANY($3::text[])) + """, + [v.lower() for v in normalized_values if v], + [str(item.get("url") or "").lower() for item in parsed], + [normalize_hash_tag(str(item.get("tag") or ""), "").lower() for item in parsed if item.get("tag")], + ) + existing_ids = {str(row["external_id"] or "").lower() for row in existing_rows} + existing_urls = {str(row["url"] or "").lower() for row in existing_rows} + existing_tags = {str(row["tag"] or "").lower() for row in existing_rows} + + seen: set[str] = set() + seen_tags: set[str] = set() + async with VKAPIClient(rps=2, timeout_total_sec=12, timeout_connect_sec=5, retry_attempts=2) as client: + preview = [] + for idx, item in enumerate(parsed, start=1): + url = item.get("url") or "" + external_id = normalize_vk_source(url) if url else "" + row = { + "line_no": idx, + "platform": PLATFORM_VK, + "name": item.get("name") or "", + "tag": normalize_hash_tag(item.get("tag") or external_id, external_id or "source"), + "url": url, + "external_id": external_id, + "external_owner_id": None, + "active": default_active, + "ok": False, + "error": item.get("error") or "", + } + key = external_id.lower() + if not row["error"] and key in seen: + row["error"] = "Дубль в этом списке" + if not row["error"] and row["tag"].lower() in seen_tags: + row["error"] = "Дубль тэга в этом списке" + if not row["error"] and (key in existing_ids or url.lower() in existing_urls): + row["error"] = "Уже есть в источниках" + if not row["error"] and row["tag"].lower() in existing_tags: + row["error"] = "Такой тэг уже есть" + if not row["error"] and not external_id: + row["error"] = "Не удалось разобрать VK-ссылку" + if not row["error"]: + try: + screen_name, owner_id, resolved_name = await client.resolve_group(url) + row["external_id"] = screen_name + row["external_owner_id"] = owner_id + row["name"] = row["name"] or resolved_name + row["tag"] = normalize_hash_tag(row["tag"] or screen_name, screen_name) + row["url"] = f"https://vk.com/{screen_name}" + row["ok"] = True + seen.add(key) + seen_tags.add(row["tag"].lower()) + except Exception as exc: + row["error"] = str(exc) + preview.append(row) + return preview + + +def pipeline_stage(post: dict[str, Any]) -> dict[str, str]: + status = str(post.get("status") or "") + qualification_status = str(post.get("qualification_status") or "").strip().lower() + if status == "failed": + return {"key": "failed", "label": "Ошибка", "class": "bad"} + if status == "skipped": + return {"key": "skipped", "label": "Пропущен", "class": "muted"} + if status in {"raw_saved", "storage_pending"}: + return {"key": status, "label": "Медиа", "class": ""} + if status != "storage_ready": + return {"key": status, "label": status or "Raw", "class": ""} + rewrite_status = str(post.get("rewrite_status") or "").strip().lower() + if rewrite_status == "ready": + return {"key": "rewrite_ready", "label": "Рерайт готов", "class": "ok"} + if rewrite_status == "processing": + return {"key": "rewrite_processing", "label": "AI пишет", "class": "warn"} + if rewrite_status == "failed": + return {"key": "rewrite_failed", "label": "Рерайт ошибка", "class": "bad"} + if qualification_status in {"accepted"}: + return {"key": "qualified_accepted", "label": "AI принят", "class": "ok"} + if qualification_status in {"rejected"}: + return {"key": "qualified_rejected", "label": "AI отклонён", "class": "bad"} + if qualification_status == "processing": + return {"key": "qualification_processing", "label": "AI проверяет", "class": "warn"} + if qualification_status == "failed": + return {"key": "qualification_failed", "label": "AI ошибка", "class": "bad"} + return {"key": "qualification_pending", "label": "Ждёт AI", "class": ""} + + +def ai_badge(post: dict[str, Any]) -> dict[str, str]: + status = str(post.get("qualification_status") or "pending").strip().lower() + score = post.get("qualification_score") + decision = str(post.get("qualification_decision") or status or "pending") + reason = str(post.get("qualification_reason") or "") + if isinstance(score, int): + if score >= 8: + css = "ok" + elif score >= 5: + css = "warn" + else: + css = "bad" + label = f"{score}/10" + sublabel = decision + elif status == "processing": + css = "warn" + label = "..." + sublabel = "processing" + elif status == "failed": + css = "bad" + label = "!" + sublabel = "failed" + else: + css = "" + label = "AI" + sublabel = "pending" + return {"class": css, "label": label, "sublabel": sublabel, "reason": reason} + + +def prepare_raw_post(row: Any) -> dict[str, Any]: + post = dict(row) + media_items = post.get("media_items") or [] + if isinstance(media_items, str): + try: + media_items = json.loads(media_items) + except json.JSONDecodeError: + media_items = [] + post["media_items"] = media_items + post["posted_at_fmt"] = format_dt(post.get("posted_at")) + post["created_at_fmt"] = format_dt(post.get("created_at")) + post["qualified_at_fmt"] = format_dt(post.get("qualified_at")) + post["stage"] = pipeline_stage(post) + post["ai_badge"] = ai_badge(post) + return post + + +def prepare_editor_post(row: Any) -> dict[str, Any]: + post = prepare_raw_post(row) + post["review_text"] = post.get("final_text") or post.get("rewritten_text") or "" + post["review_category"] = post.get("final_category") or post.get("rewrite_category") or "" + post["review_category_tag"] = post.get("final_category_tag") or post.get("rewrite_category_tag") or "" + post["review_source_tag"] = post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "" + post["editorial_status_label"] = next( + (item["label"] for item in EDITOR_STATUS_OPTIONS if item["value"] == (post.get("editorial_status") or "review")), + post.get("editorial_status") or "review", + ) + post["reviewed_at_fmt"] = format_dt(post.get("reviewed_at")) + post["edited_at_fmt"] = format_dt(post.get("edited_at")) + return post + + +def uploaded_editor_media_path(url: str) -> Path | None: + raw = str(url or "").strip() + prefix = "/uploads/editor_media/" + if not raw.startswith(prefix): + return None + name = Path(raw.removeprefix(prefix)).name + if not name: + return None + path = (EDITOR_MEDIA_DIR / name).resolve() + try: + path.relative_to(EDITOR_MEDIA_DIR.resolve()) + except ValueError: + return None + return path + + +async def writer_categories() -> list[str]: + pool = await get_pool() + rows = await pool.fetch( + """ + SELECT name + FROM content_categories + WHERE is_active=TRUE + ORDER BY sort_order, name + """ + ) + categories = [str(row["name"]) for row in rows] + if categories: + return categories + legacy_categories = parse_categories(await fetch_setting("ai_writer_categories", [])) + return legacy_categories or [ + "защита", + "одежда", + "разгрузка", + "рюкзаки", + "airsoft", + "патчи", + "электроника", + "аксессуары", + "производство", + ] + + +async def category_rows() -> list[dict[str, Any]]: + pool = await get_pool() + rows = await pool.fetch( + """ + SELECT id, name, tag, is_active, sort_order, created_at, updated_at + FROM content_categories + ORDER BY is_active DESC, sort_order, name + """ + ) + result = [] + for row in rows: + item = dict(row) + item["created_at_fmt"] = format_dt(item.get("created_at")) + item["updated_at_fmt"] = format_dt(item.get("updated_at")) + result.append(item) + return result + + +def setting_value(settings_rows: list[dict], key: str, default: Any = None) -> Any: + for row in settings_rows: + if row.get("key") == key: + return row.get("value_json", default) + return default + + +def setting_sort_key(item: dict) -> tuple[int, int, str]: + category = str(item.get("category") or "General") + key = str(item.get("key") or "") + keys = SETTING_ORDER.get(category, []) + pos = keys.index(key) if key in keys else 1000 + return (CATEGORY_ORDER.get(category, 999), pos, key) + + +def model_label(model_id: str, name: str | None = None, context: int | None = None) -> str: + label = name or model_id + if context: + label = f"{label} · ctx {context}" + return label + + +async def fetch_openrouter_models() -> list[dict[str, str]]: + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=8)) as session: + async with session.get("https://openrouter.ai/api/v1/models") as resp: + resp.raise_for_status() + data = await resp.json() + models = [] + for item in data.get("data", []): + model_id = str(item.get("id") or "").strip() + if not model_id: + continue + context = item.get("context_length") + models.append( + { + "value": f"openrouter/{model_id}", + "label": model_label(model_id, item.get("name"), context if isinstance(context, int) else None), + } + ) + return models + + +async def fetch_anthropic_models(api_key: str) -> list[dict[str, str]]: + if not api_key: + return [] + headers = { + "x-api-key": api_key, + "anthropic-version": "2023-06-01", + } + models: list[dict[str, str]] = [] + url = "https://api.anthropic.com/v1/models" + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=10)) as session: + for _ in range(5): + async with session.get(url, headers=headers) as resp: + resp.raise_for_status() + data = await resp.json() + for item in data.get("data", []): + model_id = str(item.get("id") or "").strip() + if not model_id: + continue + display_name = item.get("display_name") or model_id + models.append({"value": f"anthropic/{model_id}", "label": f"{display_name} · live Anthropic"}) + if not data.get("has_more"): + break + last_id = data.get("last_id") + if not last_id: + break + url = f"https://api.anthropic.com/v1/models?after_id={last_id}" + return models + + +async def fetch_openai_models(api_key: str) -> list[dict[str, str]]: + if not api_key: + return [] + headers = {"Authorization": f"Bearer {api_key}"} + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=10)) as session: + async with session.get("https://api.openai.com/v1/models", headers=headers) as resp: + resp.raise_for_status() + data = await resp.json() + models = [] + for item in data.get("data", []): + model_id = str(item.get("id") or "").strip() + if not model_id: + continue + if model_id.startswith(("gpt-", "o1", "o3", "o4")): + models.append({"value": model_id, "label": f"{model_id} · live OpenAI"}) + return sorted(models, key=lambda item: item["value"]) + + +def litellm_catalog_models(provider: str) -> list[dict[str, str]]: + try: + import litellm + except Exception: + return [] + + catalog = getattr(litellm, "model_cost", {}) or {} + models: list[dict[str, str]] = [] + for model_id, meta in catalog.items(): + if not isinstance(model_id, str): + continue + normalized = model_id + if provider == "openai": + if "/" in model_id or not model_id.startswith(("gpt-", "o1", "o3", "o4")): + continue + elif provider == "anthropic": + if model_id.startswith("anthropic/"): + normalized = model_id + elif model_id.startswith("claude"): + normalized = f"anthropic/{model_id}" + else: + continue + elif provider == "gemini": + if model_id.startswith("gemini/"): + normalized = model_id + elif model_id.startswith("gemini"): + normalized = f"gemini/{model_id}" + else: + continue + else: + continue + context = meta.get("max_input_tokens") if isinstance(meta, dict) else None + models.append( + { + "value": normalized, + "label": f"{model_label(normalized, context=context if isinstance(context, int) else None)} · LiteLLM catalog", + } + ) + unique = {m["value"]: m for m in models} + preferred = PREFERRED_MODELS.get(provider, []) + + def sort_key(item: dict[str, str]) -> tuple[int, str]: + value = item["value"] + if value in preferred: + return (preferred.index(value), "") + return (1000, item["label"].lower()) + + return sorted(unique.values(), key=sort_key) + + +def sort_model_options(provider: str, models: list[dict[str, str]]) -> list[dict[str, str]]: + preferred = PREFERRED_MODELS.get(provider, []) + + def sort_key(item: dict[str, str]) -> tuple[int, str]: + value = item["value"] + if value in preferred: + return (preferred.index(value), "") + return (1000, item["label"].lower()) + + unique = {m["value"]: m for m in models} + return sorted(unique.values(), key=sort_key) + + +async def model_options_for_provider(provider: str, api_key: str = "") -> list[dict[str, str]]: + provider = (provider or "openrouter").strip().lower() + now = time.time() + cache_key = f"{provider}:{hashlib.sha256((api_key or '').encode('utf-8')).hexdigest()[:10]}" + if MODEL_CACHE["key"] == cache_key and now - float(MODEL_CACHE["at"]) < 3600: + return list(MODEL_CACHE["models"]) + + models: list[dict[str, str]] = [] + if provider == "openrouter": + try: + models = await fetch_openrouter_models() + except Exception as exc: + logger.warning("OpenRouter model list failed: {}", exc) + elif provider == "anthropic": + try: + models = await fetch_anthropic_models(api_key) + except Exception as exc: + logger.warning("Anthropic model list failed: {}", exc) + elif provider == "openai": + try: + models = await fetch_openai_models(api_key) + except Exception as exc: + logger.warning("OpenAI model list failed: {}", exc) + if not models: + models = litellm_catalog_models(provider) + if not models: + models = [{"value": m, "label": f"{m} · fallback"} for m in MODEL_FALLBACKS.get(provider, [])] + + models = sort_model_options(provider, models) + MODEL_CACHE.update({"key": cache_key, "at": now, "models": models}) + return list(models) + + +async def audit(actor_id: int | None, action: str, entity_type: str, entity_id: int | None, after: Any = None) -> None: + pool = await get_pool() + await pool.execute( + """ + INSERT INTO audit_log(actor_id, action, entity_type, entity_id, after_json) + VALUES($1, $2, $3, $4, $5::jsonb) + """, + actor_id, + action, + entity_type, + entity_id, + json.dumps(after, ensure_ascii=False) if after is not None else None, + ) + + +@app.on_event("startup") +async def startup() -> None: + await bootstrap_admin() + + +@app.get("/health") +async def health() -> dict[str, str]: + return {"ok": "true"} + + +@app.get("/vk/oauth/callback", response_class=HTMLResponse) +async def vk_oauth_callback(request: Request, code: str = "", error: str = "", error_description: str = ""): + return templates.TemplateResponse( + "vk_oauth_callback.html", + base_context( + request, + None, + code=code, + code_verifier=request.cookies.get(VK_OAUTH_VERIFIER_COOKIE, ""), + device_id=request.query_params.get("device_id", ""), + state=request.query_params.get("state", ""), + expected_state=request.cookies.get(VK_OAUTH_STATE_COOKIE, ""), + error=error, + error_description=error_description, + ), + ) + + +@app.get("/vk/oauth/start") +async def vk_oauth_start() -> RedirectResponse: + verifier = secrets.token_urlsafe(64) + state = secrets.token_urlsafe(24) + redirect_uri = "https://sw.exostring.xyz/vk/oauth/callback" + params = { + "client_id": "54635120", + "redirect_uri": redirect_uri, + "response_type": "code", + "scope": "wall photos video groups offline", + "state": state, + "code_challenge": pkce_challenge(verifier), + "code_challenge_method": "s256", + "origin": "https://sw.exostring.xyz", + "v": "5.199", + } + response = RedirectResponse(f"https://id.vk.ru/authorize?{urlencode(params)}") + response.set_cookie(VK_OAUTH_VERIFIER_COOKIE, verifier, httponly=True, samesite="lax", max_age=15 * 60) + response.set_cookie(VK_OAUTH_STATE_COOKIE, state, httponly=True, samesite="lax", max_age=15 * 60) + return response + + +@app.get("/vk/group-oauth/start") +async def vk_group_oauth_start() -> RedirectResponse: + group_id = abs(int(settings.vk_storage_group_id)) + params = { + "client_id": "54635120", + "group_ids": str(group_id), + "display": "page", + "redirect_uri": "https://sw.exostring.xyz/vk/oauth/callback", + "scope": "manage,photos,docs", + "response_type": "token", + "state": secrets.token_urlsafe(24), + "v": "5.199", + } + return RedirectResponse(f"https://oauth.vk.com/authorize?{urlencode(params)}") + + +@app.get("/", response_class=HTMLResponse) +async def index(request: Request): + user = await get_current_user(request) + if not user: + return redirect("/login") + return redirect("/sources") + + +@app.get("/login", response_class=HTMLResponse) +async def login_get(request: Request): + return templates.TemplateResponse("login.html", base_context(request, None, error="")) + + +@app.post("/login", response_class=HTMLResponse) +async def login_post(request: Request, login: str = Form(...), password: str = Form(...)): + pool = await get_pool() + login_value = login.strip().lower() + ip = client_ip(request) + await pool.execute("DELETE FROM admin_login_attempts WHERE created_at < NOW() - INTERVAL '7 days'") + ip_failures = int( + await pool.fetchval( + """ + SELECT COUNT(*) + FROM admin_login_attempts + WHERE ip=$1 AND success=FALSE AND created_at > NOW() - INTERVAL '15 minutes' + """, + ip, + ) + or 0 + ) + login_failures = int( + await pool.fetchval( + """ + SELECT COUNT(*) + FROM admin_login_attempts + WHERE login=$1 AND success=FALSE AND created_at > NOW() - INTERVAL '15 minutes' + """, + login_value, + ) + or 0 + ) + if ip_failures >= 10 or login_failures >= 5: + await pool.execute( + "INSERT INTO admin_login_attempts(ip, login, success) VALUES($1, $2, FALSE)", + ip, + login_value, + ) + return templates.TemplateResponse( + "login.html", + base_context(request, None, error="Слишком много попыток. Подожди 15 минут и попробуй снова."), + status_code=status.HTTP_429_TOO_MANY_REQUESTS, + ) + row = await pool.fetchrow( + "SELECT id, login, password_hash, role, is_active FROM admin_users WHERE login=$1", + login_value, + ) + if not row or not row["is_active"] or not verify_password(password, row["password_hash"]): + await pool.execute( + "INSERT INTO admin_login_attempts(ip, login, success) VALUES($1, $2, FALSE)", + ip, + login_value, + ) + return templates.TemplateResponse("login.html", base_context(request, None, error="Неверный логин или пароль")) + + await pool.execute("INSERT INTO admin_login_attempts(ip, login, success) VALUES($1, $2, TRUE)", ip, login_value) + token = new_token() + csrf = new_token() + await pool.execute( + """ + INSERT INTO admin_sessions(token_hash, user_id, csrf_token, expires_at) + VALUES($1, $2, $3, $4) + """, + token_hash(token, settings.app_secret_key), + row["id"], + csrf, + now_utc() + timedelta(days=14), + ) + response = redirect("/sources") + response.set_cookie(COOKIE_NAME, token, httponly=True, secure=True, samesite="lax", max_age=14 * 24 * 3600) + return response + + +@app.post("/logout") +async def logout(request: Request): + token = request.cookies.get(COOKIE_NAME) + if token: + pool = await get_pool() + await pool.execute("DELETE FROM admin_sessions WHERE token_hash=$1", token_hash(token, settings.app_secret_key)) + response = redirect("/login") + response.delete_cookie(COOKIE_NAME) + return response + + +@app.get("/sources", response_class=HTMLResponse) +async def sources_list(request: Request, q: str = "", status_filter: str = "", page: int = 1, per_page: int = 50): + user = await get_current_user(request) + if not user: + return redirect("/login") + pool = await get_pool() + table_filters = parse_table_filters(request, SOURCE_FIELD_SPECS) + table_sorts = parse_table_sorts(request, SOURCE_FIELD_SPECS) + filter_clauses, filter_args = build_filter_sql(table_filters, SOURCE_FIELD_SPECS, 3) + where_parts = [ + "s.archived_at IS NULL", + "($1='' OR s.name ILIKE '%' || $1 || '%' OR COALESCE(s.tag,'') ILIKE '%' || $1 || '%' OR s.url ILIKE '%' || $1 || '%' OR COALESCE(s.external_id,'') ILIKE '%' || $1 || '%')", + "($2='' OR s.status=$2)", + *filter_clauses, + ] + where_sql = " AND ".join(where_parts) + base_args = [q.strip(), status_filter.strip(), *filter_args] + total = int( + await pool.fetchval( + f""" + SELECT COUNT(*) + FROM sources s + WHERE {where_sql} + """, + *base_args, + ) + or 0 + ) + pager = pagination(page, per_page, total) + limit_index = len(base_args) + 1 + offset_index = len(base_args) + 2 + order_sql = build_sort_sql(table_sorts, SOURCE_FIELD_SPECS, "s.active DESC, s.priority ASC, s.id DESC") + rows = await pool.fetch( + f""" + SELECT s.*, + COUNT(rp.id) AS posts_count, + COUNT(rp.id) FILTER (WHERE rp.created_at > NOW() - INTERVAL '24 hours') AS posts_24h + FROM sources s + LEFT JOIN raw_posts rp ON rp.source_id=s.id + WHERE {where_sql} + GROUP BY s.id + ORDER BY {order_sql} + LIMIT ${limit_index} OFFSET ${offset_index} + """, + *base_args, + pager["per_page"], + pager["offset"], + ) + sources = [] + for row in rows: + item = dict(row) + item["last_parsed_at_fmt"] = format_dt(item.get("last_parsed_at")) + item["last_checked_at_fmt"] = format_dt(item.get("last_checked_at")) + item["created_at_fmt"] = format_dt(item.get("created_at")) + sources.append(item) + return templates.TemplateResponse( + "sources.html", + base_context( + request, + user, + sources=sources, + q=q, + status_filter=status_filter, + pagination=pager, + table_filters=table_filters, + table_sorts=table_sorts, + field_options=field_options(SOURCE_FIELD_SPECS), + filter_ops=FILTER_OPS, + filter_op_options=FILTER_OP_OPTIONS, + source_preview=[], + bulk_text="", + bulk_active=True, + ), + ) + + +@app.post("/sources/preview", response_class=HTMLResponse) +async def sources_preview( + request: Request, + csrf_token: str = Form(...), + bulk_text: str = Form(""), + bulk_active: str = Form("off"), + q: str = Form(""), + status_filter: str = Form(""), + per_page: int = Form(50), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + preview = await build_sources_preview(bulk_text, default_active=bulk_active == "on") + pool = await get_pool() + table_filters = parse_table_filters(request, SOURCE_FIELD_SPECS) + table_sorts = parse_table_sorts(request, SOURCE_FIELD_SPECS) + filter_clauses, filter_args = build_filter_sql(table_filters, SOURCE_FIELD_SPECS, 3) + where_parts = [ + "s.archived_at IS NULL", + "($1='' OR s.name ILIKE '%' || $1 || '%' OR COALESCE(s.tag,'') ILIKE '%' || $1 || '%' OR s.url ILIKE '%' || $1 || '%' OR COALESCE(s.external_id,'') ILIKE '%' || $1 || '%')", + "($2='' OR s.status=$2)", + *filter_clauses, + ] + where_sql = " AND ".join(where_parts) + base_args = [q.strip(), status_filter.strip(), *filter_args] + total = int(await pool.fetchval(f"SELECT COUNT(*) FROM sources s WHERE {where_sql}", *base_args) or 0) + pager = pagination(1, per_page, total) + limit_index = len(base_args) + 1 + offset_index = len(base_args) + 2 + order_sql = build_sort_sql(table_sorts, SOURCE_FIELD_SPECS, "s.active DESC, s.priority ASC, s.id DESC") + rows = await pool.fetch( + f""" + SELECT s.*, + COUNT(rp.id) AS posts_count, + COUNT(rp.id) FILTER (WHERE rp.created_at > NOW() - INTERVAL '24 hours') AS posts_24h + FROM sources s + LEFT JOIN raw_posts rp ON rp.source_id=s.id + WHERE {where_sql} + GROUP BY s.id + ORDER BY {order_sql} + LIMIT ${limit_index} OFFSET ${offset_index} + """, + *base_args, + pager["per_page"], + pager["offset"], + ) + sources = [] + for row in rows: + item = dict(row) + item["last_parsed_at_fmt"] = format_dt(item.get("last_parsed_at")) + item["last_checked_at_fmt"] = format_dt(item.get("last_checked_at")) + item["created_at_fmt"] = format_dt(item.get("created_at")) + sources.append(item) + return templates.TemplateResponse( + "sources.html", + base_context( + request, + user, + sources=sources, + q=q, + status_filter=status_filter, + pagination=pager, + table_filters=table_filters, + table_sorts=table_sorts, + field_options=field_options(SOURCE_FIELD_SPECS), + filter_ops=FILTER_OPS, + filter_op_options=FILTER_OP_OPTIONS, + source_preview=preview, + bulk_text=bulk_text, + bulk_active=bulk_active == "on", + ), + ) + + +@app.post("/sources/bulk") +async def sources_bulk_create( + request: Request, + csrf_token: str = Form(...), + bulk_payload: str = Form("[]"), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + try: + items = json.loads(bulk_payload) + except json.JSONDecodeError: + items = [] + pool = await get_pool() + created = 0 + for item in items: + if not isinstance(item, dict) or not item.get("ok"): + continue + try: + row = await pool.fetchrow( + """ + INSERT INTO sources(platform, name, tag, url, external_id, external_owner_id, active, priority, created_by) + VALUES($1, $2, $3, $4, $5, $6, $7, 100, $8) + ON CONFLICT DO NOTHING + RETURNING id + """, + PLATFORM_VK, + str(item.get("name") or item.get("external_id") or "").strip(), + normalize_hash_tag(str(item.get("tag") or item.get("external_id") or ""), str(item.get("external_id") or "source")), + str(item.get("url") or "").strip(), + str(item.get("external_id") or "").strip(), + item.get("external_owner_id"), + bool(item.get("active", True)), + user["id"], + ) + if row: + created += 1 + await audit(user["id"], "source.create", "source", int(row["id"]), {"url": item.get("url"), "bulk": True}) + except Exception: + logger.exception("Bulk source insert failed: {}", item) + return redirect(f"/sources?q=&status_filter=&created={created}") + + +@app.get("/sources/new", response_class=HTMLResponse) +async def source_new(request: Request): + user = await get_current_user(request) + if not user: + return redirect("/login") + return templates.TemplateResponse( + "source_form.html", + base_context(request, user, source=None, action="/sources/new", title="Новый источник"), + ) + + +@app.post("/sources/new") +async def source_create( + request: Request, + csrf_token: str = Form(...), + platform: str = Form(PLATFORM_VK), + name: str = Form(...), + tag: str = Form(""), + url: str = Form(...), + active: str = Form("off"), + priority: int = Form(100), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + platform = platform.strip().lower() or PLATFORM_VK + external_id = normalize_vk_source(url) if platform == PLATFORM_VK else "" + external_owner_id = None + resolved_name = name.strip() + resolved_url = url.strip() + status_value = "new" + status_msg = None + if platform == PLATFORM_VK and external_id: + try: + async with VKAPIClient(rps=2, timeout_total_sec=12, timeout_connect_sec=5, retry_attempts=2) as client: + screen_name, owner_id, vk_name = await client.resolve_group(url) + external_id = screen_name + external_owner_id = owner_id + resolved_name = resolved_name or vk_name + resolved_url = f"https://vk.com/{screen_name}" + except Exception as exc: + resolved_name = resolved_name or external_id + status_value = "error" + status_msg = str(exc)[:500] + resolved_tag = normalize_hash_tag(tag or external_id or resolved_name, external_id or "source") + pool = await get_pool() + row = await pool.fetchrow( + """ + INSERT INTO sources(platform, name, tag, url, external_id, external_owner_id, active, priority, status, status_msg, created_by) + VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + ON CONFLICT DO NOTHING + RETURNING id + """, + platform, + resolved_name, + resolved_tag, + resolved_url, + external_id, + external_owner_id, + active == "on", + priority, + status_value, + status_msg, + user["id"], + ) + if row: + await audit(user["id"], "source.create", "source", int(row["id"]), {"url": url, "platform": platform}) + return redirect("/sources") + + +@app.get("/sources/{source_id}/edit", response_class=HTMLResponse) +async def source_edit(request: Request, source_id: int): + user = await get_current_user(request) + if not user: + return redirect("/login") + pool = await get_pool() + source = await pool.fetchrow("SELECT * FROM sources WHERE id=$1 AND archived_at IS NULL", source_id) + if not source: + return redirect("/sources") + return templates.TemplateResponse( + "source_form.html", + base_context( + request, + user, + source=dict(source), + action=f"/sources/{source_id}/edit", + title=f"Источник #{source_id}", + ), + ) + + +@app.post("/sources/{source_id}/edit") +async def source_update( + request: Request, + source_id: int, + csrf_token: str = Form(...), + platform: str = Form(PLATFORM_VK), + name: str = Form(...), + tag: str = Form(""), + url: str = Form(...), + active: str = Form("off"), + priority: int = Form(100), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + platform = platform.strip().lower() or PLATFORM_VK + external_id = normalize_vk_source(url) if platform == PLATFORM_VK else "" + pool = await get_pool() + await pool.execute( + """ + UPDATE sources + SET platform=$2, + name=$3, + tag=$4, + url=$5, + external_id=$6, + active=$7, + priority=$8, + updated_at=NOW() + WHERE id=$1 + """, + source_id, + platform, + name.strip(), + normalize_hash_tag(tag or external_id or name, external_id or "source"), + url.strip(), + external_id, + active == "on", + priority, + ) + await audit(user["id"], "source.update", "source", source_id, {"url": url, "platform": platform}) + return redirect("/sources") + + +@app.post("/sources/{source_id}/toggle") +async def source_toggle(request: Request, source_id: int, csrf_token: str = Form(...)): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + pool = await get_pool() + row = await pool.fetchrow( + """ + UPDATE sources + SET active=NOT active, + status=CASE WHEN active THEN 'paused' ELSE 'new' END, + updated_at=NOW() + WHERE id=$1 + RETURNING active + """, + source_id, + ) + await audit(user["id"], "source.toggle", "source", source_id, {"active": bool(row["active"]) if row else None}) + return redirect("/sources") + + +@app.post("/sources/{source_id}/delete") +async def source_delete(request: Request, source_id: int, csrf_token: str = Form(...)): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + pool = await get_pool() + await pool.execute("UPDATE sources SET archived_at=NOW(), active=FALSE, updated_at=NOW() WHERE id=$1", source_id) + await audit(user["id"], "source.archive", "source", source_id) + return redirect("/sources") + + +@app.get("/raw", response_class=HTMLResponse) +async def raw_posts( + request: Request, + status_filter: str = "", + date_from: str = "", + date_to: str = "", + score_min: str = "", + score_max: str = "", + sort: str = "created_desc", + page: int = 1, + per_page: int = 50, +): + user = await get_current_user(request) + if not user: + return redirect("/login") + pool = await get_pool() + order_map = { + "id_desc": "rp.id DESC", + "id_asc": "rp.id ASC", + "created_desc": "rp.created_at DESC", + "created_asc": "rp.created_at ASC", + "posted_desc": "rp.posted_at DESC NULLS LAST, rp.created_at DESC", + "posted_asc": "rp.posted_at ASC NULLS LAST, rp.created_at ASC", + "source_asc": "lower(s.name) ASC, rp.created_at DESC", + "source_desc": "lower(s.name) DESC, rp.created_at DESC", + "stage_asc": "rp.status ASC, COALESCE(rp.qualification_status, 'pending') ASC, COALESCE(rp.rewrite_status, 'pending') ASC, rp.created_at DESC", + "stage_desc": "rp.status DESC, COALESCE(rp.qualification_status, 'pending') DESC, COALESCE(rp.rewrite_status, 'pending') DESC, rp.created_at DESC", + "text_asc": "lower(rp.raw_text) ASC, rp.created_at DESC", + "text_desc": "lower(rp.raw_text) DESC, rp.created_at DESC", + "score_desc": "rp.qualification_score DESC NULLS LAST, rp.created_at DESC", + "score_asc": "rp.qualification_score ASC NULLS LAST, rp.created_at DESC", + } + raw_statuses = selected_values(request, "raw_status") + if status_filter.strip() and not raw_statuses: + raw_statuses = [status_filter.strip()] + source_ids = selected_int_values(request, "source_id") + categories_selected = selected_values(request, "category") + qualification_statuses = selected_values(request, "qualification_status") + rewrite_statuses = selected_values(request, "rewrite_status") + where_parts: list[str] = [] + base_args: list[Any] = [] + if raw_statuses: + add_where(where_parts, base_args, "rp.status = ANY(${i}::text[])", raw_statuses) + if source_ids: + add_where(where_parts, base_args, "rp.source_id = ANY(${i}::bigint[])", source_ids) + if categories_selected: + add_where(where_parts, base_args, "COALESCE(rp.final_category, rp.rewrite_category, '') = ANY(${i}::text[])", categories_selected) + if qualification_statuses: + add_where(where_parts, base_args, "COALESCE(rp.qualification_status, 'pending') = ANY(${i}::text[])", qualification_statuses) + if rewrite_statuses: + add_where(where_parts, base_args, "COALESCE(rp.rewrite_status, 'pending') = ANY(${i}::text[])", rewrite_statuses) + parsed_date_from = parse_date_filter(date_from) + parsed_date_to = parse_date_filter(date_to) + if parsed_date_from: + add_where(where_parts, base_args, "timezone('Asia/Yekaterinburg', COALESCE(rp.posted_at, rp.created_at))::date >= ${i}::date", parsed_date_from) + if parsed_date_to: + add_where(where_parts, base_args, "timezone('Asia/Yekaterinburg', COALESCE(rp.posted_at, rp.created_at))::date <= ${i}::date", parsed_date_to) + if score_min.strip().isdigit(): + add_where(where_parts, base_args, "rp.qualification_score >= ${i}::int", int(score_min.strip())) + if score_max.strip().isdigit(): + add_where(where_parts, base_args, "rp.qualification_score <= ${i}::int", int(score_max.strip())) + where_sql = " AND ".join(where_parts) if where_parts else "TRUE" + order_sql = order_map.get(sort, order_map["created_desc"]) + total = int( + await pool.fetchval( + f""" + SELECT COUNT(*) + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + WHERE {where_sql} + """, + *base_args, + ) + or 0 + ) + pager = pagination(page, per_page, total) + limit_index = len(base_args) + 1 + offset_index = len(base_args) + 2 + rows = await pool.fetch( + f""" + SELECT rp.*, s.name AS source_name, s.tag AS source_tag, s.platform AS source_platform, + COUNT(rpm.id) AS media_count, + COALESCE( + jsonb_agg( + jsonb_build_object( + 'id', rpm.id, + 'type', rpm.media_type, + 'url', rpm.original_url, + 'status', rpm.status, + 'error', rpm.error, + 'duration_sec', rpm.duration_sec + ) + ORDER BY rpm.sort_order ASC, rpm.id ASC + ) FILTER (WHERE rpm.id IS NOT NULL), + '[]'::jsonb + ) AS media_items + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + LEFT JOIN raw_post_media rpm ON rpm.raw_post_id=rp.id AND COALESCE(rpm.editor_added, FALSE)=FALSE + WHERE {where_sql} + GROUP BY rp.id, s.name, s.tag, s.platform + ORDER BY {order_sql} + LIMIT ${limit_index} OFFSET ${offset_index} + """, + *base_args, + pager["per_page"], + pager["offset"], + ) + posts = [prepare_raw_post(row) for row in rows] + return templates.TemplateResponse( + "raw_posts.html", + base_context( + request, + user, + posts=posts, + raw_statuses=raw_statuses, + source_ids=source_ids, + categories_selected=categories_selected, + qualification_statuses=qualification_statuses, + rewrite_statuses=rewrite_statuses, + date_from=date_from, + date_to=date_to, + score_min=score_min, + score_max=score_max, + sort=sort, + facets=await raw_filter_facets(pool), + preserved_query=preserved_query(request, {"page", "per_page"}), + sort_headers=raw_sort_headers(request, sort), + pagination=pager, + ), + ) + + +@app.get("/editor", response_class=HTMLResponse) +async def editor_feed( + request: Request, + status_filter: str = "review", + category_filter: str = "", + date_from: str = "", + date_to: str = "", + score_min: str = "", + score_max: str = "", + sort: str = "rewritten_desc", + page: int = 1, + per_page: int = 25, +): + user = await get_current_user(request) + if not user: + return redirect("/login") + pool = await get_pool() + editorial_statuses = selected_values(request, "editorial_status") + if status_filter.strip() and not editorial_statuses: + editorial_statuses = [status_filter.strip()] + categories_selected = selected_values(request, "category") + if category_filter.strip() and not categories_selected: + categories_selected = [category_filter.strip()] + source_ids = selected_int_values(request, "source_id") + sort_map = { + "rewritten_desc": "rp.rewritten_at DESC NULLS LAST, rp.id DESC", + "rewritten_asc": "rp.rewritten_at ASC NULLS LAST, rp.id ASC", + "score_desc": "rp.qualification_score DESC NULLS LAST, rp.rewritten_at DESC NULLS LAST", + "score_asc": "rp.qualification_score ASC NULLS LAST, rp.rewritten_at DESC NULLS LAST", + "posted_desc": "rp.posted_at DESC NULLS LAST, rp.rewritten_at DESC NULLS LAST", + "posted_asc": "rp.posted_at ASC NULLS LAST, rp.rewritten_at DESC NULLS LAST", + } + where_parts = ["rp.rewrite_status='ready'"] + base_args: list[Any] = [] + if editorial_statuses: + add_where(where_parts, base_args, "COALESCE(rp.editorial_status, 'review') = ANY(${i}::text[])", editorial_statuses) + if categories_selected: + add_where(where_parts, base_args, "COALESCE(rp.final_category, rp.rewrite_category, '') = ANY(${i}::text[])", categories_selected) + if source_ids: + add_where(where_parts, base_args, "rp.source_id = ANY(${i}::bigint[])", source_ids) + parsed_date_from = parse_date_filter(date_from) + parsed_date_to = parse_date_filter(date_to) + if parsed_date_from: + add_where(where_parts, base_args, "timezone('Asia/Yekaterinburg', COALESCE(rp.posted_at, rp.created_at))::date >= ${i}::date", parsed_date_from) + if parsed_date_to: + add_where(where_parts, base_args, "timezone('Asia/Yekaterinburg', COALESCE(rp.posted_at, rp.created_at))::date <= ${i}::date", parsed_date_to) + if score_min.strip().isdigit(): + add_where(where_parts, base_args, "rp.qualification_score >= ${i}::int", int(score_min.strip())) + if score_max.strip().isdigit(): + add_where(where_parts, base_args, "rp.qualification_score <= ${i}::int", int(score_max.strip())) + where_sql = " AND ".join(where_parts) + total = int( + await pool.fetchval( + f""" + SELECT COUNT(*) + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + WHERE {where_sql} + """, + *base_args, + ) + or 0 + ) + pager = pagination(page, per_page, total) + limit_index = len(base_args) + 1 + offset_index = len(base_args) + 2 + rows = await pool.fetch( + f""" + SELECT rp.*, s.name AS source_name, s.tag AS source_tag, s.platform AS source_platform, s.url AS source_url, + u.login AS reviewed_by_login, + COUNT(rpm.id) AS media_count, + COALESCE( + jsonb_agg( + jsonb_build_object( + 'id', rpm.id, + 'type', rpm.media_type, + 'url', rpm.original_url, + 'status', rpm.status, + 'error', rpm.error, + 'duration_sec', rpm.duration_sec + ) + ORDER BY rpm.sort_order ASC, rpm.id ASC + ) FILTER (WHERE rpm.id IS NOT NULL), + '[]'::jsonb + ) AS media_items + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + LEFT JOIN admin_users u ON u.id=rp.reviewed_by + LEFT JOIN raw_post_media rpm ON rpm.raw_post_id=rp.id AND COALESCE(rpm.editor_hidden, FALSE)=FALSE + WHERE {where_sql} + GROUP BY rp.id, s.name, s.tag, s.platform, s.url, u.login + ORDER BY + CASE COALESCE(rp.editorial_status, 'review') + WHEN 'review' THEN 0 + WHEN 'edited' THEN 1 + WHEN 'regenerating' THEN 2 + WHEN 'rejected' THEN 3 + WHEN 'accepted' THEN 4 + ELSE 5 + END, + {sort_map.get(sort, sort_map["rewritten_desc"])} + LIMIT ${limit_index} OFFSET ${offset_index} + """, + *base_args, + pager["per_page"], + pager["offset"], + ) + posts = [prepare_editor_post(row) for row in rows] + categories = await writer_categories() + counts_rows = await pool.fetch( + """ + SELECT COALESCE(editorial_status, 'review') AS status, COUNT(*) AS count + FROM raw_posts + WHERE rewrite_status='ready' + GROUP BY COALESCE(editorial_status, 'review') + """ + ) + counts = {str(row["status"]): int(row["count"]) for row in counts_rows} + return templates.TemplateResponse( + "editor.html", + base_context( + request, + user, + posts=posts, + categories=categories, + status_options=EDITOR_STATUS_OPTIONS, + status_filter=editorial_statuses[0] if len(editorial_statuses) == 1 else "", + editorial_statuses=editorial_statuses, + categories_selected=categories_selected, + source_ids=source_ids, + date_from=date_from, + date_to=date_to, + score_min=score_min, + score_max=score_max, + sort=sort, + facets=await editor_filter_facets(pool), + counts=counts, + preserved_query=preserved_query(request, {"page", "per_page"}), + pagination=pager, + ), + ) + + +@app.post("/editor/{post_id}/accept") +async def editor_accept(request: Request, post_id: int, csrf_token: str = Form(...)): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + pool = await get_pool() + await pool.execute( + """ + UPDATE raw_posts + SET editorial_status='accepted', + final_text=COALESCE(final_text, rewritten_text), + final_category=COALESCE(final_category, rewrite_category), + final_category_tag=COALESCE(final_category_tag, rewrite_category_tag), + final_source_tag=COALESCE(final_source_tag, rewrite_source_tag), + reviewed_by=$2, + reviewed_at=NOW(), + updated_at=NOW() + WHERE id=$1 AND rewrite_status='ready' + """, + post_id, + user["id"], + ) + await audit(user["id"], "editor.accept", "raw_post", post_id) + return redirect("/editor") + + +@app.post("/editor/{post_id}/reject") +async def editor_reject(request: Request, post_id: int, csrf_token: str = Form(...), editor_notes: str = Form("")): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + pool = await get_pool() + await pool.execute( + """ + UPDATE raw_posts + SET editorial_status='rejected', + editor_notes=$2, + reviewed_by=$3, + reviewed_at=NOW(), + updated_at=NOW() + WHERE id=$1 AND rewrite_status='ready' + """, + post_id, + editor_notes.strip()[:1000], + user["id"], + ) + await audit(user["id"], "editor.reject", "raw_post", post_id) + return redirect("/editor") + + +@app.post("/editor/{post_id}/save") +async def editor_save( + request: Request, + post_id: int, + csrf_token: str = Form(...), + final_text: str = Form(...), + final_category: str = Form(...), + final_source_tag: str = Form(""), + editor_notes: str = Form(""), + action: str = Form("save"), + delete_media_ids: list[int] | None = Form(default=None), + media_files: list[UploadFile] | None = File(default=None), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + final_category = final_category.strip() + final_category_tag = normalize_hash_tag(final_category, "category") + final_source_tag = normalize_hash_tag(final_source_tag, "source") + status_value = "accepted" if action == "accept" else "edited" + pool = await get_pool() + await pool.execute( + """ + UPDATE raw_posts + SET editorial_status=$2, + final_text=$3, + final_category=$4, + final_category_tag=$5, + final_source_tag=$6, + editor_notes=$7, + reviewed_by=CASE WHEN $2='accepted' THEN $8 ELSE reviewed_by END, + reviewed_at=CASE WHEN $2='accepted' THEN NOW() ELSE reviewed_at END, + edited_at=NOW(), + updated_at=NOW() + WHERE id=$1 AND rewrite_status='ready' + """, + post_id, + status_value, + final_text.strip(), + final_category, + final_category_tag, + final_source_tag, + editor_notes.strip()[:1000], + user["id"], + ) + delete_media_ids = delete_media_ids or [] + media_files = media_files or [] + if delete_media_ids: + local_media_paths = [ + uploaded_editor_media_path(str(row["original_url"] or "")) + for row in await pool.fetch( + """ + SELECT original_url + FROM raw_post_media + WHERE raw_post_id=$1 + AND id=ANY($2::bigint[]) + AND editor_added=TRUE + """, + post_id, + delete_media_ids, + ) + ] + await pool.execute( + """ + UPDATE raw_post_media + SET editor_hidden=TRUE, + updated_at=NOW() + WHERE raw_post_id=$1 AND id=ANY($2::bigint[]) + """, + post_id, + delete_media_ids, + ) + for path in local_media_paths: + if path and path.is_file(): + try: + path.unlink(missing_ok=True) + except OSError: + logger.warning("Could not delete editor media file {}", path) + for media_file in media_files: + if not media_file.filename: + continue + content = await media_file.read() + if not content or len(content) > 50 * 1024 * 1024: + continue + original_name = Path(media_file.filename or "media").name + suffix = Path(original_name).suffix.lower() + if not suffix: + guessed = mimetypes.guess_extension(media_file.content_type or "") + suffix = guessed or ".bin" + safe_name = f"{post_id}_{int(time.time())}_{secrets.token_hex(6)}{suffix}" + target = EDITOR_MEDIA_DIR / safe_name + target.write_bytes(content) + public_url = f"/uploads/editor_media/{safe_name}" + content_type = media_file.content_type or mimetypes.guess_type(original_name)[0] or "" + if content_type.startswith("image/"): + media_type = "photo" + elif content_type.startswith("video/"): + media_type = "video" + else: + media_type = "doc" + sort_order = int( + await pool.fetchval("SELECT COALESCE(MAX(sort_order), -1) + 1 FROM raw_post_media WHERE raw_post_id=$1", post_id) + or 0 + ) + await pool.execute( + """ + INSERT INTO raw_post_media( + raw_post_id, platform, media_type, original_url, preview_url, sort_order, status, + editor_added, editor_uploaded_by, editor_uploaded_at + ) + VALUES($1, 'manual', $2, $3, $3, $4, 'ready', TRUE, $5, NOW()) + """, + post_id, + media_type, + public_url, + sort_order, + user["id"], + ) + await audit(user["id"], "editor.save" if status_value == "edited" else "editor.save_accept", "raw_post", post_id) + return redirect("/editor") + + +@app.get("/raw/{post_id}", response_class=HTMLResponse) +async def raw_post_detail(request: Request, post_id: int): + user = await get_current_user(request) + if not user: + return redirect("/login") + pool = await get_pool() + row = await pool.fetchrow( + """ + SELECT rp.*, s.name AS source_name, s.tag AS source_tag, s.platform AS source_platform, s.url AS source_url, + COUNT(rpm.id) AS media_count, + COALESCE( + jsonb_agg( + jsonb_build_object( + 'id', rpm.id, + 'type', rpm.media_type, + 'url', rpm.original_url, + 'status', rpm.status, + 'error', rpm.error, + 'duration_sec', rpm.duration_sec + ) + ORDER BY rpm.sort_order ASC, rpm.id ASC + ) FILTER (WHERE rpm.id IS NOT NULL), + '[]'::jsonb + ) AS media_items, + b.provider AS qualification_provider, + b.model AS batch_model, + b.status AS batch_status, + b.posts_count AS batch_posts_count, + b.accepted_count AS batch_accepted_count, + b.rejected_count AS batch_rejected_count, + b.maybe_count AS batch_maybe_count, + b.error AS batch_error, + b.prompt_text AS batch_prompt_text, + b.created_at AS batch_created_at, + b.completed_at AS batch_completed_at, + b.response_json AS batch_response_json, + b.prompt_tokens AS batch_prompt_tokens, + b.completion_tokens AS batch_completion_tokens, + b.total_tokens AS batch_total_tokens, + b.estimated_cost_usd AS batch_estimated_cost_usd, + wb.provider AS rewrite_provider, + wb.model AS writer_batch_model, + wb.status AS writer_batch_status, + wb.posts_count AS writer_batch_posts_count, + wb.ready_count AS writer_batch_ready_count, + wb.failed_count AS writer_batch_failed_count, + wb.error AS writer_batch_error, + wb.prompt_text AS writer_batch_prompt_text, + wb.created_at AS writer_batch_created_at, + wb.completed_at AS writer_batch_completed_at, + wb.response_json AS writer_batch_response_json, + wb.prompt_tokens AS writer_prompt_tokens, + wb.completion_tokens AS writer_completion_tokens, + wb.total_tokens AS writer_total_tokens, + wb.estimated_cost_usd AS writer_estimated_cost_usd + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + LEFT JOIN raw_post_media rpm ON rpm.raw_post_id=rp.id + LEFT JOIN ai_qualification_batches b ON b.id=rp.qualification_batch_id + LEFT JOIN ai_writer_batches wb ON wb.id=rp.rewrite_batch_id + WHERE rp.id=$1 + GROUP BY rp.id, s.name, s.tag, s.platform, s.url, b.id, wb.id + """, + post_id, + ) + if not row: + return redirect("/raw") + post = prepare_raw_post(row) + post["batch_created_at_fmt"] = format_dt(post.get("batch_created_at")) + post["batch_completed_at_fmt"] = format_dt(post.get("batch_completed_at")) + response_json = post.get("batch_response_json") or {} + if isinstance(response_json, str): + try: + response_json = json.loads(response_json) + except json.JSONDecodeError: + pass + post["batch_response_pretty"] = json.dumps(response_json, ensure_ascii=False, indent=2) + if not post.get("batch_prompt_text") and post.get("qualification_batch_id"): + post["batch_prompt_text"] = ( + "Точный prompt для этого старого batch ещё не сохранялся.\n" + f"Prompt hash: {post.get('qualification_prompt_hash') or '—'}" + ) + else: + post["batch_prompt_text"] = post.get("batch_prompt_text") or "" + writer_response_json = post.get("writer_batch_response_json") or {} + if isinstance(writer_response_json, str): + try: + writer_response_json = json.loads(writer_response_json) + except json.JSONDecodeError: + pass + post["writer_batch_created_at_fmt"] = format_dt(post.get("writer_batch_created_at")) + post["writer_batch_completed_at_fmt"] = format_dt(post.get("writer_batch_completed_at")) + post["rewritten_at_fmt"] = format_dt(post.get("rewritten_at")) + post["writer_batch_response_pretty"] = json.dumps(writer_response_json, ensure_ascii=False, indent=2) + if not post.get("writer_batch_prompt_text") and post.get("rewrite_batch_id"): + post["writer_batch_prompt_text"] = ( + "Точный prompt для этого старого writer batch ещё не сохранялся.\n" + f"Prompt hash: {post.get('rewrite_prompt_hash') or '—'}" + ) + else: + post["writer_batch_prompt_text"] = post.get("writer_batch_prompt_text") or "" + return templates.TemplateResponse("raw_post_detail.html", base_context(request, user, post=post)) + + +@app.get("/workers", response_class=HTMLResponse) +async def workers(request: Request): + user = await get_current_user(request) + if not user: + return redirect("/login") + pool = await get_pool() + rows = await pool.fetch( + """ + SELECT wc.name, wc.enabled, wc.settings_json, wh.heartbeat_at, wh.status, wh.current_job_id, wh.meta_json + FROM worker_controls wc + LEFT JOIN worker_heartbeats wh ON wh.name=wc.name + ORDER BY wc.name + """ + ) + settings_rows = await pool.fetch("SELECT * FROM app_settings ORDER BY category, key") + settings = [] + for row in settings_rows: + item = dict(row) + if isinstance(item.get("value_json"), str): + try: + item["value_json"] = json.loads(item["value_json"]) + except json.JSONDecodeError: + pass + if item.get("key") == "ai_writer_categories": + continue + settings.append(item) + settings.sort(key=setting_sort_key) + ai_provider = str(setting_value(settings, "ai_qualifier_provider", "openrouter") or "openrouter") + ai_model = str(setting_value(settings, "ai_qualifier_model", "") or "") + ai_api_key = str(setting_value(settings, "ai_qualifier_api_key", "") or "") + ai_model_options = await model_options_for_provider(ai_provider, ai_api_key) + writer_provider = str(setting_value(settings, "ai_writer_provider", "anthropic") or "anthropic") + writer_model = str(setting_value(settings, "ai_writer_model", "") or "") + writer_api_key = str(setting_value(settings, "ai_writer_api_key", "") or "") + writer_model_options = await model_options_for_provider(writer_provider, writer_api_key) + return templates.TemplateResponse( + "workers.html", + base_context( + request, + user, + workers=[dict(r) for r in rows], + settings=settings, + provider_options=PROVIDER_OPTIONS, + ai_provider=ai_provider, + ai_model=ai_model, + ai_model_options=ai_model_options, + writer_provider=writer_provider, + writer_model=writer_model, + writer_model_options=writer_model_options, + categories=await category_rows(), + category_titles=CATEGORY_TITLES, + prompt_hints=PROMPT_HINTS, + ), + ) + + +@app.post("/categories/create") +async def category_create( + request: Request, + csrf_token: str = Form(...), + name: str = Form(...), + tag: str = Form(""), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + name = name.strip().strip("#") + if not name: + return redirect("/workers") + tag = normalize_hash_tag(tag or name, "category") + pool = await get_pool() + sort_order = int(await pool.fetchval("SELECT COALESCE(MAX(sort_order), 0) + 10 FROM content_categories") or 10) + await pool.execute( + """ + INSERT INTO content_categories(name, tag, sort_order) + VALUES($1, $2, $3) + ON CONFLICT (name) DO UPDATE + SET tag=$2, + is_active=TRUE, + updated_at=NOW() + """, + name, + tag, + sort_order, + ) + await audit(user["id"], "category.create", "content_category", None, {"name": name, "tag": tag}) + return redirect("/workers") + + +@app.post("/categories/{category_id}/update") +async def category_update( + request: Request, + category_id: int, + csrf_token: str = Form(...), + name: str = Form(...), + tag: str = Form(""), + sort_order: int = Form(0), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + name = name.strip().strip("#") + if not name: + return redirect("/workers") + tag = normalize_hash_tag(tag or name, "category") + pool = await get_pool() + await pool.execute( + """ + UPDATE content_categories + SET name=$2, + tag=$3, + sort_order=$4, + updated_at=NOW() + WHERE id=$1 + """, + category_id, + name, + tag, + sort_order, + ) + await audit(user["id"], "category.update", "content_category", category_id, {"name": name, "tag": tag}) + return redirect("/workers") + + +@app.post("/categories/{category_id}/toggle") +async def category_toggle(request: Request, category_id: int, csrf_token: str = Form(...)): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + pool = await get_pool() + row = await pool.fetchrow( + """ + UPDATE content_categories + SET is_active=NOT is_active, + updated_at=NOW() + WHERE id=$1 + RETURNING name, is_active + """, + category_id, + ) + await audit( + user["id"], + "category.toggle", + "content_category", + category_id, + {"name": row["name"] if row else None, "is_active": bool(row["is_active"]) if row else None}, + ) + return redirect("/workers") + + +@app.post("/categories/{category_id}/delete") +async def category_delete(request: Request, category_id: int, csrf_token: str = Form(...)): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + pool = await get_pool() + row = await pool.fetchrow( + """ + UPDATE content_categories + SET is_active=FALSE, + updated_at=NOW() + WHERE id=$1 + RETURNING name + """, + category_id, + ) + await audit(user["id"], "category.archive", "content_category", category_id, {"name": row["name"] if row else None}) + return redirect("/workers") + + +@app.post("/workers/{worker_name}/toggle") +async def worker_toggle(request: Request, worker_name: str, csrf_token: str = Form(...)): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + pool = await get_pool() + row = await pool.fetchrow( + """ + UPDATE worker_controls + SET enabled=NOT enabled, + updated_by=$2, + updated_at=NOW() + WHERE name=$1 + RETURNING enabled + """, + worker_name, + user["id"], + ) + await audit(user["id"], "worker.toggle", "worker", None, {"name": worker_name, "enabled": bool(row["enabled"])}) + return redirect("/workers") + + +@app.post("/settings/save") +async def settings_save(request: Request, csrf_token: str = Form(...), key: str = Form(...), value: str = Form(...)): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + pool = await get_pool() + row = await pool.fetchrow("SELECT value_type FROM app_settings WHERE key=$1", key) + if not row: + return redirect("/workers") + value_type = str(row["value_type"]) + if value_type == "int": + parsed: Any = int(value) + elif value_type == "float": + parsed = float(value) + elif value_type == "bool": + parsed = str(value).lower() in {"1", "true", "yes", "on"} + elif value_type == "secret": + if not value.strip(): + return redirect("/workers") + parsed = value.strip() + else: + parsed = value + if key in {"ai_qualifier_prompt", "ai_qualifier_contract", "ai_writer_prompt", "ai_writer_contract"}: + parsed = parsed.replace("\\r\\n", "\n").replace("\\n", "\n") + await pool.execute( + """ + UPDATE app_settings + SET value_json=$2::jsonb, + updated_by=$3, + updated_at=NOW() + WHERE key=$1 + """, + key, + json.dumps(parsed, ensure_ascii=False), + user["id"], + ) + await audit(user["id"], "setting.update", "setting", None, {"key": key, "value": parsed}) + return redirect("/workers") + + +@app.get("/users", response_class=HTMLResponse) +async def users(request: Request, page: int = 1, per_page: int = 50): + user = await get_current_user(request) + if not user: + return redirect("/login") + pool = await get_pool() + total = int(await pool.fetchval("SELECT COUNT(*) FROM admin_users") or 0) + pager = pagination(page, per_page, total) + rows = await pool.fetch( + "SELECT id, login, role, is_active, created_at FROM admin_users ORDER BY id LIMIT $1 OFFSET $2", + pager["per_page"], + pager["offset"], + ) + users_rows = [] + for row in rows: + item = dict(row) + item["created_at_fmt"] = format_dt(item.get("created_at")) + users_rows.append(item) + return templates.TemplateResponse("users.html", base_context(request, user, users=users_rows, pagination=pager)) + + +@app.post("/users/create") +async def user_create( + request: Request, + csrf_token: str = Form(...), + login: str = Form(...), + password: str = Form(...), + role: str = Form("editor"), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + if user["role"] != "admin": + return redirect("/users") + role = role if role in {"admin", "editor", "viewer"} else "editor" + pool = await get_pool() + await pool.execute( + "INSERT INTO admin_users(login, password_hash, role) VALUES($1, $2, $3)", + login.strip().lower(), + hash_password(password), + role, + ) + await audit(user["id"], "user.create", "user", None, {"login": login, "role": role}) + return redirect("/users") + + +@app.post("/users/{user_id}/toggle") +async def user_toggle(request: Request, user_id: int, csrf_token: str = Form(...)): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + if user["role"] != "admin" or int(user["id"]) == user_id: + return redirect("/users") + pool = await get_pool() + await pool.execute("UPDATE admin_users SET is_active=NOT is_active, updated_at=NOW() WHERE id=$1", user_id) + await audit(user["id"], "user.toggle", "user", user_id) + return redirect("/users") diff --git a/src/vk_parser_app/config.py b/src/vk_parser_app/config.py new file mode 100644 index 0000000..994af9a --- /dev/null +++ b/src/vk_parser_app/config.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + app_env: str = "production" + app_secret_key: str = "change-me" + admin_bootstrap_login: str = "admin" + admin_bootstrap_password: str = "" + + db_host: str = "localhost" + db_port: int = 5432 + db_name: str = "vk_parser" + db_user: str = "vk_parser_user" + db_password: str = "" + + vk_access_token: str = "" + vk_group_access_token: str = "" + vk_api_version: str = "5.199" + vk_storage_group_id: int = 0 + + tg_bot_token: str = "" + tg_media_channel_id: str = "" + local_bot_api_url: str = "" + + admin_host: str = "0.0.0.0" + admin_port: int = 8080 + log_level: str = "INFO" + + model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") + + @property + def db_dsn(self) -> str: + return ( + f"postgresql://{self.db_user}:{self.db_password}" + f"@{self.db_host}:{self.db_port}/{self.db_name}" + ) + + @property + def vk_storage_owner_id(self) -> int: + return -abs(int(self.vk_storage_group_id)) + + +settings = Settings() diff --git a/src/vk_parser_app/constants.py b/src/vk_parser_app/constants.py new file mode 100644 index 0000000..dca6219 --- /dev/null +++ b/src/vk_parser_app/constants.py @@ -0,0 +1,30 @@ +PLATFORM_VK = "vk" + +SOURCE_STATUS_NEW = "new" +SOURCE_STATUS_OK = "ok" +SOURCE_STATUS_ERROR = "error" +SOURCE_STATUS_PAUSED = "paused" + +POST_STATUS_RAW_SAVED = "raw_saved" +POST_STATUS_STORAGE_PENDING = "storage_pending" +POST_STATUS_STORAGE_READY = "storage_ready" +POST_STATUS_SKIPPED = "skipped" +POST_STATUS_FAILED = "failed" + +MEDIA_STATUS_PENDING = "pending" +MEDIA_STATUS_UPLOADED = "uploaded" +MEDIA_STATUS_LINK_ONLY = "link_only" +MEDIA_STATUS_FAILED = "failed" + +JOB_STATUS_PENDING = "pending" +JOB_STATUS_IN_PROGRESS = "in_progress" +JOB_STATUS_RETRY = "retry" +JOB_STATUS_DONE = "done" +JOB_STATUS_DEAD = "dead" + +JOB_TYPE_VK_STORAGE_COPY = "vk.storage.copy" + +WORKER_PARSER = "vk-parser" +WORKER_STORAGE_UPLOADER = "vk-storage-uploader" +WORKER_AI_QUALIFIER = "ai-qualifier" +WORKER_AI_WRITER = "ai-writer" diff --git a/src/vk_parser_app/db.py b/src/vk_parser_app/db.py new file mode 100644 index 0000000..deadc9f --- /dev/null +++ b/src/vk_parser_app/db.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import asyncpg +from loguru import logger + +from .config import settings + +_pool: asyncpg.Pool | None = None + + +async def get_pool() -> asyncpg.Pool: + global _pool + if _pool is None: + _pool = await asyncpg.create_pool( + host=settings.db_host, + port=settings.db_port, + database=settings.db_name, + user=settings.db_user, + password=settings.db_password, + min_size=1, + max_size=10, + command_timeout=60, + ) + logger.info("Database pool created: {}:{}/{}", settings.db_host, settings.db_port, settings.db_name) + return _pool + + +async def close_pool() -> None: + global _pool + if _pool is not None: + await _pool.close() + _pool = None + + +async def fetch_setting(key: str, default: Any = None) -> Any: + pool = await get_pool() + row = await pool.fetchrow("SELECT value_json FROM app_settings WHERE key=$1", key) + if not row: + return default + value = row["value_json"] + if isinstance(value, str): + try: + return json.loads(value) + except json.JSONDecodeError: + return value + return value + + +async def fetch_int_setting(key: str, default: int) -> int: + try: + return int(await fetch_setting(key, default)) + except Exception: + return default + + +async def fetch_float_setting(key: str, default: float) -> float: + try: + return float(await fetch_setting(key, default)) + except Exception: + return default + + +async def fetch_bool_setting(key: str, default: bool) -> bool: + try: + value = await fetch_setting(key, default) + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() in {"1", "true", "yes", "on"} + return bool(value) + except Exception: + return default + + +async def apply_migrations(migrations_dir: Path) -> None: + pool = await get_pool() + async with pool.acquire() as conn: + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ) + """ + ) + for path in sorted(migrations_dir.glob("*.sql")): + version = path.name + exists = await conn.fetchval("SELECT 1 FROM schema_migrations WHERE version=$1", version) + if exists: + continue + sql = path.read_text(encoding="utf-8") + async with conn.transaction(): + await conn.execute(sql) + await conn.execute("INSERT INTO schema_migrations(version) VALUES($1)", version) + logger.info("Applied migration {}", version) diff --git a/src/vk_parser_app/heartbeat.py b/src/vk_parser_app/heartbeat.py new file mode 100644 index 0000000..8634cc0 --- /dev/null +++ b/src/vk_parser_app/heartbeat.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import json +import time +from typing import Any + + +class HeartbeatReporter: + def __init__(self, name: str, interval_sec: int = 30) -> None: + self.name = name + self.interval_sec = max(5, int(interval_sec)) + self._last = 0.0 + + async def beat( + self, + pool, + status: str = "running", + current_job_id: int | None = None, + meta: dict[str, Any] | None = None, + force: bool = False, + ) -> None: + now = time.monotonic() + if not force and now - self._last < self.interval_sec: + return + self._last = now + try: + async with pool.acquire() as conn: + await conn.execute( + """ + INSERT INTO worker_heartbeats(name, heartbeat_at, status, current_job_id, meta_json, updated_at) + VALUES ($1, NOW(), $2, $3, $4::jsonb, NOW()) + ON CONFLICT (name) DO UPDATE + SET heartbeat_at=NOW(), + status=EXCLUDED.status, + current_job_id=EXCLUDED.current_job_id, + meta_json=EXCLUDED.meta_json, + updated_at=NOW() + """, + self.name, + status, + current_job_id, + json.dumps(meta or {}, ensure_ascii=False), + ) + except Exception: + return diff --git a/src/vk_parser_app/jobs.py b/src/vk_parser_app/jobs.py new file mode 100644 index 0000000..a432849 --- /dev/null +++ b/src/vk_parser_app/jobs.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from .constants import JOB_STATUS_IN_PROGRESS, JOB_STATUS_PENDING, JOB_STATUS_RETRY + + +async def is_worker_enabled(pool, name: str) -> bool: + value = await pool.fetchval("SELECT enabled FROM worker_controls WHERE name=$1", name) + return bool(value) + + +async def claim_job(pool, job_type: str, worker_id: str) -> dict | None: + row = await pool.fetchrow( + """ + WITH cte AS ( + SELECT id + FROM jobs + WHERE type=$1 + AND status IN ($2, $3) + AND next_run_at <= NOW() + ORDER BY next_run_at ASC, id ASC + FOR UPDATE SKIP LOCKED + LIMIT 1 + ) + UPDATE jobs j + SET status=$4, + locked_by=$5, + locked_at=NOW(), + updated_at=NOW() + FROM cte + WHERE j.id=cte.id + RETURNING j.* + """, + job_type, + JOB_STATUS_PENDING, + JOB_STATUS_RETRY, + JOB_STATUS_IN_PROGRESS, + worker_id, + ) + return dict(row) if row else None + + +async def ack_done(pool, job_id: int) -> None: + await pool.execute( + """ + UPDATE jobs + SET status='done', + locked_by=NULL, + locked_at=NULL, + last_error=NULL, + updated_at=NOW() + WHERE id=$1 + """, + job_id, + ) + + +async def ack_retry(pool, job: dict, error: str, delay_sec: int = 60) -> None: + attempt = int(job.get("attempts") or 0) + 1 + max_attempts = int(job.get("max_attempts") or 5) + status = "dead" if attempt >= max_attempts else "retry" + await pool.execute( + """ + UPDATE jobs + SET status=$2, + attempts=$3, + next_run_at=CASE WHEN $2='retry' THEN NOW() + ($4 * INTERVAL '1 second') ELSE next_run_at END, + locked_by=NULL, + locked_at=NULL, + last_error=$5, + updated_at=NOW() + WHERE id=$1 + """, + int(job["id"]), + status, + attempt, + int(delay_sec), + error[:1000], + ) + + +async def recover_stale_jobs(pool, job_type: str, stale_minutes: int = 20) -> int: + result = await pool.execute( + """ + UPDATE jobs + SET status='retry', + locked_by=NULL, + locked_at=NULL, + next_run_at=NOW(), + last_error=COALESCE(last_error, 'recovered stale lock'), + updated_at=NOW() + WHERE type=$1 + AND status='in_progress' + AND locked_at < NOW() - ($2 * INTERVAL '1 minute') + """, + job_type, + stale_minutes, + ) + return int(str(result).split()[-1]) diff --git a/src/vk_parser_app/main.py b/src/vk_parser_app/main.py new file mode 100644 index 0000000..8bb4ef6 --- /dev/null +++ b/src/vk_parser_app/main.py @@ -0,0 +1,3 @@ +from .admin import app + +__all__ = ["app"] diff --git a/src/vk_parser_app/security.py b/src/vk_parser_app/security.py new file mode 100644 index 0000000..d5b875a --- /dev/null +++ b/src/vk_parser_app/security.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import base64 +import hashlib +import secrets + + +def hash_password(password: str, iterations: int = 390_000) -> str: + salt = secrets.token_bytes(16) + digest = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, iterations) + return "pbkdf2_sha256${}${}${}".format( + iterations, + base64.urlsafe_b64encode(salt).decode("ascii"), + base64.urlsafe_b64encode(digest).decode("ascii"), + ) + + +def verify_password(password: str, encoded: str) -> bool: + try: + algo, iterations_raw, salt_raw, digest_raw = encoded.split("$", 3) + if algo != "pbkdf2_sha256": + return False + iterations = int(iterations_raw) + salt = base64.urlsafe_b64decode(salt_raw.encode("ascii")) + expected = base64.urlsafe_b64decode(digest_raw.encode("ascii")) + except Exception: + return False + actual = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, iterations) + return secrets.compare_digest(actual, expected) + + +def token_hash(token: str, secret: str) -> str: + return hashlib.sha256((secret + ":" + token).encode("utf-8")).hexdigest() + + +def new_token() -> str: + return secrets.token_urlsafe(32) diff --git a/src/vk_parser_app/templates/base.html b/src/vk_parser_app/templates/base.html new file mode 100644 index 0000000..7413b8a --- /dev/null +++ b/src/vk_parser_app/templates/base.html @@ -0,0 +1,229 @@ + + + + + + {{ title or "VK Parser Admin" }} + + + +{% if user %} +
+ +
+
+{% endif %} +
{% block body %}{% endblock %}
+ + diff --git a/src/vk_parser_app/templates/editor.html b/src/vk_parser_app/templates/editor.html new file mode 100644 index 0000000..3ac6778 --- /dev/null +++ b/src/vk_parser_app/templates/editor.html @@ -0,0 +1,338 @@ +{% extends "base.html" %} +{% block body %} +
+
+

Редакторская

+
Посты после AI-рерайта: быстрая проверка, правка и принятие.
+
+
+ {% for st in status_options %} + + {{ st.label }} {{ counts.get(st.value, 0) }} + + {% endfor %} +
+
+ +
+
+
+
Всего: {{ pagination.total }}
+
+ {% for item in preserved_query %} + {% if item.key != "per_page" and item.key != "sort" %} + + {% endif %} + {% endfor %} + + +
+
+ +
+ {% for p in posts %} +
+
+
+ {% for m in p.media_items %} + {% if m.type == "photo" and m.url %} + + {% elif m.url %} + ▶ + {% endif %} + {% endfor %} +
+
{{ p.media_count or 0 }} медиа
+
+ +
+
+ {{ p.source_name }} + #{{ p.review_source_tag or p.source_tag or "source" }} + {{ p.posted_at_fmt or p.created_at_fmt }} +
+
+ + {{ p.editorial_status_label }} + + {{ p.review_category or "без категории" }} + AI {{ p.qualification_score or "?" }}/10 +
+
{{ p.review_text }}
+
+ Исходник и AI-заметки +
+
+
Оригинал
+
{{ p.raw_text }}
+
+
+
Заметки
+
{{ p.rewrite_notes or p.qualification_reason or "—" }}
+
+
+
+
+ +
+
+ + +
+ +
+ + +
+
+
+ + +
+
+

Пост #{{ p.id }}

+
{{ p.source_name }} · {{ p.posted_at_fmt or p.created_at_fmt }}
+
+ +
+
+ + + +
+ +
+ + +
+
#{{ p.review_category_tag or p.review_category or "category" }} #{{ p.review_source_tag or p.source_tag or "source" }}
+ +
+ + +
+
+
+ + {% else %} +
В этой очереди пока нет постов.
+ {% endfor %} +
+ +{% include "pagination.html" %} +
+ + +
+ + + + +{% endblock %} diff --git a/src/vk_parser_app/templates/login.html b/src/vk_parser_app/templates/login.html new file mode 100644 index 0000000..9dd4d56 --- /dev/null +++ b/src/vk_parser_app/templates/login.html @@ -0,0 +1,12 @@ +{% extends "base.html" %} +{% block body %} +
+

Вход

+ {% if error %}

{{ error }}

{% endif %} +
+ + + +
+
+{% endblock %} diff --git a/src/vk_parser_app/templates/pagination.html b/src/vk_parser_app/templates/pagination.html new file mode 100644 index 0000000..9af310f --- /dev/null +++ b/src/vk_parser_app/templates/pagination.html @@ -0,0 +1,37 @@ +{% if pagination and pagination.pages > 1 %} + +{% else %} + {% if pagination %}{% endif %} +{% endif %} diff --git a/src/vk_parser_app/templates/raw_post_detail.html b/src/vk_parser_app/templates/raw_post_detail.html new file mode 100644 index 0000000..6dd9ae8 --- /dev/null +++ b/src/vk_parser_app/templates/raw_post_detail.html @@ -0,0 +1,194 @@ +{% extends "base.html" %} +{% block body %} +
+
+ ← Raw-посты +

Raw #{{ post.id }}

+
+ {{ post.source_name }} + · {{ post.source_platform }} + · #{{ post.source_tag or "—" }} + {% if post.posted_at_fmt %}· пост VK: {{ post.posted_at_fmt }}{% endif %} +
+
+ {{ post.stage.label }} +
+ +
+
+
+ {% for m in post.media_items %} + {% if m.type == "photo" and m.url %} + + {% elif m.url %} + + ▶ + + {% endif %} + {% endfor %} +
+ {% if post.media_count %}
{{ post.media_count }} медиа
{% endif %} +
{{ post.raw_text }}
+
Загружен парсером: {{ post.created_at_fmt }}
+
+ + +
+ +
+
+
+

Рерайт

+
Готовый текст для будущей редакторской проверки.
+
+ + {{ post.rewrite_status or "pending" }} + +
+ {% if post.rewritten_text %} +
{{ post.rewritten_text }}
+ {% else %} +
Рерайт ещё не готов.
+ {% endif %} +
+
Категория
{{ post.rewrite_category or "—" }}
+
Хэштеги
+
+ {% if post.rewrite_category_tag or post.rewrite_source_tag %} + #{{ post.rewrite_category_tag or "—" }} #{{ post.rewrite_source_tag or "—" }} + {% else %} + — + {% endif %} +
+
Заметка
{{ post.rewrite_notes or "—" }}
+
Модель
{{ post.rewrite_model or post.writer_batch_model or "—" }}
+
Готов
{{ post.rewritten_at_fmt or "—" }}
+
Batch
{% if post.rewrite_batch_id %}#{{ post.rewrite_batch_id }}{% else %}—{% endif %}
+
Токены
{{ post.writer_total_tokens or "—" }}
+
Стоимость
{% if post.writer_estimated_cost_usd %}${{ post.writer_estimated_cost_usd }}{% else %}—{% endif %}
+
+
+ +{% if post.qualification_batch_id %} +
+ Технический ответ AI +
+
Provider
{{ post.qualification_provider or "—" }}
+
Batch status
{{ post.batch_status or "—" }}
+
Создан
{{ post.batch_created_at_fmt or "—" }}
+
Завершён
{{ post.batch_completed_at_fmt or "—" }}
+
Постов в batch
{{ post.batch_posts_count or "—" }}
+
Accepted / rejected / maybe
+
{{ post.batch_accepted_count or 0 }} / {{ post.batch_rejected_count or 0 }} / {{ post.batch_maybe_count or 0 }}
+
Токены
+
{{ post.batch_prompt_tokens or 0 }} / {{ post.batch_completion_tokens or 0 }} / {{ post.batch_total_tokens or 0 }}
+
Стоимость
{% if post.batch_estimated_cost_usd %}${{ post.batch_estimated_cost_usd }}{% else %}—{% endif %}
+
Ошибка
{{ post.batch_error or "—" }}
+
+ {% if post.batch_prompt_text %} +
+ Использованный prompt квалификатора + 
{{ post.batch_prompt_text }}
+
+ {% endif %} + 
{{ post.batch_response_pretty or "{}" }}
+
+{% endif %} + +{% if post.rewrite_batch_id %} +
+ Технический ответ AI-райтера +
+
Provider
{{ post.rewrite_provider or "—" }}
+
Batch status
{{ post.writer_batch_status or "—" }}
+
Создан
{{ post.writer_batch_created_at_fmt or "—" }}
+
Завершён
{{ post.writer_batch_completed_at_fmt or "—" }}
+
Постов в batch
{{ post.writer_batch_posts_count or "—" }}
+
Ready / failed
{{ post.writer_batch_ready_count or 0 }} / {{ post.writer_batch_failed_count or 0 }}
+
Токены
+
{{ post.writer_prompt_tokens or 0 }} / {{ post.writer_completion_tokens or 0 }} / {{ post.writer_total_tokens or 0 }}
+
Стоимость
{% if post.writer_estimated_cost_usd %}${{ post.writer_estimated_cost_usd }}{% else %}—{% endif %}
+
Ошибка
{{ post.writer_batch_error or "—" }}
+
+ {% if post.writer_batch_prompt_text %} +
+ Использованный prompt райтера + 
{{ post.writer_batch_prompt_text }}
+
+ {% endif %} + 
{{ post.writer_batch_response_pretty or "{}" }}
+
+{% endif %} + + + +{% endblock %} diff --git a/src/vk_parser_app/templates/raw_posts.html b/src/vk_parser_app/templates/raw_posts.html new file mode 100644 index 0000000..99075a5 --- /dev/null +++ b/src/vk_parser_app/templates/raw_posts.html @@ -0,0 +1,211 @@ +{% extends "base.html" %} +{% block body %} +
+
+

Raw-посты

+
Исходный пост, этап обработки и AI-оценка.
+
+
+
+
+
+
Всего: {{ pagination.total }}
+
+ + + + + + + + + {% for p in posts %} + + + + + + + + {% endfor %} +
#ИсточникЭтапПостAI
{{ p.id }} + {{ p.source_name }} +
{{ p.source_platform }} · #{{ p.source_tag or "—" }}
+ {% if p.posted_at_fmt %}
{{ p.posted_at_fmt }}
{% endif %} + 		+ {{ p.stage.label }} + {% if p.error_reason %}
{{ p.error_reason }}
{% endif %} + 		+
+ {% for m in p.media_items %} + {% if m.type == "photo" and m.url %} + + {% elif m.url %} + + ▶ + + {% endif %} + {% endfor %} +
+ {% if p.media_count %}
{{ p.media_count }} медиа
{% endif %} + {% if p.raw_text|length > 520 %} +
+ {{ p.raw_text[:520] }}... +
{{ p.raw_text }}
+
+ {% else %} +
{{ p.raw_text }}
+ {% endif %} +
Загружен парсером: {{ p.created_at_fmt }}
+ 		+ + {{ p.ai_badge.label }} + итог: {{ p.ai_badge.sublabel }} + + {% if p.qualification_model_decision %}
модель: {{ p.qualification_model_decision }}
{% endif %} + {% if p.rewrite_category %}
{{ p.rewrite_category }}
{% endif %} +
+
+
+ {% for item in preserved_query %} + {% if item.key != "per_page" %} + + {% endif %} + {% endfor %} + +
+ {% include "pagination.html" %} +
+
+ + +
+ + +{% endblock %} diff --git a/src/vk_parser_app/templates/source_form.html b/src/vk_parser_app/templates/source_form.html new file mode 100644 index 0000000..cf80ad5 --- /dev/null +++ b/src/vk_parser_app/templates/source_form.html @@ -0,0 +1,25 @@ +{% extends "base.html" %} +{% block body %} +
+

{{ title }}

+ Назад +
+
+
+ +
+ + + + + + +
+ +
+
+{% endblock %} diff --git a/src/vk_parser_app/templates/sources.html b/src/vk_parser_app/templates/sources.html new file mode 100644 index 0000000..6a632eb --- /dev/null +++ b/src/vk_parser_app/templates/sources.html @@ -0,0 +1,123 @@ +{% extends "base.html" %} +{% block body %} +
+
+

Источники

+
VK-источники для парсинга. Название идёт в prompt, тэг — в будущие хэштеги.
+
+
+ +
+ Добавить источники +
+
+ + + + + + + + +
+ +
+ + + + + + + +
+
+ + {% if source_preview %} +
+

Предпросмотр

+ + + {% for item in source_preview %} + + + + + + + + + {% endfor %} +
#НазваниеТэгСсылкаVK idПроверка
{{ item.line_no }}{{ item.name or "—" }}#{{ item.tag or "—" }}{{ item.url }}{{ item.external_id }}
{{ item.external_owner_id or "" }}		{% if item.ok %}готов{% else %}{{ item.error }}{% endif %}
+
+ + + + Строки с ошибками и дубли будут пропущены. +
+
+ {% endif %} +
+ +
+
+ + + + {% include "table_tools.html" %} + +
+
+ +
+ + + + + {% for s in sources %} + + + + + + + + + + + + {% endfor %} +
#ПлощадкаНазваниеТэгСсылкаСтатусПостыПоследний парсинг
{{ s.id }}{{ s.platform }} + {{ s.name }} + {% if s.active %}on{% else %}off{% endif %} +
{{ s.external_id or "" }}
+ 		#{{ s.tag or "—" }}{{ s.url }}{{ s.status }}{% if s.status_msg %}
{{ s.status_msg }}
{% endif %}		{{ s.posts_count }} +{{ s.posts_24h }}/24ч{{ s.last_parsed_at_fmt or "нет" }} +
+ ✎ +
+ + +
+
+ + +
+
+
+ {% include "pagination.html" %} +
+{% endblock %} diff --git a/src/vk_parser_app/templates/table_tools.html b/src/vk_parser_app/templates/table_tools.html new file mode 100644 index 0000000..4e713c2 --- /dev/null +++ b/src/vk_parser_app/templates/table_tools.html @@ -0,0 +1,43 @@ +
+ Расширенные фильтры и сортировка +
+
+
Фильтры
+ {% for item in table_filters %} +
+ + + +
+ {% endfor %} +
Пустые строки игнорируются. Для дат используй формат YYYY-MM-DD, для true/false — true или false.
+
+
+
Сортировка
+ {% for item in table_sorts %} +
+ + +
+ {% endfor %} +
Сортировки применяются сверху вниз. Если сортировка задана здесь, обычная сортировка страницы становится запасной.
+
+
+
diff --git a/src/vk_parser_app/templates/users.html b/src/vk_parser_app/templates/users.html new file mode 100644 index 0000000..444e87f --- /dev/null +++ b/src/vk_parser_app/templates/users.html @@ -0,0 +1,35 @@ +{% extends "base.html" %} +{% block body %} +

Пользователи

+
+

Пользователи

+ + + {% for u in users %} + + + + + + + {% endfor %} +
IDЛогинРольСозданАктивен
{{ u.id }}{{ u.login }}{{ u.role }}{{ u.created_at_fmt }}{% if u.is_active %}yes{% else %}no{% endif %}{% if user.role == "admin" and user.id != u.id %}
{% endif %}
+ {% include "pagination.html" %} +
+{% if user.role == "admin" %} +
+

Добавить

+
+ +
+ + + +
+ +
+
+{% endif %} +{% endblock %} diff --git a/src/vk_parser_app/templates/vk_oauth_callback.html b/src/vk_parser_app/templates/vk_oauth_callback.html new file mode 100644 index 0000000..64977cc --- /dev/null +++ b/src/vk_parser_app/templates/vk_oauth_callback.html @@ -0,0 +1,52 @@ +{% extends "base.html" %} +{% block body %} +
+

VK OAuth callback

+ {% if code %} +

Скопируйте code и code_verifier для обмена на access token.

+ + + + + {% if device_id %} + + + {% endif %} + {% if state and expected_state and state != expected_state %} +

state не совпал, такой code лучше не использовать.

+ {% endif %} + {% elif error %} +

{{ error }}

+

{{ error_description }}

+ {% else %} +

VK вернул callback без code.

+ {% endif %} + + +
+ +{% endblock %} diff --git a/src/vk_parser_app/templates/workers.html b/src/vk_parser_app/templates/workers.html new file mode 100644 index 0000000..3834511 --- /dev/null +++ b/src/vk_parser_app/templates/workers.html @@ -0,0 +1,141 @@ +{% extends "base.html" %} +{% block body %} +

Воркеры

+
+ + + {% for w in workers %} + + + + + + + + + {% endfor %} +
ИмяEnabledHeartbeatСтатусJob
{{ w.name }}{% if w.enabled %}enabled{% else %}disabled{% endif %}{{ w.heartbeat_at or "нет" }}{{ w.status or "" }}{{ w.current_job_id or "" }}
+
+ +
+ Категории публикаций +

Активные категории попадают в выпадающий список редактора и передаются AI-райтеру. Удаление здесь выключает категорию, но не ломает старые посты.

+
+ + + + +
+ + + {% for c in categories %} + + + + + + + + {% endfor %} +
НазваниеТэгПорядокСтатус
+
+ + +
+ 		{% if c.is_active %}активна{% else %}выключена{% endif %} + +
+ + +
+
+ + +
+
+
+ +

Настройки

+{% for category, rows in settings|groupby("category") %} +
+ {{ category_titles.get(category, category) }} + + + {% for s in rows %} + + + + + + {% endfor %} +
КлючЗначениеОписание
{{ s.key }}
{{ s.title }}
 +
+ + + {% if s.value_type == "bool" %} + + {% elif s.key in ["ai_qualifier_provider", "ai_writer_provider"] %} + + {% elif s.key == "ai_qualifier_model" %} + + {% elif s.key == "ai_writer_model" %} + + {% elif s.value_type == "secret" %} +
+ +
Ключ показан полностью для удобства. Не открывай эту страницу в демонстрации экрана.
+
+ {% elif s.value_type == "text" %} +
+ {% if prompt_hints and s.key in prompt_hints %} +
+ {{ prompt_hints[s.key].title }} +
{{ prompt_hints[s.key].body }}
+
+
+
Как безопасно менять
+
{{ prompt_hints[s.key].safe }}
+
+
+
Что приходит на вход
+
{{ prompt_hints[s.key].input }}
+
+
+
+ Базовая форма/пример + 
{{ prompt_hints[s.key].contract }}
+
+
+ {% endif %} + +
+ {% else %} + + {% endif %} + +
+ 		{{ s.description }}
+
+{% endfor %} +{% endblock %} diff --git a/src/vk_parser_app/text_utils.py b/src/vk_parser_app/text_utils.py new file mode 100644 index 0000000..0066630 --- /dev/null +++ b/src/vk_parser_app/text_utils.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import re + + +def normalize_hash_tag(value: str, fallback: str = "source") -> str: + tag = (value or "").strip().lstrip("#").lower() + tag = re.sub(r"\s+", "_", tag) + tag = re.sub(r"[^\wа-яё_]+", "_", tag, flags=re.IGNORECASE) + tag = re.sub(r"_+", "_", tag).strip("_") + return tag or fallback + + +def parse_categories(value: object) -> list[str]: + if isinstance(value, list): + raw_items = [str(item) for item in value] + else: + text = str(value or "") + raw_items = re.split(r"[\n,|]+", text) + categories: list[str] = [] + seen: set[str] = set() + for item in raw_items: + category = item.strip().strip("#") + key = category.lower() + if category and key not in seen: + categories.append(category) + seen.add(key) + return categories diff --git a/src/vk_parser_app/vk_api.py b/src/vk_parser_app/vk_api.py new file mode 100644 index 0000000..4569e54 --- /dev/null +++ b/src/vk_parser_app/vk_api.py @@ -0,0 +1,290 @@ +from __future__ import annotations + +import asyncio +import json +import re +import time +from dataclasses import dataclass +from typing import Any + +import aiohttp +from loguru import logger + +from .config import settings + + +class VKAPIError(RuntimeError): + def __init__(self, code: int | None, message: str) -> None: + self.code = code + super().__init__(f"VK API error {code}: {message}") + + +class VKRateLimiter: + def __init__(self, rps: int = 3) -> None: + self.rps = max(1, int(rps)) + self.interval = 1.0 / self.rps + self._last = 0.0 + self._lock = asyncio.Lock() + + async def acquire(self) -> None: + async with self._lock: + now = time.monotonic() + wait = self.interval - (now - self._last) + if wait > 0: + await asyncio.sleep(wait) + self._last = time.monotonic() + + +class VKAPIClient: + base_url = "https://api.vk.com/method" + + def __init__( + self, + token: str | None = None, + version: str | None = None, + rps: int = 3, + timeout_total_sec: int = 60, + timeout_connect_sec: int = 10, + rate_limit_sleep_sec: float = 1.0, + retry_attempts: int = 3, + retry_min_delay_sec: float = 2.0, + retry_max_delay_sec: float = 10.0, + ) -> None: + self.token = token or settings.vk_access_token + self.version = version or settings.vk_api_version + self.limiter = VKRateLimiter(rps) + self.timeout_total_sec = max(1, int(timeout_total_sec)) + self.timeout_connect_sec = max(1, int(timeout_connect_sec)) + self.rate_limit_sleep_sec = max(0.1, float(rate_limit_sleep_sec)) + self.retry_attempts = max(1, int(retry_attempts)) + self.retry_min_delay_sec = max(0.1, float(retry_min_delay_sec)) + self.retry_max_delay_sec = max(self.retry_min_delay_sec, float(retry_max_delay_sec)) + self.session: aiohttp.ClientSession | None = None + + async def __aenter__(self) -> "VKAPIClient": + self.session = aiohttp.ClientSession( + timeout=aiohttp.ClientTimeout(total=self.timeout_total_sec, connect=self.timeout_connect_sec) + ) + return self + + async def __aexit__(self, *args) -> None: + if self.session: + await self.session.close() + + async def call(self, method: str, **params: Any) -> dict | list: + if not self.session: + raise RuntimeError("VKAPIClient is not initialized") + if not self.token: + raise RuntimeError("VK_ACCESS_TOKEN is empty") + + payload = dict(params) + payload["access_token"] = self.token + payload["v"] = self.version + + for attempt in range(1, self.retry_attempts + 1): + try: + await self.limiter.acquire() + async with self.session.post(f"{self.base_url}/{method}", data=payload) as resp: + resp.raise_for_status() + data = await resp.json(content_type=None) + if "error" not in data: + return data.get("response", {}) + + err = data["error"] + code = err.get("error_code") + message = err.get("error_msg", "unknown") + if code == 6 and attempt < self.retry_attempts: + logger.warning("VK rate limit hit, sleeping {}s", self.rate_limit_sleep_sec) + await asyncio.sleep(self.rate_limit_sleep_sec) + continue + raise VKAPIError(code, message) + except VKAPIError: + raise + except Exception: + if attempt >= self.retry_attempts: + raise + delay = min(self.retry_min_delay_sec * (2 ** (attempt - 1)), self.retry_max_delay_sec) + await asyncio.sleep(delay) + + raise VKAPIError(None, "retry exhausted") + + async def resolve_group(self, input_value: str) -> tuple[str, int, str]: + external_id = normalize_vk_source(input_value) + if external_id.lstrip("-").isdigit(): + group_id = abs(int(external_id)) + response = await self.call("groups.getById", group_id=str(group_id)) + else: + response = await self.call("groups.getById", group_id=external_id) + items = response if isinstance(response, list) else response.get("groups", []) + if not items: + raise VKAPIError(None, f"cannot resolve VK group: {input_value}") + group = items[0] + group_id = int(group["id"]) + screen_name = str(group.get("screen_name") or external_id) + name = str(group.get("name") or screen_name) + return screen_name, -group_id, name + + async def get_wall_posts(self, owner_id: int, count: int, offset: int = 0) -> dict: + response = await self.call("wall.get", owner_id=owner_id, count=count, offset=offset, filter="owner") + if not isinstance(response, dict): + raise VKAPIError(None, "wall.get returned non-object response") + return response + + async def get_wall_upload_server(self, group_id: int) -> str: + response = await self.call("photos.getWallUploadServer", group_id=abs(int(group_id))) + if not isinstance(response, dict) or not response.get("upload_url"): + raise VKAPIError(None, "photos.getWallUploadServer returned no upload_url") + return str(response["upload_url"]) + + async def upload_wall_photo_bytes(self, group_id: int, data: bytes, filename: str = "photo.jpg") -> dict: + if not self.session: + raise RuntimeError("VKAPIClient is not initialized") + upload_url = await self.get_wall_upload_server(group_id) + form = aiohttp.FormData() + form.add_field("photo", data, filename=filename, content_type="image/jpeg") + async with self.session.post(upload_url, data=form) as resp: + uploaded = await resp.json(content_type=None) + saved = await self.call( + "photos.saveWallPhoto", + group_id=abs(int(group_id)), + photo=uploaded.get("photo"), + server=uploaded.get("server"), + hash=uploaded.get("hash"), + ) + if not isinstance(saved, list) or not saved: + raise VKAPIError(None, f"photos.saveWallPhoto returned invalid response: {json.dumps(saved)[:300]}") + return saved[0] + + async def create_wall_post( + self, + owner_id: int, + message: str, + attachments: list[str], + from_group: bool = True, + ) -> int: + response = await self.call( + "wall.post", + owner_id=int(owner_id), + from_group=1 if from_group else 0, + message=message or "", + attachments=",".join(attachments) if attachments else "", + ) + if not isinstance(response, dict) or "post_id" not in response: + raise VKAPIError(None, f"wall.post returned invalid response: {response}") + return int(response["post_id"]) + + +def normalize_vk_source(value: str) -> str: + raw = str(value or "").strip() + if not raw: + return "" + if "vk.com" in raw: + raw = raw.split("vk.com", 1)[1] + raw = raw.lstrip("/") + raw = raw.split("?", 1)[0].split("#", 1)[0].strip() + raw = re.sub(r"^(club|public)", "", raw, flags=re.IGNORECASE) + return raw.lower() + + +def is_repost(post: dict) -> bool: + return bool(post.get("copy_history")) + + +def is_deleted_or_invalid(post: dict) -> bool: + if post.get("is_deleted") or post.get("deleted"): + return True + if not post.get("id") or not post.get("date"): + return True + return False + + +def is_fatal_source_error(error: Exception | str) -> bool: + message = str(error or "").lower() + fatal_patterns = ( + "vk api error 15:", + "vk api error 18:", + "vk api error 19:", + "vk api error 100:", + "vk api error 113:", + "vk api error 1051:", + "vk api error 200:", + "vk api error 201:", + "vk api error 203:", + "[15]", + "[18]", + "[19]", + "[100]", + "[113]", + "[1051]", + "[200]", + "[201]", + "[203]", + "access denied", + "private profile", + "cannot resolve vk group", + "cannot resolve screen_name", + "method is unavailable with current profile type", + "wall is disabled", + ) + return any(pattern in message for pattern in fatal_patterns) + + +def post_vk_url(owner_id: int, post_id: int | str) -> str: + return f"https://vk.com/wall{int(owner_id)}_{post_id}" + + +@dataclass +class ExtractedMedia: + media_type: str + original_url: str | None + attachment_id: str + width: int | None = None + height: int | None = None + duration_sec: int | None = None + sort_order: int = 0 + + +def extract_media(post: dict) -> list[ExtractedMedia]: + out: list[ExtractedMedia] = [] + for idx, att in enumerate(post.get("attachments", []) or []): + att_type = att.get("type") + if att_type == "photo": + photo = att.get("photo") or {} + sizes = sorted( + photo.get("sizes", []) or [], + key=lambda s: int(s.get("width") or 0) * int(s.get("height") or 0), + reverse=True, + ) + best = sizes[0] if sizes else {} + owner_id = photo.get("owner_id") + media_id = photo.get("id") + if owner_id is None or media_id is None: + continue + out.append( + ExtractedMedia( + media_type="photo", + original_url=best.get("url"), + attachment_id=f"photo{owner_id}_{media_id}", + width=best.get("width"), + height=best.get("height"), + sort_order=idx, + ) + ) + elif att_type == "video": + video = att.get("video") or {} + owner_id = video.get("owner_id") + media_id = video.get("id") + if owner_id is None or media_id is None: + continue + out.append( + ExtractedMedia( + media_type="video", + original_url=f"https://vk.com/video{owner_id}_{media_id}", + attachment_id=f"video{owner_id}_{media_id}", + width=video.get("width"), + height=video.get("height"), + duration_sec=video.get("duration"), + sort_order=idx, + ) + ) + return out diff --git a/src/vk_parser_app/workers/__init__.py b/src/vk_parser_app/workers/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/vk_parser_app/workers/__init__.py @@ -0,0 +1 @@ + diff --git a/src/vk_parser_app/workers/ai_qualifier.py b/src/vk_parser_app/workers/ai_qualifier.py new file mode 100644 index 0000000..a7f34a0 --- /dev/null +++ b/src/vk_parser_app/workers/ai_qualifier.py @@ -0,0 +1,409 @@ +from __future__ import annotations + +import asyncio +import hashlib +import json +from datetime import datetime, timezone +from decimal import Decimal +from typing import Any + +import litellm +from loguru import logger + +from ..config import settings +from ..constants import WORKER_AI_QUALIFIER +from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool +from ..heartbeat import HeartbeatReporter +from ..jobs import is_worker_enabled + + +def now_utc() -> datetime: + return datetime.now(timezone.utc) + + +def prompt_hash(prompt: str) -> str: + return hashlib.sha256((prompt or "").encode("utf-8")).hexdigest() + + +def normalize_prompt(prompt: str) -> str: + return (prompt or "").replace("\\r\\n", "\n").replace("\\n", "\n").strip() + + +DEFAULT_CONTRACT_PROMPT = """ +OUTPUT SCHEMA (return array matching input order): +{"results":[{"id":123,"score":8,"decision":"accepted","reason":"до 10 слов на русском","reject_tag":null}]} + +Rules: +- Input is a JSON array of posts. +- Return JSON only. No markdown. No text outside JSON. +- score must be integer 1..10. +- decision must be exactly one of: "accepted", "rejected", "maybe". +- reject_tag (rejected/maybe only) must be one of: "meme", "no_product", "politics", "discount_only", "off_topic", "vacancy", "low_content", "wrong_language", "injection", "weapon", null. +- Return one result for every input post id. +""" + + +def build_system_prompt(user_prompt: str, contract_prompt: str) -> str: + parts = [normalize_prompt(user_prompt), normalize_prompt(contract_prompt or DEFAULT_CONTRACT_PROMPT)] + return "\n\n".join(part for part in parts if part).strip() + + +def normalize_model(provider: str, model: str) -> str: + provider = (provider or "").strip().lower() + model = (model or "").strip() + if not model: + return model + if "/" in model: + return model + if provider in {"openrouter", "anthropic", "gemini", "vertex_ai", "bedrock"}: + return f"{provider}/{model}" + return model + + +def parse_ai_json(content: str) -> dict: + raw = (content or "").strip() + if raw.startswith("```"): + raw = raw.strip("`") + if raw.lower().startswith("json"): + raw = raw[4:].strip() + return json.loads(raw) + + +def response_usage(response: Any) -> dict[str, Any]: + usage = getattr(response, "usage", None) + if usage is None and isinstance(response, dict): + usage = response.get("usage") + + def get(name: str) -> int | None: + if usage is None: + return None + value = getattr(usage, name, None) + if value is None and isinstance(usage, dict): + value = usage.get(name) + try: + return int(value) if value is not None else None + except Exception: + return None + + cost = getattr(response, "_hidden_params", None) + if isinstance(cost, dict): + cost = cost.get("response_cost") + else: + cost = None + try: + cost_value = Decimal(str(cost)) if cost is not None else None + except Exception: + cost_value = None + + return { + "prompt_tokens": get("prompt_tokens"), + "completion_tokens": get("completion_tokens"), + "total_tokens": get("total_tokens"), + "estimated_cost_usd": cost_value, + } + + +def validate_results(data: dict, expected_ids: set[int], min_score: int) -> list[dict]: + results = data.get("results") + if not isinstance(results, list) and isinstance(data.get("data"), dict): + results = data["data"].get("results") + if not isinstance(results, list): + raise ValueError("AI response has no results list") + + out: list[dict] = [] + seen: set[int] = set() + for item in results: + if not isinstance(item, dict): + continue + post_id = int(item.get("id")) + if post_id not in expected_ids: + raise ValueError(f"AI returned unexpected id={post_id}") + score = max(1, min(10, int(item.get("score")))) + decision = str(item.get("decision") or "").strip().lower() + if decision not in {"accepted", "rejected", "maybe"}: + decision = "accepted" if score >= min_score else "rejected" + reason = str(item.get("reason") or "").strip()[:1000] + reject_tag = item.get("reject_tag") + reject_tag = str(reject_tag).strip().lower()[:80] if reject_tag not in {None, ""} else None + seen.add(post_id) + out.append({"id": post_id, "score": score, "decision": decision, "reason": reason, "reject_tag": reject_tag}) + + missing = expected_ids - seen + if missing: + raise ValueError(f"AI response missing ids: {sorted(missing)[:10]}") + return out + + +class AIQualifierWorker: + def __init__(self) -> None: + self.pool = None + self.heartbeat = HeartbeatReporter(WORKER_AI_QUALIFIER, 30) + + async def init(self) -> None: + self.pool = await get_pool() + + async def claim_posts(self, batch_size: int) -> list[dict]: + async with self.pool.acquire() as conn: + async with conn.transaction(): + count = await conn.fetchval( + """ + SELECT COUNT(*) + FROM raw_posts + WHERE status='storage_ready' + AND COALESCE(qualification_status, 'pending') IN ('pending', 'failed') + """ + ) + if int(count or 0) < batch_size: + return [] + + rows = await conn.fetch( + """ + WITH cte AS ( + SELECT id + FROM raw_posts + WHERE status='storage_ready' + AND COALESCE(qualification_status, 'pending') IN ('pending', 'failed') + ORDER BY created_at ASC, id ASC + FOR UPDATE SKIP LOCKED + LIMIT $1 + ) + UPDATE raw_posts rp + SET qualification_status='processing', + updated_at=NOW() + FROM cte + WHERE rp.id=cte.id + RETURNING rp.id, rp.raw_text, rp.original_url, rp.storage_post_url, rp.created_at, + rp.source_id, + (SELECT s.name FROM sources s WHERE s.id=rp.source_id) AS source_name, + (SELECT COUNT(*) FROM raw_post_media m WHERE m.raw_post_id=rp.id) AS media_count, + (SELECT ARRAY_AGG(DISTINCT m.media_type ORDER BY m.media_type) + FROM raw_post_media m WHERE m.raw_post_id=rp.id) AS media_types + """, + batch_size, + ) + return [dict(r) for r in rows] + + async def create_batch(self, provider: str, model: str, prompt: str, payload: dict) -> int: + return int( + await self.pool.fetchval( + """ + INSERT INTO ai_qualification_batches(provider, model, prompt_hash, prompt_text, posts_count, request_json) + VALUES($1, $2, $3, $4, $5, $6::jsonb) + RETURNING id + """, + provider, + model, + prompt_hash(prompt), + prompt, + len(payload), + json.dumps(payload, ensure_ascii=False), + ) + ) + + async def finish_batch( + self, + batch_id: int, + status: str, + response: Any = None, + error: str | None = None, + usage: dict[str, Any] | None = None, + ) -> None: + counts = {"accepted": 0, "rejected": 0, "maybe": 0} + if isinstance(response, dict) and isinstance(response.get("results"), list): + for item in response["results"]: + decision = str(item.get("decision") or "") + if decision in counts: + counts[decision] += 1 + usage = usage or {} + await self.pool.execute( + """ + UPDATE ai_qualification_batches + SET status=$2, + response_json=$3::jsonb, + error=$4, + accepted_count=$5, + rejected_count=$6, + maybe_count=$7, + prompt_tokens=$8, + completion_tokens=$9, + total_tokens=$10, + estimated_cost_usd=$11, + completed_at=NOW() + WHERE id=$1 + """, + batch_id, + status, + json.dumps(response, ensure_ascii=False) if response is not None else None, + error[:2000] if error else None, + counts["accepted"], + counts["rejected"], + counts["maybe"], + usage.get("prompt_tokens"), + usage.get("completion_tokens"), + usage.get("total_tokens"), + usage.get("estimated_cost_usd"), + ) + + async def mark_posts_failed(self, post_ids: list[int], error: str) -> None: + if not post_ids: + return + await self.pool.execute( + """ + UPDATE raw_posts + SET qualification_status='failed', + qualification_reason=$2, + updated_at=NOW() + WHERE id=ANY($1::bigint[]) + """, + post_ids, + error[:1000], + ) + + async def apply_results(self, batch_id: int, results: list[dict], model: str, prompt: str, min_score: int) -> None: + for item in results: + score = int(item["score"]) + decision = str(item["decision"]) + model_decision = decision + if score >= min_score and decision == "maybe": + decision = "accepted" + if score < min_score and decision == "accepted": + decision = "rejected" + status = "accepted" if score >= min_score and decision == "accepted" else "rejected" + await self.pool.execute( + """ + UPDATE raw_posts + SET qualification_status=$2, + qualification_score=$3, + qualification_decision=$4, + qualification_reason=$5, + qualification_model=$6, + qualification_prompt_hash=$7, + qualification_batch_id=$8, + qualification_reject_tag=$9, + qualification_model_decision=$10, + qualified_at=NOW(), + updated_at=NOW() + WHERE id=$1 + """, + int(item["id"]), + status, + score, + decision, + item["reason"], + model, + prompt_hash(prompt), + batch_id, + item.get("reject_tag"), + model_decision, + ) + + async def call_ai( + self, + provider: str, + model: str, + api_key: str, + api_base: str, + prompt: str, + payload: dict, + temperature: float, + timeout: int, + ) -> tuple[dict, dict[str, Any]]: + messages = [ + {"role": "system", "content": prompt}, + {"role": "user", "content": json.dumps(payload, ensure_ascii=False)}, + ] + kwargs: dict[str, Any] = { + "model": normalize_model(provider, model), + "messages": messages, + "temperature": temperature, + "timeout": timeout, + "response_format": {"type": "json_object"}, + } + if api_key: + kwargs["api_key"] = api_key + if api_base: + kwargs["api_base"] = api_base + response = await asyncio.to_thread(litellm.completion, **kwargs) + content = response.choices[0].message.content + return parse_ai_json(content), response_usage(response) + + async def run_once(self) -> bool: + enabled = await is_worker_enabled(self.pool, WORKER_AI_QUALIFIER) + setting_enabled = await fetch_bool_setting("ai_qualifier_enabled", False) + if not enabled or not setting_enabled: + await self.heartbeat.beat(self.pool, status="disabled", force=True) + return False + + provider = str(await fetch_setting("ai_qualifier_provider", "openrouter") or "openrouter") + model = str(await fetch_setting("ai_qualifier_model", "") or "").strip() + api_key = str(await fetch_setting("ai_qualifier_api_key", "") or "").strip() + api_base = str(await fetch_setting("ai_qualifier_api_base", "") or "").strip() + batch_size = max(1, await fetch_int_setting("ai_qualifier_batch_size", 30)) + min_score = max(1, min(10, await fetch_int_setting("ai_qualifier_min_score", 7))) + max_text_chars = max(100, await fetch_int_setting("ai_qualifier_max_text_chars", 2000)) + temperature = max(0.0, await fetch_float_setting("ai_qualifier_temperature", 0.0)) + timeout = max(10, await fetch_int_setting("ai_qualifier_timeout_sec", 120)) + prompt = build_system_prompt( + str(await fetch_setting("ai_qualifier_prompt", "") or ""), + str(await fetch_setting("ai_qualifier_contract", DEFAULT_CONTRACT_PROMPT) or DEFAULT_CONTRACT_PROMPT), + ) + + if not model or not api_key or not prompt: + await self.heartbeat.beat(self.pool, status="not_configured", force=True) + return False + + posts = await self.claim_posts(batch_size) + await self.heartbeat.beat(self.pool, meta={"claimed": len(posts), "batch_size": batch_size}) + if not posts: + return False + + payload = [ + { + "id": int(p["id"]), + "source": p.get("source_name") or "", + "original_url": p.get("original_url") or "", + "media_count": int(p.get("media_count") or 0), + "media_types": list(p.get("media_types") or []), + "text": str(p.get("raw_text") or "")[:max_text_chars], + } + for p in posts + ] + post_ids = [int(p["id"]) for p in posts] + batch_id = await self.create_batch(provider, model, prompt, payload) + try: + response, usage = await self.call_ai(provider, model, api_key, api_base, prompt, payload, temperature, timeout) + results = validate_results(response, set(post_ids), min_score) + await self.apply_results(batch_id, results, normalize_model(provider, model), prompt, min_score) + await self.finish_batch(batch_id, "done", {"results": results}, usage=usage) + logger.info("AI qualifier batch done: id={} posts={} usage={}", batch_id, len(results), usage) + return True + except Exception as exc: + await self.mark_posts_failed(post_ids, str(exc)) + await self.finish_batch(batch_id, "failed", error=str(exc)) + logger.exception("AI qualifier batch failed: id={} error={}", batch_id, exc) + return True + + async def run_loop(self) -> None: + await self.init() + worker_id = f"{WORKER_AI_QUALIFIER}" + logger.info("{} started", worker_id) + while True: + try: + had_work = await self.run_once() + except Exception as exc: + logger.exception("AI qualifier loop error: {}", exc) + had_work = False + interval = max(10, await fetch_int_setting("ai_qualifier_interval_sec", 60)) + if not had_work: + await asyncio.sleep(interval) + + +async def main() -> None: + logger.remove() + logger.add(lambda msg: print(msg, end=""), level=settings.log_level) + worker = AIQualifierWorker() + await worker.run_loop() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/vk_parser_app/workers/ai_writer.py b/src/vk_parser_app/workers/ai_writer.py new file mode 100644 index 0000000..0ebc4f5 --- /dev/null +++ b/src/vk_parser_app/workers/ai_writer.py @@ -0,0 +1,456 @@ +from __future__ import annotations + +import asyncio +import hashlib +import json +from datetime import datetime, timezone +from decimal import Decimal +from typing import Any + +import litellm +from loguru import logger + +from ..config import settings +from ..constants import WORKER_AI_WRITER +from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool +from ..heartbeat import HeartbeatReporter +from ..jobs import is_worker_enabled +from ..text_utils import normalize_hash_tag, parse_categories + + +def now_utc() -> datetime: + return datetime.now(timezone.utc) + + +def prompt_hash(prompt: str) -> str: + return hashlib.sha256((prompt or "").encode("utf-8")).hexdigest() + + +def normalize_prompt(prompt: str) -> str: + return (prompt or "").replace("\\r\\n", "\n").replace("\\n", "\n").strip() + + +DEFAULT_CONTRACT_PROMPT = """ +OUTPUT SCHEMA (return array matching input order): +{"rewrites":[{"id":123,"category":"разгрузка","text":"готовый текст без хэштегов","notes":"короткая заметка для редактора или null"}]} + +Rules: +- Input is a JSON object with key "posts" containing accepted posts. +- Each post includes producer_name and producer_tag. Use producer_name when it helps, but do not invent facts. +- Do not add hashtags to rewritten text. Pick exactly one category from the categories list in input. +- Return JSON only. No markdown. No text outside JSON. +- Return one rewrite for every input post id. +""" + + +def build_system_prompt(user_prompt: str, contract_prompt: str) -> str: + parts = [normalize_prompt(user_prompt), normalize_prompt(contract_prompt or DEFAULT_CONTRACT_PROMPT)] + return "\n\n".join(part for part in parts if part).strip() + + +def normalize_model(provider: str, model: str) -> str: + provider = (provider or "").strip().lower() + model = (model or "").strip() + if not model: + return model + if "/" in model: + return model + if provider in {"openrouter", "anthropic", "gemini", "vertex_ai", "bedrock"}: + return f"{provider}/{model}" + return model + + +def parse_ai_json(content: str) -> dict: + raw = (content or "").strip() + if raw.startswith("```"): + raw = raw.strip("`") + if raw.lower().startswith("json"): + raw = raw[4:].strip() + return json.loads(raw) + + +def response_usage(response: Any) -> dict[str, Any]: + usage = getattr(response, "usage", None) + if usage is None and isinstance(response, dict): + usage = response.get("usage") + + def get(name: str) -> int | None: + if usage is None: + return None + value = getattr(usage, name, None) + if value is None and isinstance(usage, dict): + value = usage.get(name) + try: + return int(value) if value is not None else None + except Exception: + return None + + cost = getattr(response, "_hidden_params", None) + if isinstance(cost, dict): + cost = cost.get("response_cost") + else: + cost = None + try: + cost_value = Decimal(str(cost)) if cost is not None else None + except Exception: + cost_value = None + + return { + "prompt_tokens": get("prompt_tokens"), + "completion_tokens": get("completion_tokens"), + "total_tokens": get("total_tokens"), + "estimated_cost_usd": cost_value, + } + + +def validate_rewrites( + data: dict, + expected_ids: set[int], + categories: list[str], + source_tags_by_id: dict[int, str], +) -> list[dict]: + rewrites = data.get("rewrites") + if not isinstance(rewrites, list) and isinstance(data.get("data"), dict): + rewrites = data["data"].get("rewrites") + if not isinstance(rewrites, list): + raise ValueError("AI response has no rewrites list") + out: list[dict] = [] + seen: set[int] = set() + for item in rewrites: + if not isinstance(item, dict): + continue + post_id = int(item.get("id")) + if post_id not in expected_ids: + raise ValueError(f"AI returned unexpected id={post_id}") + text = str(item.get("text") or "").strip() + if len(text) < 40: + raise ValueError(f"AI returned too short rewrite for id={post_id}") + category = str(item.get("category") or "").strip().strip("#") + category_map = {c.lower(): c for c in categories} + if category.lower() not in category_map: + raise ValueError(f"AI returned unknown category={category!r} for id={post_id}") + category = category_map[category.lower()] + notes = str(item.get("notes") or "").strip()[:1000] + seen.add(post_id) + out.append( + { + "id": post_id, + "category": category, + "category_tag": normalize_hash_tag(category, "category"), + "source_tag": normalize_hash_tag(source_tags_by_id.get(post_id) or "", "source"), + "text": text, + "notes": notes, + } + ) + missing = expected_ids - seen + if missing: + raise ValueError(f"AI response missing ids: {sorted(missing)[:10]}") + return out + + +async def load_writer_categories(pool) -> list[str]: + try: + rows = await pool.fetch( + """ + SELECT name + FROM content_categories + WHERE is_active=TRUE + ORDER BY sort_order, name + """ + ) + except Exception: + rows = [] + categories = [str(row["name"]) for row in rows] + if categories: + return categories + legacy = parse_categories(await fetch_setting("ai_writer_categories", [])) + return legacy or [ + "защита", + "одежда", + "разгрузка", + "рюкзаки", + "airsoft", + "патчи", + "электроника", + "аксессуары", + "производство", + ] + + +class AIWriterWorker: + def __init__(self) -> None: + self.pool = None + self.heartbeat = HeartbeatReporter(WORKER_AI_WRITER, 30) + + async def init(self) -> None: + self.pool = await get_pool() + + async def claim_posts(self, batch_size: int) -> list[dict]: + async with self.pool.acquire() as conn: + async with conn.transaction(): + count = await conn.fetchval( + """ + SELECT COUNT(*) + FROM raw_posts + WHERE status='storage_ready' + AND qualification_status='accepted' + AND COALESCE(rewrite_status, 'pending') IN ('pending', 'failed') + """ + ) + if int(count or 0) < batch_size: + return [] + + rows = await conn.fetch( + """ + WITH cte AS ( + SELECT id + FROM raw_posts + WHERE status='storage_ready' + AND qualification_status='accepted' + AND COALESCE(rewrite_status, 'pending') IN ('pending', 'failed') + ORDER BY qualified_at ASC NULLS LAST, id ASC + FOR UPDATE SKIP LOCKED + LIMIT $1 + ) + UPDATE raw_posts rp + SET rewrite_status='processing', + updated_at=NOW() + FROM cte + WHERE rp.id=cte.id + RETURNING rp.id, rp.raw_text, rp.original_url, rp.created_at, rp.qualification_score, + rp.qualification_reason, + (SELECT s.name FROM sources s WHERE s.id=rp.source_id) AS source_name, + (SELECT s.tag FROM sources s WHERE s.id=rp.source_id) AS source_tag, + (SELECT COUNT(*) FROM raw_post_media m WHERE m.raw_post_id=rp.id) AS media_count, + (SELECT ARRAY_AGG(DISTINCT m.media_type ORDER BY m.media_type) + FROM raw_post_media m WHERE m.raw_post_id=rp.id) AS media_types + """, + batch_size, + ) + return [dict(r) for r in rows] + + async def create_batch(self, provider: str, model: str, prompt: str, payload: dict) -> int: + return int( + await self.pool.fetchval( + """ + INSERT INTO ai_writer_batches(provider, model, prompt_hash, prompt_text, posts_count, request_json) + VALUES($1, $2, $3, $4, $5, $6::jsonb) + RETURNING id + """, + provider, + model, + prompt_hash(prompt), + prompt, + len(payload["posts"]), + json.dumps(payload, ensure_ascii=False), + ) + ) + + async def finish_batch( + self, + batch_id: int, + status: str, + response: Any = None, + error: str | None = None, + usage: dict[str, Any] | None = None, + ) -> None: + ready_count = 0 + if isinstance(response, dict) and isinstance(response.get("rewrites"), list): + ready_count = len(response["rewrites"]) + usage = usage or {} + await self.pool.execute( + """ + UPDATE ai_writer_batches + SET status=$2, + response_json=$3::jsonb, + error=$4, + ready_count=$5, + failed_count=$6, + prompt_tokens=$7, + completion_tokens=$8, + total_tokens=$9, + estimated_cost_usd=$10, + completed_at=NOW() + WHERE id=$1 + """, + batch_id, + status, + json.dumps(response, ensure_ascii=False) if response is not None else None, + error[:2000] if error else None, + ready_count if status == "done" else 0, + 0 if status == "done" else 1, + usage.get("prompt_tokens"), + usage.get("completion_tokens"), + usage.get("total_tokens"), + usage.get("estimated_cost_usd"), + ) + + async def mark_posts_failed(self, post_ids: list[int], error: str) -> None: + if not post_ids: + return + await self.pool.execute( + """ + UPDATE raw_posts + SET rewrite_status='failed', + rewrite_notes=$2, + updated_at=NOW() + WHERE id=ANY($1::bigint[]) + """, + post_ids, + error[:1000], + ) + + async def apply_rewrites(self, batch_id: int, rewrites: list[dict], model: str, prompt: str) -> None: + for item in rewrites: + await self.pool.execute( + """ + UPDATE raw_posts + SET rewrite_status='ready', + rewritten_text=$2, + rewrite_notes=$3, + rewrite_model=$4, + rewrite_prompt_hash=$5, + rewrite_batch_id=$6, + rewrite_category=$7, + rewrite_category_tag=$8, + rewrite_source_tag=$9, + editorial_status='review', + final_text=$2, + final_category=$7, + final_category_tag=$8, + final_source_tag=$9, + rewritten_at=NOW(), + updated_at=NOW() + WHERE id=$1 + """, + int(item["id"]), + item["text"], + item["notes"], + model, + prompt_hash(prompt), + batch_id, + item["category"], + item["category_tag"], + item["source_tag"], + ) + + async def call_ai( + self, + provider: str, + model: str, + api_key: str, + api_base: str, + prompt: str, + payload: dict, + temperature: float, + timeout: int, + ) -> tuple[dict, dict[str, Any]]: + messages = [ + {"role": "system", "content": prompt}, + {"role": "user", "content": json.dumps(payload, ensure_ascii=False)}, + ] + kwargs: dict[str, Any] = { + "model": normalize_model(provider, model), + "messages": messages, + "temperature": temperature, + "timeout": timeout, + "response_format": {"type": "json_object"}, + } + if api_key: + kwargs["api_key"] = api_key + if api_base: + kwargs["api_base"] = api_base + response = await asyncio.to_thread(litellm.completion, **kwargs) + content = response.choices[0].message.content + return parse_ai_json(content), response_usage(response) + + async def run_once(self) -> bool: + enabled = await is_worker_enabled(self.pool, WORKER_AI_WRITER) + setting_enabled = await fetch_bool_setting("ai_writer_enabled", False) + if not enabled or not setting_enabled: + await self.heartbeat.beat(self.pool, status="disabled", force=True) + return False + + provider = str(await fetch_setting("ai_writer_provider", "anthropic") or "anthropic") + model = str(await fetch_setting("ai_writer_model", "") or "").strip() + api_key = str(await fetch_setting("ai_writer_api_key", "") or "").strip() + api_base = str(await fetch_setting("ai_writer_api_base", "") or "").strip() + batch_size = max(1, await fetch_int_setting("ai_writer_batch_size", 1)) + max_text_chars = max(200, await fetch_int_setting("ai_writer_max_text_chars", 3500)) + temperature = max(0.0, await fetch_float_setting("ai_writer_temperature", 0.4)) + timeout = max(10, await fetch_int_setting("ai_writer_timeout_sec", 180)) + prompt = build_system_prompt( + str(await fetch_setting("ai_writer_prompt", "") or ""), + str(await fetch_setting("ai_writer_contract", DEFAULT_CONTRACT_PROMPT) or DEFAULT_CONTRACT_PROMPT), + ) + categories = await load_writer_categories(self.pool) + + if not model or not api_key or not prompt or not categories: + await self.heartbeat.beat(self.pool, status="not_configured", force=True) + return False + + posts = await self.claim_posts(batch_size) + await self.heartbeat.beat(self.pool, meta={"claimed": len(posts), "batch_size": batch_size}) + if not posts: + return False + + payload = { + "task": "rewrite_accepted_posts", + "categories": categories, + "posts": [ + { + "id": int(p["id"]), + "producer_name": p.get("source_name") or "", + "producer_tag": normalize_hash_tag(p.get("source_tag") or p.get("source_name") or "", "source"), + "original_url": p.get("original_url") or "", + "qualification_score": p.get("qualification_score"), + "qualification_reason": p.get("qualification_reason") or "", + "media_count": int(p.get("media_count") or 0), + "media_types": list(p.get("media_types") or []), + "text": str(p.get("raw_text") or "")[:max_text_chars], + } + for p in posts + ], + } + post_ids = [int(p["id"]) for p in posts] + source_tags_by_id = { + int(p["id"]): normalize_hash_tag(p.get("source_tag") or p.get("source_name") or "", "source") + for p in posts + } + batch_id = await self.create_batch(provider, model, prompt, payload) + try: + response, usage = await self.call_ai(provider, model, api_key, api_base, prompt, payload, temperature, timeout) + rewrites = validate_rewrites(response, set(post_ids), categories, source_tags_by_id) + await self.apply_rewrites(batch_id, rewrites, normalize_model(provider, model), prompt) + await self.finish_batch(batch_id, "done", {"rewrites": rewrites}, usage=usage) + logger.info("AI writer batch done: id={} posts={} usage={}", batch_id, len(rewrites), usage) + return True + except Exception as exc: + await self.mark_posts_failed(post_ids, str(exc)) + await self.finish_batch(batch_id, "failed", error=str(exc)) + logger.exception("AI writer batch failed: id={} error={}", batch_id, exc) + return True + + async def run_loop(self) -> None: + await self.init() + logger.info("{} started", WORKER_AI_WRITER) + while True: + try: + had_work = await self.run_once() + except Exception as exc: + logger.exception("AI writer loop error: {}", exc) + had_work = False + interval = max(10, await fetch_int_setting("ai_writer_interval_sec", 60)) + if not had_work: + await asyncio.sleep(interval) + + +async def main() -> None: + logger.remove() + logger.add(lambda msg: print(msg, end=""), level=settings.log_level) + worker = AIWriterWorker() + await worker.run_loop() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/vk_parser_app/workers/vk_storage_uploader.py b/src/vk_parser_app/workers/vk_storage_uploader.py new file mode 100644 index 0000000..c064e88 --- /dev/null +++ b/src/vk_parser_app/workers/vk_storage_uploader.py @@ -0,0 +1,567 @@ +from __future__ import annotations + +import asyncio +import html +import os +import re +import sys +import time +from pathlib import Path + +import aiohttp +from aiogram import Bot +from aiogram.client.session.aiohttp import AiohttpSession +from aiogram.client.telegram import TelegramAPIServer +from aiogram.exceptions import TelegramRetryAfter +from aiogram.types import BufferedInputFile, FSInputFile, InputMediaPhoto, InputMediaVideo +from loguru import logger + +from ..config import settings +from ..constants import ( + JOB_TYPE_VK_STORAGE_COPY, + MEDIA_STATUS_FAILED, + MEDIA_STATUS_LINK_ONLY, + MEDIA_STATUS_UPLOADED, + POST_STATUS_FAILED, + POST_STATUS_STORAGE_READY, + WORKER_STORAGE_UPLOADER, +) +from ..db import fetch_float_setting, fetch_int_setting, fetch_setting, get_pool +from ..heartbeat import HeartbeatReporter +from ..jobs import ack_done, ack_retry, claim_job, is_worker_enabled, recover_stale_jobs + +TMP_DIR = Path("/tmp") +TMP_PREFIX = "vkparser_tg_media_" +MAX_MEDIA_GROUP = 10 + + +def parse_topic(value: str) -> tuple[int, int | None]: + raw = str(value or "").strip() + if not raw: + raise RuntimeError("TG_MEDIA_CHANNEL_ID is empty") + if ":" in raw: + chat_id, thread_id = raw.split(":", 1) + return int(chat_id), int(thread_id) + return int(raw), None + + +def parse_vk_video_url(vk_url: str) -> tuple[int, int] | None: + match = re.search(r"video(-?\d+)_(\d+)", vk_url or "") + if not match: + return None + return int(match.group(1)), int(match.group(2)) + + +def split_message_chunks(text: str, limit: int) -> list[str]: + text = str(text or "").strip() + if not text: + return [] + chunks: list[str] = [] + while len(text) > limit: + split_at = text.rfind("\n", 0, limit) + if split_at < limit // 2: + split_at = text.rfind(" ", 0, limit) + if split_at < limit // 2: + split_at = limit + chunks.append(text[:split_at].strip()) + text = text[split_at:].strip() + if text: + chunks.append(text) + return chunks + + +def original_link(post: dict) -> str: + url = str(post.get("original_url") or "").strip() + return f'#{int(post["id"])}' if url else f'#{int(post["id"])}' + + +class TelegramStorageUploader: + def __init__(self) -> None: + self.pool = None + self.bot: Bot | None = None + self.storage_chat_id: int | None = None + self.storage_thread_id: int | None = None + self.heartbeat = HeartbeatReporter(WORKER_STORAGE_UPLOADER, 30) + self.media_group_max_items = MAX_MEDIA_GROUP + self.media_upload_delay_sec = 1.0 + self.tg_retry_max_attempts = 4 + self.tg_retry_backoff_max_sec = 15 + self.caption_limit = 1024 + self.message_limit = 4096 + self.text_overflow_marker = "Продолжение следующим сообщением." + self.download_timeout_sec = 45 + self.video_max_size_mb = 49 + self.video_max_duration_sec = 300 + self.yt_dlp_timeout_sec = 300 + self.max_media_attempts = 3 + self.post_job_pause_sec = 0.2 + + async def init(self) -> None: + self.pool = await get_pool() + recovered = await recover_stale_jobs(self.pool, JOB_TYPE_VK_STORAGE_COPY, stale_minutes=20) + if recovered: + logger.warning("Recovered stale storage jobs: {}", recovered) + + if not settings.tg_bot_token: + raise RuntimeError("TG_BOT_TOKEN is empty") + media_channel = str(await fetch_setting("tg_media_channel_id", settings.tg_media_channel_id) or "").strip() + self.storage_chat_id, self.storage_thread_id = parse_topic(media_channel) + + local_bot_api_url = str(await fetch_setting("local_bot_api_url", settings.local_bot_api_url) or "").strip() + if local_bot_api_url: + session = AiohttpSession( + api=TelegramAPIServer.from_base(local_bot_api_url.rstrip("/"), is_local=True) + ) + self.bot = Bot(token=settings.tg_bot_token, session=session) + logger.info("Using local Telegram Bot API: {}", local_bot_api_url) + else: + self.bot = Bot(token=settings.tg_bot_token) + + self.media_group_max_items = max(1, min(MAX_MEDIA_GROUP, await fetch_int_setting("telegram_media_group_max_items", 10))) + self.media_upload_delay_sec = max(0.0, await fetch_float_setting("telegram_media_upload_delay_sec", 1.0)) + self.tg_retry_max_attempts = max(1, await fetch_int_setting("telegram_retry_max_attempts", 4)) + self.tg_retry_backoff_max_sec = max(1, await fetch_int_setting("telegram_retry_backoff_max_sec", 15)) + self.caption_limit = max(128, await fetch_int_setting("telegram_caption_limit", 1024)) + self.message_limit = max(512, await fetch_int_setting("telegram_message_limit", 4096)) + self.text_overflow_marker = str(await fetch_setting("telegram_text_overflow_marker", self.text_overflow_marker)) + self.download_timeout_sec = max(5, await fetch_int_setting("uploader_download_timeout_sec", 45)) + self.video_max_size_mb = max(1, await fetch_int_setting("video_max_size_mb", 49)) + self.video_max_duration_sec = max(1, await fetch_int_setting("video_max_duration_sec", 300)) + self.yt_dlp_timeout_sec = max(30, await fetch_int_setting("uploader_yt_dlp_timeout_sec", 300)) + self.max_media_attempts = max(1, await fetch_int_setting("uploader_max_media_attempts", 3)) + self.post_job_pause_sec = max(0.0, await fetch_float_setting("media_post_job_pause_sec", 0.2)) + + logger.info( + "Telegram storage config: media_delay={}s, media_group_max_items={}, tg_retry={}, tg_backoff_max={}s, dl_timeout={}s, ytdlp_timeout={}s, post_pause={}s", + self.media_upload_delay_sec, + self.media_group_max_items, + self.tg_retry_max_attempts, + self.tg_retry_backoff_max_sec, + self.download_timeout_sec, + self.yt_dlp_timeout_sec, + self.post_job_pause_sec, + ) + + async def close(self) -> None: + if self.bot: + await self.bot.session.close() + + def chat_kwargs(self) -> dict: + kwargs = {"chat_id": self.storage_chat_id} + if self.storage_thread_id: + kwargs["message_thread_id"] = self.storage_thread_id + return kwargs + + async def tg_retry(self, fn): + for attempt in range(1, self.tg_retry_max_attempts + 1): + try: + return await fn() + except TelegramRetryAfter as exc: + delay = float(exc.retry_after) + 0.5 + logger.warning("Telegram flood control, sleep {}s", delay) + await asyncio.sleep(delay) + except Exception as exc: + if attempt >= self.tg_retry_max_attempts: + raise + delay = min(2**attempt, self.tg_retry_backoff_max_sec) + logger.warning("Telegram send error: {}. retry in {}s", exc, delay) + await asyncio.sleep(delay) + raise RuntimeError("telegram retry exhausted") + + async def load_raw_post(self, raw_post_id: int) -> dict | None: + row = await self.pool.fetchrow( + """ + SELECT rp.*, s.name AS source_name + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + WHERE rp.id=$1 + """, + raw_post_id, + ) + return dict(row) if row else None + + async def load_media(self, raw_post_id: int) -> list[dict]: + rows = await self.pool.fetch( + """ + SELECT * + FROM raw_post_media + WHERE raw_post_id=$1 + ORDER BY sort_order ASC, id ASC + """, + raw_post_id, + ) + return [dict(row) for row in rows] + + async def download_bytes(self, session: aiohttp.ClientSession, url: str) -> bytes | None: + try: + async with session.get(url, timeout=aiohttp.ClientTimeout(total=self.download_timeout_sec)) as response: + if response.status != 200: + return None + return await response.read() + except Exception: + return None + + async def mark_media_uploaded(self, media_id: int, file_id: str, unique_id: str | None) -> None: + await self.pool.execute( + """ + UPDATE raw_post_media + SET status=$2, + storage_attachment_id=$3, + storage_url=$3, + tg_file_id=$3, + tg_file_unique_id=$4, + error=NULL, + updated_at=NOW() + WHERE id=$1 + """, + media_id, + MEDIA_STATUS_UPLOADED, + file_id, + unique_id, + ) + + async def mark_media_link_only(self, media_id: int, error: str | None = None) -> None: + await self.pool.execute( + """ + UPDATE raw_post_media + SET status=$2, + error=COALESCE($3, error), + updated_at=NOW() + WHERE id=$1 + """, + media_id, + MEDIA_STATUS_LINK_ONLY, + error, + ) + + async def mark_media_failed_attempt(self, media_id: int, error: str) -> None: + await self.pool.execute( + """ + UPDATE raw_post_media + SET attempts=attempts+1, + status=CASE WHEN attempts + 1 >= $3 THEN $2 ELSE status END, + error=$4, + updated_at=NOW() + WHERE id=$1 + """, + media_id, + MEDIA_STATUS_FAILED, + self.max_media_attempts, + error[:1000], + ) + + async def download_video(self, vk_url: str, output_path: str) -> dict | None: + parsed = parse_vk_video_url(vk_url) + if not parsed: + return None + owner_id, video_id = parsed + max_size = self.video_max_size_mb * 1024 * 1024 + netrc_path = f"{output_path}.netrc" + fd = os.open(netrc_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as fh: + fh.write(f"machine vk.com login vk_token password {settings.vk_access_token}\n") + cmd = [ + sys.executable, + "-m", + "yt_dlp", + "--netrc-location", + netrc_path, + f"https://vk.com/video{owner_id}_{video_id}", + "-o", + output_path, + "--no-playlist", + "-f", + f"best[filesize<{max_size}]/bestvideo[filesize<{max_size}]+bestaudio/best", + "--quiet", + "--no-warnings", + ] + try: + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + _, stderr = await asyncio.wait_for(proc.communicate(), timeout=self.yt_dlp_timeout_sec) + except asyncio.TimeoutError: + proc.kill() + await proc.communicate() + return {"error": "video download timeout", "permanent": False} + finally: + try: + os.unlink(netrc_path) + except FileNotFoundError: + pass + if proc.returncode != 0 or not os.path.exists(output_path): + err = (stderr or b"").decode("utf-8", errors="ignore").lower() + permanent = any(marker in err for marker in ("removed", "unavailable", "private", "access denied")) + return {"error": "video unavailable or download failed", "permanent": permanent} + size = os.path.getsize(output_path) + if size > max_size: + return {"error": "video too large", "permanent": True} + return {"path": output_path, "size_bytes": size} + + async def prepare_media(self, raw_post_id: int, media: list[dict]) -> list[dict]: + prepared: list[dict] = [] + async with aiohttp.ClientSession() as session: + for item in media: + media_id = int(item["id"]) + media_type = str(item["media_type"]) + url = str(item.get("original_url") or "") + + if item.get("tg_file_id"): + prepared.append({"media_id": media_id, "media_type": media_type, "media": item["tg_file_id"]}) + continue + if int(item.get("attempts") or 0) >= self.max_media_attempts: + continue + + if media_type == "photo": + data = await self.download_bytes(session, url) + if not data: + await self.mark_media_failed_attempt(media_id, "photo download failed") + continue + prepared.append( + { + "media_id": media_id, + "media_type": "photo", + "media": BufferedInputFile(data, filename=f"photo_{media_id}.jpg"), + } + ) + continue + + if media_type == "video": + duration = item.get("duration_sec") + if duration and int(duration) > self.video_max_duration_sec: + await self.mark_media_link_only(media_id, "video too long") + continue + temp_path = str(TMP_DIR / f"{TMP_PREFIX}{raw_post_id}_{media_id}.mp4") + info = await self.download_video(url, temp_path) + if not info: + await self.mark_media_failed_attempt(media_id, "video download failed") + continue + if info.get("error"): + if info.get("permanent"): + await self.mark_media_link_only(media_id, str(info["error"])) + else: + await self.mark_media_failed_attempt(media_id, str(info["error"])) + continue + prepared.append( + { + "media_id": media_id, + "media_type": "video", + "media": FSInputFile(temp_path, filename="video.mp4"), + "tmp_path": temp_path, + } + ) + continue + + await self.mark_media_link_only(media_id, "unsupported media type") + return prepared + + async def send_text_chunk(self, text: str) -> int: + message = await self.tg_retry( + lambda: self.bot.send_message( + text=text, + parse_mode="HTML", + disable_web_page_preview=True, + **self.chat_kwargs(), + ) + ) + return int(message.message_id) + + def build_text_parts(self, post: dict, has_media: bool, link_only_media: list[dict] | None = None) -> tuple[str | None, list[str]]: + link = original_link(post) + raw_text = html.escape(str(post.get("raw_text") or "").strip()) + link_only_media = link_only_media or [] + media_lines = [] + for item in link_only_media: + url = str(item.get("original_url") or "").strip() + if not url: + continue + media_type = html.escape(str(item.get("media_type") or "media")) + reason = html.escape(str(item.get("error") or "link only")) + media_lines.append(f"- {media_type}: ссылка ({reason})") + media_note = "\n\nМедиа по ссылке:\n" + "\n".join(media_lines) if media_lines else "" + suffix = f"\n\n{link}" + body = raw_text + media_note + if has_media: + if len(body + suffix) <= self.caption_limit: + return body + suffix if body else link, [] + chunks = split_message_chunks(body + suffix, self.message_limit) + return self.text_overflow_marker[: self.caption_limit], chunks + chunks = split_message_chunks(body + suffix if body else link, self.message_limit) + return None, chunks + + async def save_sent_media_ids(self, prepared: list[dict], messages: list) -> None: + for item, message in zip(prepared, messages): + if item["media_type"] == "photo" and message.photo: + photo = message.photo[-1] + await self.mark_media_uploaded(int(item["media_id"]), photo.file_id, photo.file_unique_id) + elif item["media_type"] == "video" and message.video: + video = message.video + await self.mark_media_uploaded(int(item["media_id"]), video.file_id, video.file_unique_id) + + async def send_storage_post(self, post: dict, prepared: list[dict], media: list[dict]) -> tuple[list[int], int | None]: + uploaded_media_ids = {int(item["media_id"]) for item in prepared} + link_only_media = [ + item + for item in media + if int(item["id"]) not in uploaded_media_ids and str(item.get("status") or "") == MEDIA_STATUS_LINK_ONLY + ] + caption, text_chunks = self.build_text_parts(post, bool(prepared), link_only_media) + message_ids: list[int] = [] + + if prepared: + first = prepared[: self.media_group_max_items] + group = [] + for idx, item in enumerate(first): + cap = caption if idx == 0 else None + if item["media_type"] == "photo": + group.append(InputMediaPhoto(media=item["media"], caption=cap, parse_mode="HTML")) + elif item["media_type"] == "video": + group.append(InputMediaVideo(media=item["media"], caption=cap, parse_mode="HTML")) + if group: + messages = await self.tg_retry(lambda: self.bot.send_media_group(media=group, **self.chat_kwargs())) + await self.save_sent_media_ids(first, messages) + message_ids.extend(int(msg.message_id) for msg in messages) + await asyncio.sleep(self.media_upload_delay_sec * len(first)) + + rest = prepared[self.media_group_max_items :] + for start in range(0, len(rest), self.media_group_max_items): + chunk = rest[start : start + self.media_group_max_items] + group = [ + InputMediaPhoto(media=item["media"]) + if item["media_type"] == "photo" + else InputMediaVideo(media=item["media"]) + for item in chunk + if item["media_type"] in ("photo", "video") + ] + if group: + messages = await self.tg_retry(lambda g=group: self.bot.send_media_group(media=g, **self.chat_kwargs())) + await self.save_sent_media_ids(chunk, messages) + message_ids.extend(int(msg.message_id) for msg in messages) + await asyncio.sleep(self.media_upload_delay_sec * len(chunk)) + for chunk in text_chunks: + message_ids.append(await self.send_text_chunk(chunk)) + return message_ids, None + + async def mark_post_ready(self, raw_post_id: int, message_ids: list[int], meta_message_id: int | None) -> None: + await self.pool.execute( + """ + UPDATE raw_posts + SET status=$2, + tg_storage_chat_id=$3, + tg_storage_thread_id=$4, + tg_storage_message_ids=$5, + tg_storage_meta_message_id=$6, + storage_post_url=$7, + copied_at=NOW(), + error_reason=NULL, + updated_at=NOW() + WHERE id=$1 + """, + raw_post_id, + POST_STATUS_STORAGE_READY, + self.storage_chat_id, + self.storage_thread_id, + message_ids, + meta_message_id, + f"tg://resolve?domain=c/{str(abs(int(self.storage_chat_id or 0))).removeprefix('100')}/{message_ids[0]}" if message_ids else None, + ) + + async def mark_post_failed(self, raw_post_id: int, error: str) -> None: + await self.pool.execute( + """ + UPDATE raw_posts + SET status=$2, + error_reason=$3, + updated_at=NOW() + WHERE id=$1 + """, + raw_post_id, + POST_STATUS_FAILED, + error[:1000], + ) + + async def process_job(self, job: dict, worker_id: str) -> None: + job_id = int(job["id"]) + raw_post_id = int(job["entity_id"]) + await self.heartbeat.beat(self.pool, current_job_id=job_id, force=True) + + post = await self.load_raw_post(raw_post_id) + if not post: + await ack_retry(self.pool, job, f"raw_post {raw_post_id} not found", delay_sec=60) + return + if post.get("tg_storage_message_ids"): + await ack_done(self.pool, job_id) + return + + media = await self.load_media(raw_post_id) + prepared = await self.prepare_media(raw_post_id, media) + media = await self.load_media(raw_post_id) + try: + message_ids, meta_message_id = await self.send_storage_post(post, prepared, media) + finally: + for item in prepared: + tmp_path = item.get("tmp_path") + if tmp_path: + try: + if os.path.exists(tmp_path): + os.remove(tmp_path) + except Exception: + pass + await self.mark_post_ready(raw_post_id, message_ids, meta_message_id) + await ack_done(self.pool, job_id) + logger.info("Telegram storage done: raw_post={} messages={}", raw_post_id, message_ids) + + async def run_once(self, worker_id: str) -> bool: + enabled = await is_worker_enabled(self.pool, WORKER_STORAGE_UPLOADER) + if not enabled: + await self.heartbeat.beat(self.pool, status="disabled", force=True) + return False + + job = await claim_job(self.pool, JOB_TYPE_VK_STORAGE_COPY, worker_id) + if not job: + await self.heartbeat.beat(self.pool, status="idle") + return False + try: + await self.process_job(job, worker_id) + except Exception as exc: + await self.mark_post_failed(int(job["entity_id"]), str(exc)) + await ack_retry(self.pool, job, str(exc), delay_sec=120) + logger.exception("Telegram storage job {} failed: {}", job["id"], exc) + return True + + async def cleanup_tmp_files(self) -> None: + now = time.time() + for path in TMP_DIR.glob(f"{TMP_PREFIX}*"): + try: + if path.is_file() and path.stat().st_mtime < now - 3600: + path.unlink(missing_ok=True) + except Exception: + pass + + async def run_loop(self) -> None: + await self.init() + worker_id = f"{WORKER_STORAGE_UPLOADER}:{os.getpid()}" + logger.info("{} started", worker_id) + try: + while True: + await self.cleanup_tmp_files() + had_job = await self.run_once(worker_id) + if not had_job: + await asyncio.sleep(max(1, await fetch_int_setting("uploader_interval_sec", 5))) + else: + await asyncio.sleep(self.post_job_pause_sec) + finally: + await self.close() + + +async def main() -> None: + logger.remove() + logger.add(sys.stdout, level=settings.log_level) + worker = TelegramStorageUploader() + await worker.run_loop() + + +if __name__ == "__main__": + asyncio.run(main())