commit d01890ff2cdca6d70769bdb226f2b3730f7624be Author: Your Name Date: Mon Aug 3 16:18:56 2026 +0500 Initial RAA parser poster copy diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..ceb1d13 --- /dev/null +++ b/.env.example @@ -0,0 +1,27 @@ +APP_ENV=production +APP_SECRET_KEY=change-me +ADMIN_SITE_TITLE=RAA Редакторская +ADMIN_APP_TITLE=RAA Редакторская +ADMIN_BOOTSTRAP_LOGIN=admin +ADMIN_BOOTSTRAP_PASSWORD=change-me-long-password + +DB_HOST=localhost +DB_PORT=5432 +DB_NAME=raa_parser +DB_USER=raa_parser_user +DB_PASSWORD=change-me + +VK_ACCESS_TOKEN= +VK_GROUP_ACCESS_TOKEN= +VK_API_VERSION=5.199 +VK_STORAGE_GROUP_ID=0 + +TG_BOT_TOKEN= +TG_MEDIA_CHANNEL_ID= +LOCAL_BOT_API_URL= +TELEGRAM_API_ID= +TELEGRAM_API_HASH= + +ADMIN_HOST=0.0.0.0 +ADMIN_PORT=8080 +LOG_LEVEL=INFO diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..fea410f --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +* text=auto eol=lf + +*.bat text eol=crlf +*.cmd text eol=crlf +*.ps1 text eol=crlf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5d9f36e --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +.env +.env.* +!.env.example +__pycache__/ +*.py[cod] +.venv/ +logs/ +uploads/ +original_project/ +/ai_worker.py +/media_worker.py +/parser.py +/raw_feed_worker.py +vkdev_main.js +sample_photo.jpg diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..03dcd8d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,59 @@ +FROM aiogram/telegram-bot-api:latest AS telegram-bot-api + +FROM python:3.12-slim + +WORKDIR /app + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + ca-certificates \ + postgresql-client \ + curl \ + ffmpeg \ + git \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /usr/local/share/ca-certificates/russian-trusted \ + && curl -fsSLk https://gu-st.ru/content/lending/russian_trusted_root_ca_pem.crt \ + -o /usr/local/share/ca-certificates/russian-trusted/russian_trusted_root_ca_pem.crt \ + && curl -fsSLk https://gu-st.ru/content/lending/russian_trusted_sub_ca_pem.crt \ + -o /usr/local/share/ca-certificates/russian-trusted/russian_trusted_sub_ca_pem.crt \ + && update-ca-certificates + +# Copy the local Telegram Bot API binary and its musl runtime from the official aiogram image. +COPY --from=telegram-bot-api /usr/local/bin/telegram-bot-api /usr/local/bin/telegram-bot-api +COPY --from=telegram-bot-api /lib/ld-musl-x86_64.so.1 /lib/ld-musl-x86_64.so.1 +COPY --from=telegram-bot-api /usr/lib/libssl.so.3 /usr/lib/libssl.so.3 +COPY --from=telegram-bot-api /usr/lib/libcrypto.so.3 /usr/lib/libcrypto.so.3 +COPY --from=telegram-bot-api /usr/lib/libz.so.1 /usr/lib/libz.so.1 +COPY --from=telegram-bot-api /usr/lib/libstdc++.so.6 /usr/lib/libstdc++.so.6 +COPY --from=telegram-bot-api /usr/lib/libgcc_s.so.1 /usr/lib/libgcc_s.so.1 + +# Install python dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy source code +COPY src/ src/ +COPY scripts/ scripts/ +COPY db/ db/ +COPY docker-entrypoint.sh /usr/local/bin/vk-parser-entrypoint +RUN chmod +x /usr/local/bin/vk-parser-entrypoint \ + && mkdir -p /var/lib/telegram-bot-api /tmp/telegram-bot-api + +COPY favi.png . +COPY .env.example .env + +ENV PYTHONPATH=/app/src +ENV PYTHONUNBUFFERED=1 +ENV LOCAL_BOT_API_URL=http://127.0.0.1:8081 +ENV TELEGRAM_WORK_DIR=/var/lib/telegram-bot-api +ENV TELEGRAM_TEMP_DIR=/tmp/telegram-bot-api +ENV TELEGRAM_HTTP_PORT=8081 +ENV TELEGRAM_LOCAL=1 + +EXPOSE 8000 + +# Start uvicorn without any workers +ENTRYPOINT ["vk-parser-entrypoint"] +CMD ["uvicorn", "vk_parser_app.admin:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/PROJECT.md b/PROJECT.md new file mode 100644 index 0000000..66f59cd --- /dev/null +++ b/PROJECT.md @@ -0,0 +1,1728 @@ +# N8 Parser: проектная документация + +Этот документ описывает текущую архитектуру проекта `n8_parser`: зачем он нужен, как устроены БД, воркеры, админка, AI-контур, медиа-хранилище, деплой и типовые операции. Документ написан как рабочая карта проекта, чтобы к нему можно было вернуться через месяц и быстро понять, где что лежит и почему сделано именно так. + +## 1. Цель проекта + +Проект автоматизирует сбор и редакционную обработку постов из VK-сообществ производителей, магазинов и площадок по тактическому, милитари и airsoft-снаряжению. + +Основная цепочка: + +```text +Источники VK + -> VK-парсер + -> raw_posts / raw_post_media в PostgreSQL + -> Telegram media storage + -> AI-квалификатор + -> AI-райтер + -> редакторская + -> принятие / отклонение / ручное редактирование +``` + +Проект не должен превращаться в набор разрозненных скриптов. Главная идея текущей реализации: все долгие операции живут в отдельных воркерах, все настройки управляются из админки, а секреты остаются в `.env`. + +## 2. Текущий стек + +- Backend: Python 3.12+, FastAPI. +- Шаблоны: Jinja2. +- БД: PostgreSQL. +- Асинхронный доступ к БД: `asyncpg`. +- VK API: собственный клиент в `src/vk_parser_app/vk_api.py`. +- Telegram: `aiogram`. +- AI: `litellm`, сейчас используется Anthropic через LiteLLM. +- Логи: `loguru`. +- Runtime/deploy: локальная инфраструктура Proxmox + Coolify + Docker. +- Reverse proxy / TLS: edge-proxy Caddy из локальной инфраструктуры. +- Публичная админка: `https://panel.f-n8.ru` (`https://admin.f-n8.ru` после обновления DNS). +- Deploy-репозиторий: Gitea `http://192.168.1.135:3000/exostring/new_vk_parser.git`, remote `gitea`. +- GitHub `origin` используется как upstream/backup, не как основной deploy remote. + +## 3. Структура проекта + +```text +. +├── db/ +│ └── migrations/ # SQL-миграции +├── scripts/ +│ ├── apply_migrations.py # применение миграций +│ └── vk_storage_spike.py # старый тест VK storage +├── src/ +│ └── vk_parser_app/ +│ ├── admin.py # FastAPI админка и маршруты +│ ├── config.py # env-настройки +│ ├── constants.py # статусы, имена воркеров, типы jobs +│ ├── db.py # pool, settings, миграции +│ ├── heartbeat.py # heartbeat воркеров +│ ├── jobs.py # очередь jobs +│ ├── main.py # ASGI entrypoint +│ ├── security.py # пароли, сессии, CSRF +│ ├── text_utils.py # тэги, категории +│ ├── vk_api.py # VK API client и извлечение media +│ ├── templates/ # HTML-шаблоны +│ └── workers/ +│ ├── parser.py # VK-парсер +│ ├── vk_storage_uploader.py# Telegram media storage uploader +│ ├── ai_qualifier.py # AI-квалификатор +│ └── ai_writer.py # AI-райтер +├── original_project/ # старый проект как справочник +├── requirements.txt +├── README.md # короткий early quick start, частично устарел +└── PROJECT.md # этот документ +``` + +## 4. Важное архитектурное решение + +### 4.0. Важные изменения фиксируются в документации + +Если меняется поведение системы, эксплуатационная схема, публикационный формат, БД, деплой или админка, это нужно отражать в `PROJECT.md` или другом релевантном `.md` рядом с кодом. + +Инфраструктурная карта вне репозитория: `D:\DEVELOPMENT\.infra.md`. Перед любыми deploy/server действиями сначала читать её. Для этого проекта актуален контур Gitea -> Coolify -> LXC 105; старые инструкции про `sw.exostring.xyz`, `109.120.156.205`, `/opt/vk-parser` и ручной `systemctl` не использовать без явной команды пользователя. + +Текущие зафиксированные изменения: + +- Публичный сайт вынесен в самостоятельное приложение `forma_n8_site/` и разворачивается на отдельном VPS. Сайт хранит собственные статьи и медиа, отдаёт серверный HTML, RSS, sitemap, robots.txt, canonical и JSON-LD `NewsArticle`. +- Интеграция с сайтом строится через idempotent `POST /api/v1/articles`: `external_id` уникален, а slug и дата первой публикации сохраняются при обновлениях. Дизайн можно менять без изменения URL и повторной загрузки постов. Медиа передаются через защищённый `POST /api/v1/media` и хранятся на стороне сайта. +- Публичные названия категорий сайта отделены от описательных AI-категорий парсера: таблица `categories` связывает стабильный латинский tag с коротким SEO-названием. API нормализует входной tag и использует справочник для H1, breadcrumbs и category URL. +- Пока у публичного сайта нет домена, `SITE_INDEXING_ENABLED=false`: HTML получает `noindex`, а robots.txt закрывает IP от обхода. Индексация включается только после настройки домена, HTTPS, корректного `SITE_BASE_URL` и редиректа с IP. + +- `favi.png` лежит в корне проекта и отдаётся FastAPI по `/favi.png` и `/favicon.ico`. +- Финальный текст публикации собирается через общий helper: к тексту добавляется строка `#category #source`. +- Категорийный хэштег берётся из `content_categories.tag`, а не из человекочитаемого названия категории. +- В админке добавлена вкладка `/logs` для просмотра `audit_log`; записи старше 30 дней удаляются автоматически. +- Добавлен TG-постер: расписание по Екб, отдельный токен/чат, ранжирование, Telegram-лимиты и статусы `published` / `publish_failed`. +- Добавлен TG-реактор: отдельный воркер ставит реакции на опубликованные Telegram-посты из `post_publications`; один bot token ставит одну реакцию, несколько bot tokens могут поставить несколько реакций. +- Добавлен ежедневный Telegram-отчет `daily-report`: в `00:04` по Екатеринбургу отправляет получателям из `daily_report_recipient_ids` операционный отчет за прошедший день: raw-сбор, AI-принято/отклонено, редакторская очередь, TG/VK публикации, ошибки и состояние воркеров. По умолчанию получатель `442509142`, bot token берется из `daily_report_bot_token`, затем `tg_poster_bot_token`, затем `TG_BOT_TOKEN`. +- Текущая схема TG-реактора: основной бот публикации + два отдельных reaction-бота; реакции распределены по порядку токенов как `👍`, `🔥`, `❤`. +- Для догонки реакций TG-реактор использует паузу `tg_reactor_reaction_pause_sec` между успешными реакциями, чтобы не отправлять пачку запросов в Telegram одним всплеском. +- Разделительная полоса в AI-рерайтах должна быть строго `━━━━━━━━━━━━━━━━━` (17 символов). +- TG-постер перечитывает настройки перед циклом и не догоняет прошедшие часы; слот срабатывает только в 10-минутном окне после своего времени. +- TG-постер публикует медиа только через Telegram `file_id`/storage attachment или локальный upload; сырые `http(s)` media URL не отправляются как Telegram media, чтобы не ловить `WEBPAGE_MEDIA_EMPTY` / `wrong type of the web page content`. +- TG-постер повторяет отправку только при `TelegramRetryAfter` flood-control. Обычные timeout/network ошибки после `send_message`/`send_media` не ретраятся, потому что Telegram мог уже принять сообщение, а повтор создаёт дубли. +- На сервере собран и запущен локальный Telegram Bot API (`telegram-bot-api.service`) на `127.0.0.1:8081`. Uploader использует его через `local_bot_api_url`, чтобы загружать видео больше облачного лимита Bot API. +- Telegram storage uploader скачивает VK-видео через `yt-dlp` и только после успешной загрузки всех pending фото/видео переводит raw-пост в `storage_ready`. Если видео временно не скачалось, job ретраится; если media окончательно failed, raw-пост получает `failed`, чтобы публикация не ушла с одной обложкой вместо ролика. Текущие лимиты после включения local Bot API: `video_max_size_mb=512`, `video_max_height=1080`, `video_max_duration_sec=1200`, `uploader_yt_dlp_timeout_sec=1800`. +- Вся цепочка после storage теперь требует минимум одно видимое загруженное `photo`/`video`: uploader не переводит пост без media в `storage_ready`, AI-квалификатор и AI-райтер не забирают посты без загруженного media, TG-постер не публикует старые готовые записи без загруженного media. Если видео ушло в `failed` или `link_only`, пост считается `failed` и не попадает даже на квалификацию. +- При финальной проверке media uploader не считает `pending` те файлы, которые уже успешно скачаны и подготовлены в текущем проходе: их Telegram `file_id` появляется только после отправки. Иначе uploader зацикливается между подготовкой файла и повтором job, не отправляя медиа в storage. +- TG-постер дополнительно не берёт кандидата в публикацию, если у видимого `photo`/`video` нет `tg_file_id`/`storage_attachment_id`; это защищает от старых `storage_ready` записей, которые успели стать готовыми до ужесточения uploader. +- Для разовой дозагрузки старых timeout-видео добавлен `scripts/backfill_pending_videos.py`: он последовательно скачивает pending VK-видео, отправляет в Telegram storage через local Bot API и проставляет `tg_file_id`. +- На странице Raw можно выделить несколько постов и вручную отправить их в AI-рерайт; обработчик переводит только `storage_ready` посты в `qualification_status='accepted'` и `rewrite_status='pending'`. +- Категория AI-райтера `18` / `Не целевой контент` / `НЦК` считается автоматическим AI-отклонением: такие посты получают `qualification_status='rejected'`, `editorial_status='rejected'` и не попадают в редакторскую `На проверке`. +- Добавлена общая таблица `post_publications`: в редакторской у каждой новости видны платформенные статусы TG/VK/MAX/Сайт. TG-постер зеркалит результат в эту таблицу, VK-постер пишет туда напрямую. +- Запрос редакторской агрегирует строки `post_publications` для каждой новости; статусы TG/VK берутся из фактических платформенных публикаций, поэтому опубликованный пост не отображается как `ожидает`. +- Добавлен VK-постер `src/vk_parser_app/workers/vk_poster.py`: публикует принятые/уже опубликованные в TG посты в целевое VK-сообщество по отдельному расписанию Екб. На текущем этапе VK-постер не загружает медиа через `photos.*`, а берёт только исходные VK attachments (`photo...` / `video...`) из `raw_post_media.original_attachment_id`; посты без такого VK attachment не попадают в кандидаты VK-публикации. +- Добавлен MAX-постер `src/vk_parser_app/workers/max_poster.py`: публикует принятые/уже опубликованные материалы в MAX-канал через Bot API от имени бота-администратора. Расписание по умолчанию повторяет TG hourly schedule. Текст режется по лимиту MAX `4000` символов, медиа загружается в MAX отдельно через `/uploads`, а результат пишется в `post_publications(platform='max')`. +- MAX-постер для видео использует не `original_url` VK-страницы, а готовый файл из Telegram storage по `tg_file_id` / `storage_attachment_id`. Если локальный Telegram Bot API возвращает абсолютный `file_path` в `/var/lib/telegram-bot-api/...`, воркер читает файл напрямую с диска. После upload видео воркер poll-ит `GET /videos/{videoToken}` и отправляет пост только когда MAX вернул `urls`. +- В MAX Bot API на момент внедрения нет публичного метода для реакции на пост: попытка `/messages/{message_id}/reactions` возвращает `404 method.not.found`. Auto-reaction оставлен как non-fatal hook и не должен помечать успешную публикацию ошибкой. +- В исходных VK-постах могут встречаться отдельные пустые/белые фото рядом с видео. Пример: `raw_post_id=398`, `photo-190867868_457284718` полностью белая (`1433x2000`, RGB `255/255/255`). MAX-постер пока прикрепляет такие фото как обычные исходные медиа; следующий безопасный шаг - добавить фильтр почти полностью белых изображений перед upload. +- Ранжирование TG/VK-постеров не использует `qualification_score`: квалификатор только допускает пост в дальнейшую цепочку. Постер сначала выбирает вариант без совпадения категории/источника с предыдущей публикацией, затем минимизирует повторы категории и источника в окне последних 20 публикаций, а при равенстве берёт самый старый ожидающий пост. +- Для VK-постера user token нужно получать через VK ID Authorization Code Flow с PKCE: `/vk-oauth/start` -> `id.vk.ru/authorize` -> `/vk-oauth/callback`. Callback обменивает `code` через `id.vk.ru/oauth2/auth` на `access_token` + `refresh_token`, сохраняет `vk_poster_access_token`, `vk_poster_refresh_token`, `vk_poster_token_device_id`, `vk_poster_token_expires_at` и проверяет `wall.get` / `photos.getWallUploadServer`. Воркер обновляет access token через refresh token до истечения. Refresh token живёт 180 дней по документации VK ID; Redirect URI в VK ID-приложении: `https://sw.exostring.xyz/vk-oauth/callback`. +- По документации VK API метод `photos.getWallUploadServer` требует пользовательское право `photos`, которое VK выдаёт в исключительных случаях через запрос в поддержку `devsupport@corp.vk.com`. Если token без `photos`, VK-постер пропускает upload редакторских/URL-фото, но всё равно публикует текст и существующие VK `photo...`/`video...` attachments. +- VK-постер исключает из очереди video-only записи с `raw_post_media.status='link_only'`: VK API не разрешает прикреплять часть чужих видео без предварительной загрузки в целевое сообщество. Медиа `link_only` также не добавляются к смешанным постам, а публикации со статусом `publish_failed` не возвращаются автоматически в каждый следующий часовой слот и требуют явного ручного повтора. +- Админка на 2026-07-29: заголовок берётся из `ADMIN_APP_TITLE`, fallback `Редакторская`; старая молния рядом с названием удалена. Боковая навигация сворачиваемая и по умолчанию свёрнута до иконок. +- В редакторской фильтры на ПК сворачиваемые и по умолчанию свёрнуты; лента занимает освободившуюся ширину. На мобильных статусные вкладки скроллятся горизонтально, а `/sources` и `/raw` не должны давать page-level horizontal scroll. +- В карточке редакторской meta-блок поста находится над постом. `Оригинальный текст` расположен спойлером сразу над `Финальный текст`; `AI заметки / Резолюция` отдельным спойлером показывает заметки/резолюцию и ниже preview того, как AI написал пост после рерайта. +- `Финальный текст` должен полностью вмещать текст без внутреннего скролла. Текущая защита от регресса: autosize через Alpine `ResizeObserver`, trimming лишних хвостовых whitespace в браузере/template и `min-h-[150px]`; не заменять на фиксированную большую высоту. +- В редакторской восстановлено скрытие медиа: крестик на фото/видео добавляет `delete_media_ids`, backend помечает media `editor_hidden=TRUE`, и такие media не публикуются. +- Действие `Отклонить` выполняется без confirm/alert. Возврат поста на проверку идёт через `/editor/{post_id}/return`; кнопка должна быть доступна во вкладках `accepted` и `rejected`. +- Редакторские action-кнопки должны обновлять только нужные HTMX partials статуса/действий, чтобы открытые спойлеры и тело поста не схлопывались после `Сохранить`, `Сохранить и Опубликовать` или `Вернуть на проверку`. +- Светлая/тёмная тема реализована CSS-переменными. Tailwind `app.*` цвета заданы как RGB channels и используются через `rgb(var(--app-...)/)`; не возвращать hex-переменные, иначе `bg-app-bg/50` и похожие opacity utilities ломаются. В светлой теме hover не должен превращать текст в белый на светлом фоне. +- AI Writer на 2026-07-29 проверен с `ai_writer_model = anthropic/claude-sonnet-4-6`. Sonnet 4.6 иногда возвращает поле `rewrites` JSON-строкой, поэтому `validate_rewrites()` парсит stringified `rewrites` перед проверкой. + +### 4.1. Секреты в `.env`, операционные настройки в БД + +В `.env` должны лежать только секреты и базовые параметры окружения: + +- `APP_SECRET_KEY` +- `ADMIN_BOOTSTRAP_LOGIN` +- `ADMIN_BOOTSTRAP_PASSWORD` +- параметры подключения к PostgreSQL +- `VK_ACCESS_TOKEN` +- `TG_BOT_TOKEN` +- и другие токены/пароли + +В БД, в таблице `app_settings`, лежат операционные настройки: + +- интервалы воркеров; +- batch size; +- лимиты текста; +- включение/выключение AI; +- выбор provider/model; +- prompt/contract; +- Telegram storage channel id; +- лимиты Telegram uploader; +- параметры парсинга. + +Причина: секреты опасно показывать в админке и хранить в обычных настройках, а операционные параметры удобно менять без деплоя. + +### 4.2. Worker control состоит из двух флагов + +Для AI-воркеров используется двойная защита: + +1. `worker_controls.enabled` +2. `app_settings.ai_*_enabled` + +Воркер начнёт работу только если включены оба. Это сделано специально, чтобы случайное включение одного тумблера не запускало расход денег на LLM. + +## 5. БД: ключевые таблицы + +### 5.1. `admin_users` + +Пользователи админки. + +Основные поля: + +- `id` +- `login` +- `password_hash` +- `is_active` +- `created_at` + +Первый пользователь создаётся из `ADMIN_BOOTSTRAP_LOGIN` и `ADMIN_BOOTSTRAP_PASSWORD`. + +### 5.2. `admin_sessions` + +Сессии админки. + +Хранит: + +- `token_hash` +- `user_id` +- `csrf_token` +- `expires_at` +- `created_at` + +Сырой session token в БД не хранится. + +### 5.3. `sources` + +Источники для парсинга. + +Сейчас реально используется только `platform='vk'`, но схема заложена под другие площадки. + +Важные поля: + +- `id` +- `platform` +- `name` +- `tag` +- `url` +- `external_id` +- `external_owner_id` +- `active` +- `status` +- `status_msg` +- `last_checked_at` +- `last_parsed_at` +- `parse_from` +- `priority` +- `archived_at` + +`tag` нужен для будущих хэштегов и финального оформления. + +### 5.4. `raw_posts` + +Главная таблица жизненного цикла поста. + +Основные поля raw: + +- `id` +- `source_id` +- `platform` +- `external_post_id` +- `external_owner_id` +- `original_url` +- `raw_text` +- `raw_json` +- `text_hash` +- `content_hash` +- `posted_at` +- `created_at` +- `updated_at` +- `status` +- `skip_reason` + +Storage-поля: + +- `storage_post_url` +- `tg_storage_chat_id` +- `tg_storage_thread_id` +- `tg_storage_message_ids` + +AI-квалификация: + +- `qualification_status` +- `qualification_score` +- `qualification_decision` +- `qualification_model_decision` +- `qualification_reason` +- `qualification_reject_tag` +- `qualification_model` +- `qualification_prompt_hash` +- `qualification_batch_id` +- `qualified_at` + +AI-райтер: + +- `rewrite_status` +- `rewritten_text` +- `rewrite_notes` +- `rewrite_model` +- `rewrite_prompt_hash` +- `rewrite_batch_id` +- `rewrite_category_id` +- `rewrite_category` +- `rewrite_category_tag` +- `rewrite_source_tag` +- `rewritten_at` + +Редакторская: + +- `editorial_status` +- `final_text` +- `final_category_id` +- `final_category` +- `final_category_tag` +- `final_source_tag` +- `editor_notes` +- `reviewed_by` +- `reviewed_at` +- `edited_at` + +### 5.5. `raw_post_media` + +Медиа исходного поста. + +Основные поля: + +- `id` +- `raw_post_id` +- `platform` +- `media_type` +- `original_url` +- `original_attachment_id` +- `preview_url` +- `storage_url` +- `storage_attachment_id` +- `tg_file_id` +- `tg_file_unique_id` +- `width` +- `height` +- `duration_sec` +- `sort_order` +- `status` +- `attempts` +- `error` +- `editor_hidden` +- `editor_added` +- `local_path` + +Удаление медиа в редакторской не обязано физически удалять старое медиа из Telegram. В редакторском контуре оно помечается как скрытое через `editor_hidden`. + +### 5.6. `jobs` + +Очередь фоновых задач. + +Основные поля: + +- `id` +- `type` +- `entity_type` +- `entity_id` +- `payload_json` +- `status` +- `attempts` +- `max_attempts` +- `next_run_at` +- `locked_by` +- `locked_at` +- `last_error` + +Сейчас основной тип: + +- `vk.storage.copy` + +Название историческое: сначала планировался VK storage, потом медиа-сторедж вернулся в Telegram. Тип задачи пока не переименован. + +### 5.7. `worker_controls` + +Включение/выключение воркеров. + +Поля: + +- `name` +- `enabled` +- `settings_json` +- `updated_by` +- `updated_at` + +Имена воркеров: + +- `vk-parser` +- `vk-storage-uploader` +- `ai-qualifier` +- `ai-writer` +- `tg-poster` +- `max-poster` + +### 5.8. `worker_heartbeats` + +Состояние воркеров для админки. + +Поля: + +- `name` +- `heartbeat_at` +- `status` +- `current_job_id` +- `meta_json` + +### 5.9. `app_settings` + +Настройки приложения. + +Поля: + +- `key` +- `value_json` +- `value_type` +- `title` +- `description` +- `category` +- `updated_at` + +Важно: `value_json` всегда JSONB, даже если настройка выглядит как строка. + +### 5.10. `ai_qualification_batches` + +История запросов AI-квалификатора. + +Хранит: + +- provider/model; +- prompt hash; +- полный prompt text; +- request JSON; +- response JSON; +- количество токенов; +- примерную стоимость; +- счётчики accepted/rejected/maybe; +- ошибку, если batch упал. + +### 5.11. `ai_writer_batches` + +История запросов AI-райтера. + +Хранит: + +- provider/model; +- prompt hash; +- полный prompt text; +- request JSON; +- response JSON; +- количество токенов; +- примерную стоимость; +- ready/failed count; +- ошибку. + +Важно: после фикса batch с ошибкой также должен сохранять `response_json`, если модель вернула валидный JSON неправильной структуры. + +### 5.12. `content_categories` + +Категории публикаций для AI-райтера и редакторской. + +Поля: + +- `id` - внутренний технический PK; +- `sort_order` - публичный стабильный ID категории для AI; +- `name` - человекочитаемое название/описание; +- `tag` - тэг, который будет подставляться системой; +- `is_active`; +- `created_at`; +- `updated_at`. + +Решение: `sort_order` используется как `category_id` для AI-райтера. В UI он отображается как `ID`, readonly. Модель возвращает только `category_id`, а код сам подставляет `name` и `tag` из БД. + +### 5.13. `audit_log` + +Журнал действий админки. + +Хранит: + +- `id`; +- `actor_id`; +- `action`; +- `entity_type`; +- `entity_id`; +- `before_json`; +- `after_json`; +- `created_at`. + +В `after_json` дополнительно пишется request-контекст, если он доступен: + +- `ip`; +- `method`; +- `path`; +- `query`; +- `user_agent`; +- `status_code` для request-level записей. + +Retention: записи старше 30 дней удаляются при старте приложения, при применении миграции `022_audit_log_retention_and_indexes.sql` и при открытии вкладки `/logs`. + +### 5.14. `publication_runs` + +Защита расписания постеров от повторной публикации одного и того же слота. + +Поля: + +- `poster` - тип постера, сейчас `tg`; +- `schedule_id` - ID строки расписания; +- `scheduled_for` - дата по Екб; +- `scheduled_time` - время строки расписания; +- `planned_count`; +- `published_count`; +- `status`; +- `error`. + +Уникальность `poster + schedule_id + scheduled_for` гарантирует, что слот расписания за день будет обработан один раз. + +Важно: TG-постер не должен публиковать все прошедшие за день слоты при включении. Он обрабатывает только текущий слот расписания в коротком окне после заданного времени и перед проверкой перечитывает настройки, включая `tg_poster_chat_id`. + +## 6. Статусы + +### 6.1. `raw_posts.status` + +Основные значения: + +- `storage_pending` - пост сохранён, ждёт загрузки в Telegram storage. +- `storage_ready` - пост и медиа загружены/сохранены в storage. +- `skipped` - пост сохранён как мусорный, если включено `parser_store_skipped_posts`. +- `failed` - ошибка на этапе storage. + +### 6.2. `raw_posts.qualification_status` + +Основные значения: + +- `pending` / `NULL` - ещё не квалифицирован. +- `processing` - взят AI-квалификатором. +- `accepted` - подходит для рерайта. +- `rejected` - мусор/не подходит. +- `maybe` - спорный по модели. Сейчас финальная логика может сводить `maybe` к accepted/rejected в зависимости от score и min_score. +- `failed` - ошибка AI-квалификации. + +### 6.3. `raw_posts.rewrite_status` + +Основные значения: + +- `pending` / `NULL` - ещё не переписан. +- `processing` - взят AI-райтером. +- `ready` - рерайт готов. +- `failed` - ошибка рерайта. + +### 6.4. `raw_posts.editorial_status` + +Основные значения: + +- `review` - на проверке редактора. +- `edited` - сохранены ручные правки. +- `accepted` - принято к публикации. +- `rejected` - отклонено. + +### 6.5. `raw_posts.publication_status` + +Статус финальной публикации. Сейчас используется TG-постером, позже может расшириться под VK/MAX/site. + +Основные значения: + +- `pending` - пост ещё не опубликован. +- `published` - пост опубликован в Telegram. +- `publish_failed` - публикация не удалась, причина лежит в `publication_error`. + +## 7. VK-парсер + +Файл: `src/vk_parser_app/workers/parser.py`. + +### 7.1. Что делает + +1. Берёт активные источники из `sources`. +2. Если источник ещё не resolved, вызывает VK API и получает: + - `external_id`; + - `external_owner_id`; + - нормальное имя. +3. Загружает посты со стены через VK API. +4. Определяет временное окно парсинга: + - если есть `last_parsed_at`, берёт его минус overlap; + - иначе если есть `parse_from`, берёт его; + - иначе берёт `now - parser_new_source_lookback_days`. +5. Убирает уже известные посты по `(source_id, external_post_id)`. +6. При включенном `parser_dedupe_content_hash` также убирает дубли по `content_hash`. +7. Применяет политику мусорных постов. +8. Сохраняет raw post и media. +9. Создаёт job `vk.storage.copy`. + +### 7.2. Политика мусорных постов + +Настройки: + +- `parser_skip_empty_text` +- `parser_skip_no_media` +- `parser_skip_text_too_short` +- `parser_min_text_length` +- `parser_skip_reposts` +- `parser_store_skipped_posts` + +Если пост не проходит фильтр: + +- при `parser_store_skipped_posts=false` он вообще не сохраняется; +- при `true` сохраняется со статусом `skipped` и `skip_reason`. + +Возможные `skip_reason`: + +- `empty_text` +- `no_media` +- `text_too_short` + +### 7.3. Дедупликация + +Есть два уровня: + +1. Точный дубль по `source_id + external_post_id`. +2. Контентный дубль по `content_hash`, если включен `parser_dedupe_content_hash`. + +`content_hash` считается по тексту и media attachment ids. + +### 7.4. Настройки VK-парсера + +Ключевые: + +- `parser_interval_sec` +- `parser_new_source_lookback_days` +- `parser_reparse_overlap_minutes` +- `parser_min_text_length` +- `parser_skip_empty_text` +- `parser_skip_no_media` +- `parser_skip_text_too_short` +- `parser_skip_reposts` +- `parser_store_skipped_posts` +- `parser_dedupe_content_hash` +- `parser_source_pause_sec` +- `vk_requests_per_second` +- `vk_wall_page_size` +- `vk_api_timeout_total_sec` +- `vk_api_timeout_connect_sec` +- `vk_rate_limit_sleep_sec` +- `vk_api_retry_attempts` +- `vk_api_retry_min_delay_sec` +- `vk_api_retry_max_delay_sec` + +## 8. Media storage uploader + +Файл: `src/vk_parser_app/workers/vk_storage_uploader.py`. + +Название файла историческое: сейчас фактическое хранилище медиа - Telegram, а не VK. + +### 8.1. Что делает + +1. Забирает job типа `vk.storage.copy`. +2. Загружает `raw_post` и media. +3. Скачивает изображения/видео. +4. Отправляет в Telegram storage channel. +5. Сохраняет Telegram file ids и message ids. +6. Помечает post как `storage_ready`. + +### 8.2. Почему Telegram storage + +Изначально рассматривался VK storage через приватную группу VK. От идеи отказались, потому что: + +- права VK API и редактирование постов создают лишнюю сложность; +- stable raw проще держать через Telegram bot API; +- старый проект уже имел рабочую логику Telegram uploader; +- Telegram file ids удобны для повторного отображения и хранения. + +### 8.3. Ограничения Telegram + +Uploader учитывает: + +- media group limit до 10 элементов; +- caption limit; +- message limit; +- flood control через `TelegramRetryAfter`; +- retry/backoff; +- размер видео; +- длительность видео; +- отдельные задержки между media upload и job. + +### 8.4. Текст и ссылка на оригинал + +Логика должна стремиться к такому формату: + +- медиа и текст отправляются вместе, если текст влезает в caption; +- если текст не влезает, текст отправляется отдельным сообщением/частями; +- в конец добавляется ссылка на оригинал; +- `#id` должен быть ссылкой на оригинальный VK-пост. + +### 8.5. Временные файлы + +Временные файлы uploader должны жить в `/tmp` с префиксом `vkparser_tg_media_`. Для видео используется `yt-dlp` и временные файлы/netrc. Нужно регулярно проверять, что временные файлы удаляются после upload/failure. Это важный пункт техдолга. + +### 8.6. Настройки uploader + +Исторически часть ключей в коде называется `telegram_*`, а часть миграций/админки - `media_*`, `tg_*`, `uploader_*`. + +Это стоит привести к единому виду. Сейчас важно проверять конкретный ключ в коде. + +Ключевые настройки: + +- `tg_media_channel_id` +- `local_bot_api_url` +- `uploader_interval_sec` +- `uploader_download_timeout_sec` +- `media_group_max_items` +- `media_upload_delay_sec` +- `media_post_job_pause_sec` +- `max_media_attempts` +- `tg_retry_attempts` +- `tg_retry_backoff_max_sec` +- `video_max_size_mb` +- `video_max_duration_sec` +- `uploader_yt_dlp_timeout_sec` + +Потенциальное расхождение: в коде встречаются `telegram_media_group_max_items`, `telegram_media_upload_delay_sec`, `telegram_retry_max_attempts`, `telegram_retry_backoff_max_sec`, `telegram_caption_limit`, `telegram_message_limit`, `telegram_text_overflow_marker`, `uploader_max_media_attempts`. Это надо унифицировать. + +## 9. AI-квалификатор + +Файл: `src/vk_parser_app/workers/ai_qualifier.py`. + +### 9.1. Задача + +Отсеять мусорные или неподходящие raw-посты до рерайта. + +Примеры мусора: + +- мемы; +- политика; +- вакансии; +- низкоконтентные посты; +- скидка без полезной новости; +- оффтоп; +- неправильный язык; +- injection/попытка управлять моделью; +- оружие, если оно не подходит под редакционную политику. + +### 9.2. Вход в модель + +AI получает JSON-массив: + +```json +[ + { + "id": 123, + "source": "Название источника", + "original_url": "https://vk.com/wall-...", + "media_count": 3, + "media_types": ["photo"], + "text": "исходный текст" + } +] +``` + +Текст обрезается по `ai_qualifier_max_text_chars`. + +### 9.3. Prompt + contract + +Промпт разделён на две части: + +1. `ai_qualifier_prompt` - свободная редакционная инструкция. +2. `ai_qualifier_contract` - технический контракт JSON. + +Контракт требует: + +```json +{ + "results": [ + { + "id": 123, + "score": 8, + "decision": "accepted", + "reason": "до 10 слов на русском", + "reject_tag": null + } + ] +} +``` + +### 9.4. Настройки + +- `ai_qualifier_enabled` +- `ai_qualifier_provider` +- `ai_qualifier_model` +- `ai_qualifier_api_key` +- `ai_qualifier_api_base` +- `ai_qualifier_prompt` +- `ai_qualifier_contract` +- `ai_qualifier_batch_size` +- `ai_qualifier_min_score` +- `ai_qualifier_max_text_chars` +- `ai_qualifier_temperature` +- `ai_qualifier_timeout_sec` +- `ai_qualifier_interval_sec` + +### 9.5. Деньги и токены + +`ai_qualification_batches` хранит: + +- `prompt_tokens` +- `completion_tokens` +- `total_tokens` +- `estimated_cost_usd` + +Эта информация нужна, чтобы контролировать стоимость экспериментов с prompt/model. + +## 10. AI-райтер + +Файл: `src/vk_parser_app/workers/ai_writer.py`. + +### 10.1. Задача + +Взять посты: + +- `status='storage_ready'` +- `qualification_status='accepted'` +- `rewrite_status is NULL/pending` + +И переписать их в короткую публикацию для Telegram/VK-канала. + +Важно: автоматический AI writer не должен забирать `rewrite_status='failed'`. +2026-07-30 один пост (`id=2348`) был платно отправлен в Sonnet 4.6 983 раза, +потому что failed снова попадал в claim и цикл сразу продолжал работу. Failed +посты нужно возвращать в `pending` только явным ручным действием. +Sonnet 4.6 в этом кейсе возвращал `category_id=18` (`Не целевой контент`) и +пустой `text`, объясняя отказ в `notes`; это валидный AI-отказ, а не ошибка +рерайта. Проверка минимальной длины текста применяется только к целевым +категориям. +После failed AI batch worker помечает конкретные посты как `failed` и продолжает +обрабатывать другие `pending` посты. Ошибка показывается на `/workers` из +heartbeat meta и отправляется Telegram-уведомлением получателям +`daily_report_recipient_ids` через `daily_report_bot_token`, затем +`tg_poster_bot_token`, затем `TG_BOT_TOKEN`. + +AI-квалификатор делает ровно один автоматический retry: после первого failed +batch пост возвращается в `qualification_status='pending'`, после второго failed +batch остаётся `failed` и больше не попадает в автоматический claim. + +### 10.2. Вход в модель + +AI получает JSON-объект: + +```json +{ + "task": "rewrite_accepted_posts", + "categories": [ + { + "id": 1, + "name": "Одежда", + "tag": "одежда" + }, + { + "id": 2, + "name": "Снаряжение - подсумки, чехол для плит, нагрудники, боевые пояса и тд", + "tag": "снаряжение" + } + ], + "posts": [ + { + "id": 123, + "producer_name": "academy_gear", + "producer_tag": "academy_gear", + "original_url": "https://vk.com/wall-...", + "qualification_score": 8, + "qualification_reason": "...", + "media_count": 4, + "media_types": ["photo"], + "text": "исходный текст" + } + ] +} +``` + +Текст обрезается по `ai_writer_max_text_chars`. + +### 10.3. Prompt + contract + +Промпт разделён на: + +1. `ai_writer_prompt` - редакционная инструкция. +2. `ai_writer_contract` - технический JSON-контракт. + +Свободная часть описывает: + +- роль редактора; +- знания в тактическом/милитари/airsoft-снаряжении; +- структуру поста; +- стиль; +- запрет на выдумывание фактов; +- обязательное явное упоминание `producer_name` в тексте каждого рерайта; +- выбор категории. + +Технический контракт требует: + +```json +{ + "rewrites": [ + { + "id": 123, + "category_id": 2, + "text": "готовый текст без ссылок и хэштегов", + "notes": "короткая заметка для редактора или null" + } + ] +} +``` + +Важно: модель возвращает `category_id`, а не тэг и не название. Код сам берёт категорию из `content_categories` и сохраняет: + +- `rewrite_category_id` +- `rewrite_category` +- `rewrite_category_tag` + +### 10.4. Почему category_id, а не category + +Название категории может быть длинным и описательным: + +```text +Снаряжение - подсумки, чехол для плит, нагрудники, боевые пояса и тд +``` + +Модели легче вернуть число `2`, чем точно повторить длинную строку. Это снижает ошибки валидации и позволяет менять описание категории без переписывания логики. + +### 10.5. Категории + +Категории редактируются в `/workers`, блок "Категории публикаций". + +Поля: + +- `Название` +- `Тэг` +- `ID` +- `Статус` + +`ID` отображается как readonly. В БД это поле `content_categories.sort_order`. + +### 10.6. Настройки + +- `ai_writer_enabled` +- `ai_writer_provider` +- `ai_writer_model` +- `ai_writer_api_key` +- `ai_writer_api_base` +- `ai_writer_prompt` +- `ai_writer_contract` +- `ai_writer_categories` - legacy, сейчас основной источник `content_categories` +- `ai_writer_batch_size` +- `ai_writer_max_text_chars` +- `ai_writer_temperature` +- `ai_writer_timeout_sec` +- `ai_writer_interval_sec` + +Текущая рекомендация: держать `ai_writer_batch_size=1`, потому что редактировать и отлаживать результат по одному посту проще и безопаснее. + +### 10.7. Последний проверенный тест + +Один контролируемый прогон `ai-writer`: + +- batch `#6` +- `posts_count=1` +- `status=done` +- model `anthropic/claude-haiku-4-5-20251001` +- результат для post `#26` +- `category_id=1` +- `rewrite_category=Одежда` +- `rewrite_category_tag=одежда` +- `rewrite_status=ready` +- `editorial_status=review` +- стоимость около `$0.007296` + +После теста writer был возвращён в безопасное состояние: + +- `ai_writer_enabled=false` +- `ai_writer_batch_size=1` +- `worker_controls.ai-writer.enabled=false` +- `ai-writer.service=active`, но работу не берёт. + +## 11. Prompt-test + +Страница: `/prompt-test`. + +Назначение: песочница для проверки prompt райтера на существующем raw-посте. + +Важно: + +- сейчас тестируется именно `ai_writer_prompt`; +- `ai_writer_contract` намеренно не добавляется в тест, чтобы можно было свободно смотреть стиль; +- входной payload должен совпадать с боевым форматом райтера; +- можно выбрать пост по фильтру квалификации; +- ответ отображается как человеческий текст с нормальными переносами, плюс raw JSON. + +Риск: если тестировать prompt без contract, модель может не вернуть JSON. Это нормально для песочницы, но боевой worker всегда склеивает prompt + contract. + +## 11.5. TG poster + +Файл: `src/vk_parser_app/workers/tg_poster.py`. + +Назначение: брать посты, принятые редактором, и публиковать их в Telegram по расписанию. Это первый финальный poster; позже рядом должны появиться VK/MAX/site poster-воркеры. + +### 11.5.1. Условия публикации + +Пост берётся в очередь TG-постера, если: + +- `raw_posts.status='storage_ready'`; +- `rewrite_status='ready'`; +- `editorial_status='accepted'`; +- `publication_status='pending'`. + +После успеха: + +- `publication_status='published'`; +- `editorial_status='published'`, поэтому пост уходит из вкладки "Принято"; +- заполняются `published_at`, `tg_publication_chat_id`, `tg_publication_thread_id`, `tg_publication_message_ids`, `tg_publication_url`. + +После ошибки: + +- `publication_status='publish_failed'`; +- `editorial_status='publish_failed'`; +- причина пишется в `publication_error`; +- автоматический повтор не делается, чтобы ошибка не перетиралась на каждом слоте расписания. + +### 11.5.2. Настройки + +Категория настроек: `TG Poster`. + +Ключевые: + +- `tg_poster_enabled` - второй защитный флаг, синхронизируется с кнопкой worker toggle; +- `tg_poster_bot_token` - отдельный токен бота для финальной публикации; если пустой, worker берёт `TG_BOT_TOKEN`; +- `tg_poster_chat_id` - куда публиковать, для теста `-1003712724327`; +- `tg_poster_schedule_json` - строки расписания; +- `tg_poster_interval_sec`; +- `tg_poster_caption_limit`; +- `tg_poster_message_limit`; +- `tg_poster_media_group_max_items`; +- `tg_poster_max_attempts`; +- `tg_poster_retry_backoff_max_sec`; +- `tg_poster_send_delay_sec`; +- `tg_poster_text_overflow_caption`; +- `tg_poster_recent_window`; +- `tg_poster_category_repeat_penalty`; +- `tg_poster_source_repeat_penalty`. + +Расписание редактируется на `/workers` отдельным блоком "Расписание TG-постера". Время указывается по Екатеринбургу (`Asia/Yekaterinburg`). Сейчас по умолчанию стоит 24 строки: каждый час `HH:00`, по 4 поста. + + +### 11.5.3. Telegram-лимиты + +Worker учитывает: + +- caption limit до 1024 символов; +- message limit до 4096 символов с разбиением длинного текста на части; +- media group limit до 10 элементов; +- одиночное медиа отправляется через `send_photo`/`send_video`, а не через album API; +- flood control через `TelegramRetryAfter`; +- retry/backoff; +- задержку между отправками. + +Если текст вместе с хэштегами влезает в caption, он публикуется под первым медиа. Если не влезает, caption становится `⬇️ Описание`, а полный текст публикуется следующим сообщением. + +### 11.5.4. Формат текста + +Финальный текст собирается системно: + +```text +пост + +#category #source +``` + +Категорийный хэштег берётся из `content_categories.tag`, source-тэг - из `final_source_tag`, `rewrite_source_tag` или `sources.tag`. + +### 11.5.5. Ранжирование + +Логика intentionally простая: + +1. Берём кандидатов из принятых неопубликованных постов. +2. Базовый порядок: выше `qualification_score`, затем более старые принятые посты. +3. Смотрим последние `tg_poster_recent_window` опубликованных постов. +4. Штрафуем кандидата за повторы категории и источника среди недавних публикаций. +5. При выборе пачки дополнительно штрафуем повторы внутри текущего слота. + +Так категории и источники реже идут подряд, но алгоритм остаётся понятным и предсказуемым. + +## 12. Админка + +Основной файл: `src/vk_parser_app/admin.py`. + +### 12.1. Основные страницы + +- `/login` - вход. +- `/sources` - источники парсинга. +- `/raw` - raw-посты. +- `/raw/{id}` - подробная карточка raw-поста, AI-ответы, batch info. +- `/editor` - редакторская лента. +- `/workers` - воркеры, настройки, категории публикаций, расписание TG-постера. +- `/logs` - журнал действий, запросов и служебных событий админки. +- `/users` - пользователи. +- `/prompt-test` - тест промпта райтера. +- `/vk/oauth/callback` - callback для VK OAuth, исторически нужен для экспериментов с VK токенами. + +### 12.2. Sources + +На странице источников можно: + +- добавлять источник; +- добавлять пачкой; +- редактировать; +- включать/выключать; +- архивировать; +- видеть статус и ошибки. + +Для VK важны: + +- platform `vk` +- name +- tag +- url +- active +- parse_from + +### 12.3. Raw + +Raw-страница показывает: + +- источник; +- этап; +- пост; +- AI-оценку; +- медиа-превью; +- дату; +- фильтры справа. + +Фильтры построены как sidebar в стиле интернет-магазина: + +- score min/max; +- дата; +- источник; +- категория; +- raw status; +- qualification status; +- rewrite status. + +### 12.4. Editor + +Редакторская нужна для постов после AI-райтера. + +Возможности: + +- посмотреть посты на проверке; +- принять; +- отклонить; +- открыть редактирование; +- изменить текст; +- поменять категорию; +- поменять source tag; +- скрыть/добавить медиа; +- сохранить правки; +- принять к публикации. + +Текущая логика: + +- `Сохранить правки` сохраняет изменения, но не принимает пост. +- `Принять к публикации` сохраняет и переводит в `accepted`. + +### 12.5. Workers + +Страница `/workers` содержит: + +- список воркеров и heartbeat; +- категории публикаций; +- расписание TG-постера; +- настройки по разделам. + +Prompt-настройки имеют подсказки: + +- что можно менять безопасно; +- что приходит на вход; +- базовая JSON-схема. + +### 12.6. Logs + +Страница `/logs` показывает `audit_log` за последние 30 дней. + +Что логируется: + +- успешные, неуспешные и заблокированные входы; +- выходы; +- GET/POST-запросы авторизованной админки, кроме `/health`, favicon и `/uploads`; +- существующие предметные действия: источники, редакторская, категории, воркеры, настройки, пользователи, prompt-test. + +Фильтры: + +- поиск по action/entity/JSON; +- action; +- пользователь; +- entity type; +- диапазон дат. + +Старые записи чистятся автоматически, чтобы журнал не рос бесконечно. + +## 13. Безопасность админки + +Сейчас реализовано: + +- password hash; +- session token hash; +- CSRF token; +- таблица `admin_login_attempts`; +- ограничение попыток входа; +- users management; +- fail2ban/ufw на сервере были добавлены в рамках настройки сервера. + +Что важно помнить: + +- API keys сейчас могут отображаться в админке целиком для удобства. Нельзя открывать страницу настроек при демонстрации экрана. +- Нужно держать `.env` вне git. +- Нужно регулярно менять временные пароли. +- Root SSH по паролю лучше заменить на ключи и отключить password login. + +## 14. Миграции + +Миграции лежат в `db/migrations`. + +Применение: + +```bash +cd /opt/vk-parser +PYTHONPATH=src .venv/bin/python scripts/apply_migrations.py +``` + +Механика: + +- `schema_migrations` хранит имена применённых файлов; +- файлы выполняются по алфавитному порядку; +- каждая миграция выполняется в transaction; +- повторно применённая миграция пропускается. + +Последние важные миграции: + +- `017_content_categories_and_media_cleanup.sql` - категории публикаций. +- `018_writer_prompt_contract_cleanup.sql` - JSON-only вынесен из prompt в contract. +- `019_writer_prompt_profile_style.sql` - новый подробный prompt райтера. +- `020_writer_category_ids.sql` - `category_id` для AI-райтера, новые поля в raw_posts. +- `021_writer_contract_no_empty_response.sql` - усиленный writer contract и сохранение неправильного ответа. +- `024_tg_poster.sql` - TG-постер, статусы публикации, расписание, лимиты и prompt-лимит райтера под caption. +- `026_writer_separator_17_chars.sql` - строгая разделительная полоса `━━━━━━━━━━━━━━━━━` в prompt/contract и существующих рерайтах. +- `031_writer_require_producer_name.sql` - AI-райтер обязан явно упоминать `producer_name` в тексте каждого рерайта. +- `032_tg_reactor.sql` - TG-реактор, таблица `post_reactions`, настройки реакций и worker flag. +- `033_tg_reactor_since.sql` - настройка `tg_reactor_since`, чтобы реактор не проходил по старым TG-публикациям при включении. +- `034_tg_reactor_reaction_pause.sql` - пауза между успешными реакциями TG-реактора, чтобы безопасно догонять архив. +- `035_uploader_video_format_limits.sql` - ограничение высоты VK-видео для `yt-dlp` и увеличение timeout скачивания, чтобы ролики укладывались в Telegram Bot API. +- `039_max_poster.sql` - MAX-постер, расписание как у TG, настройки MAX Bot API, лимиты сообщения/медиа, ожидание обработки видео и double safety flags. + +## 15. Локальный запуск + +Установка: + +```bash +python -m venv .venv +.venv\Scripts\activate +pip install -r requirements.txt +``` + +Для Windows PowerShell: + +```powershell +$env:PYTHONPATH="src" +python scripts/apply_migrations.py +uvicorn vk_parser_app.admin:app --host 0.0.0.0 --port 8080 +``` + +Воркеры локально: + +```powershell +$env:PYTHONPATH="src" +python -m vk_parser_app.workers.parser +python -m vk_parser_app.workers.vk_storage_uploader +python -m vk_parser_app.workers.ai_qualifier +python -m vk_parser_app.workers.ai_writer +python -m vk_parser_app.workers.tg_poster +python -m vk_parser_app.workers.max_poster +``` + +На Windows при `py_compile` может быть ошибка доступа к `__pycache__`. Тогда: + +```powershell +$env:PYTHONPYCACHEPREFIX="$env:TEMP\n8_pycache" +python -m py_compile src\vk_parser_app\admin.py +``` + +## 16. Runtime / Deploy + +Актуальный runtime живёт в локальной инфраструктуре. Источник правды по хостам, +ключам, Coolify, Gitea и edge-proxy: `D:\DEVELOPMENT\.infra.md`. + +Текущая схема: + +```text +local repo + -> git push gitea main + -> Gitea webhook on 192.168.1.135 + -> Coolify on 192.168.1.135 + -> Docker app on LXC 105 / 192.168.1.105 + -> edge-proxy / Caddy + -> https://panel.f-n8.ru +``` + +Важно: + +- Не использовать для этого проекта старый VPS `109.120.156.205`. +- Не использовать `https://sw.exostring.xyz` как проверку текущей админки. +- Не выполнять `/opt/vk-parser`, `git pull` на VPS и `systemctl restart ...` + без явного указания пользователя, что речь именно о старом VPS. +- Основной deploy remote: `gitea`. +- Перед push/deploy обязательно делать `git pull --rebase gitea main`. +- Coolify app UUID: `n6cnr60anmruiwkiey0pukye` (`FN-8 admin`). API token хранится только в `D:\DEVELOPMENT\.infra.md`; не переносить секрет в репозиторий. +- URL репозитория в Coolify должен резолвиться из build/helper container на + deploy-сервере. Не использовать `http://coolify.lan:3000/...`, если это имя + не настроено внутри LXC/Docker; рабочий локальный URL Gitea: + `http://192.168.1.135:3000/exostring/new_vk_parser.git`. +- Coolify должен собирать и деплоить приложение после push/webhook; при ручном + деплое использовать Coolify API/UI из `D:\DEVELOPMENT\.infra.md`. + +Публичная админка: + +```text +https://panel.f-n8.ru +https://admin.f-n8.ru +``` + +## 17. Git workflow + +Локальный репозиторий: + +```bash +git status +git add ... +git commit -m "Message" +git pull --rebase gitea main +git push gitea main +``` + +`origin` остаётся GitHub/upstream/backup. Для deploy использовать `gitea`, +потому что Gitea webhook запускает Coolify. + +Перед каждым push не пропускать `git pull --rebase gitea main`: пользователь отдельно просил не забывать pull, потому что Coolify должен видеть свежий commit из Gitea. + +## 18. Контролируемый тест AI-райтера + +Важно: не включать AI writer обычным способом, если нужно проверить один пост. Иначе живой systemd service может начать брать очередь самостоятельно. + +Правильная схема: + +1. Остановить сервис: + +```bash +systemctl stop ai-writer.service +``` + +2. Временно включить: + +- `worker_controls.ai-writer.enabled=true` +- `app_settings.ai_writer_enabled=true` +- `app_settings.ai_writer_batch_size=1` + +3. Выполнить один `AIWriterWorker.run_once()`. + +4. Вернуть: + +- `worker_controls.ai-writer.enabled=false` +- `app_settings.ai_writer_enabled=false` +- `app_settings.ai_writer_batch_size=1` + +5. Запустить сервис обратно: + +```bash +systemctl start ai-writer.service +``` + +Так service остаётся active, но в disabled-режиме. + +## 19. Контролируемый тест AI-квалификатора + +Схема такая же: + +1. Остановить `ai-qualifier.service`. +2. Включить оба флага. +3. Поставить нужный `ai_qualifier_batch_size`. +4. Выполнить `AIQualifierWorker.run_once()`. +5. Вернуть настройки. +6. Запустить service обратно. + +Важно: при batch size 25 и активном service можно случайно получить несколько batch подряд. Так уже происходило. Поэтому сначала останавливать service. + +## 20. Известные хвосты и техдолг + +### 20.1. Старые AI writer batches + +В истории есть старые тестовые batches: + +- batch `#4` со статусом `processing`; +- batch `#5` со статусом `failed`, созданный во время теста на 5 постов. + +Они не влияют на текущую очередь, но могут путать в админке. Их стоит аккуратно пометить как failed/cancelled или скрывать служебные тесты. + +### 20.2. Название `vk_storage_uploader.py` + +Файл исторически называется VK storage uploader, но фактически загружает в Telegram. Можно позже переименовать: + +```text +vk_storage_uploader.py -> tg_storage_uploader.py +JOB_TYPE_VK_STORAGE_COPY -> tg.storage.copy +``` + +Делать только миграцией и аккуратно, чтобы не потерять jobs. + +### 20.3. Настройки Telegram uploader + +Часть ключей в UI/миграциях и в коде отличается по именам. Нужно унифицировать, чтобы админка реально управляла всеми параметрами uploader. + +### 20.4. README устарел + +`README.md` был написан на раннем этапе, когда ещё рассматривался VK storage. Сейчас он полезен как quick start, но его надо обновить или заменить ссылкой на `PROJECT.md`. + +### 20.5. Медиа cleanup + +Нужно отдельно проверить: + +- удаляются ли временные файлы из `/tmp`; +- что происходит с локально загруженными медиа редактора; +- как чистить orphan files; +- как чистить Telegram storage, если редактор удалил медиа. + +### 20.6. Категории и старые посты + +Новые посты получают `rewrite_category_id`. Старые посты могут иметь только текстовую категорию. Это нормально, но фильтры/отображение должны учитывать оба варианта. + +### 20.7. Prompt-test без contract + +Это осознанное решение. Но если хочется тестировать боевой формат, стоит добавить переключатель: + +- "тестировать только стиль" +- "тестировать style + contract" + +## 21. Что делать дальше + +Ближайшие полезные задачи: + +1. Обновить `README.md`, чтобы он не противоречил реальности. +2. Унифицировать ключи настроек Telegram uploader. +3. Добавить отдельную страницу batch history для AI. +4. Аккуратно закрыть/пометить старые зависшие AI writer batches. +5. Улучшить отображение категорий в editor/raw: + - показывать `category_id`; + - показывать `tag`; + - показывать source tag. +6. Добавить режим "боевой prompt-test" с contract. +7. Добавить будущие постеры рядом с TG-постером: + - VK; + - MAX; + - сайт. +8. Добавить backup strategy: + - PostgreSQL dump; + - `.env` отдельно; + - uploads/editor_media. +9. Навести порядок в UI, если админка кажется слишком самописной: + - либо продолжить текущий server-rendered UI; + - либо вынести frontend в React/shadcn; + - либо подключить готовую admin-систему. + +## 22. Короткий mental model + +Если нужно быстро вспомнить проект: + +```text +sources + -> parser + -> raw_posts + raw_post_media + -> jobs(vk.storage.copy) + -> Telegram storage uploader + -> status=storage_ready + -> AI qualifier + -> qualification_status=accepted/rejected + -> AI writer + -> rewrite_status=ready, editorial_status=review + -> editor + -> accepted/rejected/edited + -> tg-poster + -> publication_status=published/publish_failed +``` + +Главные таблицы: + +```text +sources +raw_posts +raw_post_media +jobs +worker_controls +worker_heartbeats +app_settings +publication_runs +ai_qualification_batches +ai_writer_batches +content_categories +admin_users +admin_sessions +``` + +Главные файлы: + +```text +src/vk_parser_app/admin.py +src/vk_parser_app/workers/parser.py +src/vk_parser_app/workers/vk_storage_uploader.py +src/vk_parser_app/workers/ai_qualifier.py +src/vk_parser_app/workers/ai_writer.py +src/vk_parser_app/workers/tg_poster.py +src/vk_parser_app/vk_api.py +db/migrations/*.sql +``` + +Главное правило эксплуатации: + +```text +Перед тестом AI-воркера останавливай systemd service, +иначе живой worker может параллельно забрать очередь. +``` + +## 23. Сайт Forma N8 на Ghost + +21 июня 2026 года лента сайта перенесена на Ghost CMS 6.46.0. Это основа +для дальнейшей работы над сайтом, SEO и отдельным site-poster worker. + +### 23.1. Сервер и домен + +- VPS: `139.100.234.51`; +- SSH: `root@139.100.234.51:22`, пароль `7LaoHsQ1JxnK`; +- подключение к этому серверу выполнять из Python через `paramiko`; +- домен: `f-n8.ru`; +- Ghost: `/var/www/f-n8.ru`; +- systemd: `ghost_f-n8-ru.service`; +- nginx vhost: `/etc/nginx/sites-available/f-n8.ru`; +- MySQL database: `ghost_prod`; +- публичный Ghost порт слушает только `127.0.0.1:2368`. + +Сервер основного парсера доступен по SSH: `root@109.120.156.205:22`, +пароль `OU8gUSCtQYcf`. Подключение к обоим серверам выполняется через +Python-библиотеку `paramiko`. + +23 июня 2026 года устранён 502 после автоматического обновления MySQL: +`unattended-upgrade` остановил MySQL, а жёсткая зависимость +`Requires=mysql.service` вслед за ним остановила Ghost без последующего +запуска. В `ghost_f-n8-ru.service` зависимость заменена на +`Wants=mysql.service` при сохранённом `After=mysql.service`; unit включён и +использует `Restart=always`. Резервная копия исходного unit: +`/etc/systemd/system/ghost_f-n8-ru.service.bak-20260623-1115`. + +25 июня 2026 года из основного CSS Ghost Admin удалены 21 внешних +`@import` с `fonts.bunny.net`: CDN сбрасывал соединения у части клиентов, +из-за чего `/ghost/` оставался на бесконечной загрузке. В `index.html` к +административному CSS добавлен cache-busting query +`?v=localfonts-20260625`; резервная копия находится в +`/root/forma-n8-backups/ghost-admin-fonts-20260624-190120/`. Обновление +Ghost может перезаписать этот патч, поэтому после обновлений нужно проверить +наличие `fonts.bunny.net` в `current/core/built/admin/assets/*.css`. + +В тот же день отключена проверка нового устройства для staff-входов: +`security.staffDeviceVerification=false` в +`/var/www/f-n8.ru/config.production.json`. Без рабочего SMTP эта проверка +завершалась ошибкой `Failed to send email` и не позволяла новым +администраторам войти. Резервная копия конфигурации: +`/var/www/f-n8.ru/config.production.json.bak-device-verification-20260624-190851`. + +22 июня 2026 года DNS начал разрешаться публично: `f-n8.ru` и +`www.f-n8.ru` указывают на `139.100.234.51`. Выпущен сертификат Let's +Encrypt для обоих имён, включено автоматическое обновление через +`certbot.timer`, HSTS и принудительный HTTPS. Основной адрес Ghost и +canonical URL: `https://f-n8.ru`; `www` перенаправляется на основной домен. + +Для подтверждения сайта в Яндекс Вебмастере nginx отдаёт статический файл +`https://f-n8.ru/yandex_9c5e1348b34e9b34.html`; исходный файл хранится в +`/var/www/f-n8.ru/yandex_9c5e1348b34e9b34.html`, exact-location настроен в +`/etc/nginx/sites-available/f-n8.ru`. + +### 23.2. Тема и контент + +- исходники темы в репозитории: `forma_n8_site/ghost-theme/forma-n8`; +- установленная тема: `/var/www/f-n8.ru/content/themes/forma-n8`; +- тема проходит Ghost gscan без ошибок; +- favicon и кириллические Roboto Condensed лежат внутри темы; +- постоянный URL публикаций: `/news/{slug}/`; +- категории и производители переносятся в Ghost tags; +- RSS: `/rss/`; +- sitemap: `/sitemap.xml`; +- админка: `/ghost/`. + +В Ghost перенесены 12 опубликованных материалов старой ленты с исходными +slug, датами, SEO-полями и локальными изображениями. Стартовые публикации +Ghost удалены. Старый FastAPI-сайт и PostgreSQL пока сохранены как резерв +на сервере; предмиграционный backup находится в +`/root/forma-n8-backups/20260621-172603/`. + +### 23.3. Доступы и будущий site-poster + +Секреты не хранятся в git: + +- MySQL credentials: `/root/ghost-db.env`; +- initial owner credentials: `/root/ghost-owner-credentials`; +- Admin API key интеграции `Forma N8 Site Poster`: + `/root/ghost-admin-api-key`. + +Site-poster использует Ghost Admin API key, создаёт публикации через Admin +API, загружает медиа в Ghost и записывает внешний Ghost post id/url в общую +таблицу публикаций. Прямые INSERT в Ghost posts для рабочего постинга +запрещены. + +### 23.4. Категории сайта + +Миграция `037_site_category_fields.sql` разделяет три разных назначения +категории: + +- `name` — подробное описание категории для AI; +- `tag` — неизменяемый тэг для хэштега в соцсетях; +- `site_name` — тэг с заменой `_` на пробел и нормальным регистром; +- `site_slug` — стабильный латинский хвост URL; +- `site_enabled` — разрешение публикации категории на сайт. + +Категория `18` / `НЦК` имеет `site_enabled=FALSE`. Архивы Ghost используют +URL `/category/{site_slug}/`; URL публикации не содержит категорию и +остаётся `/news/{slug}/`, чтобы смена категории не ломала canonical. +Производители хранятся во внутренних Ghost tags и не попадают в sitemap +как категории. + +### 23.5. Изображения и аналитика + +В ленте показывается только `feature_image`. Если у публикации есть +дополнительные изображения, site-poster должен добавить их в тело статьи +нативной Ghost Gallery; тема поддерживает плитку и полноэкранный просмотр. + +Перед загрузкой изображения нужно: + +- применить EXIF orientation и удалить метаданные; +- ограничить размер до 2000x2000 без увеличения; +- сохранить в WebP quality 82; +- первое изображение назначить обложкой, остальные поместить в галерею. + +Контрольный замер на первых 12 публикациях: исходные изображения занимали +12,09 МБ, после такой обработки — 3,18 МБ, в среднем 272 КБ на файл. + +В тему добавлен счётчик Яндекс Метрики `110060335` с clickmap, +trackLinks, accurateTrackBounce и webvisor. +Google Tag Manager подключён в общем Ghost-шаблоне с контейнером +`GTM-MM4FR8LJ`: script находится в начале ``, noscript iframe — сразу +после открывающего ``. + +### 23.6. Ручной batch site-poster + +Скрипт `scripts/publish_site_batch.py` публикует явно выбранные raw-посты +через Ghost Admin API: + +```bash +PYTHONPATH=/opt/vk-parser/src .venv/bin/python \ + scripts/publish_site_batch.py --ids 26,455 +``` + +Скрипт: + +- не публикует `site_enabled=FALSE` и уже опубликованные на сайт записи; +- скачивает сохранённые фото через локальный Telegram Bot API; +- применяет EXIF orientation, 2000x2000, WebP quality 82; +- назначает первое фото `feature_image`; +- собирает остальные фото в Ghost Gallery; +- создаёт статью через Admin API; +- записывает результат или ошибку в `post_publications` с platform `site`. + +Первый боевой batch 22 июня 2026 года: 10 статей, 46 фотографий, +все публикации успешны. Upload payload после нормализации занял около +10,9 МБ. Ghost штатно хранит рабочую версию и исходную WebP-копию `_o` +для последующей регенерации размеров: суммарно 92 файла / 21,12 МБ на диске. + +В тот же день выполнен полный photo-ready backfill. В `post_publications` +создан backfill для 12 ранних импортов, после чего опубликован весь +оставшийся хвост без дублей. Итог: + +- 310 Ghost posts / 310 `site · published`; +- 0 `site · publish_failed`; +- 0 оставшихся photo-ready постов; +- 16 video-only постов опубликованы через нативные Ghost video cards; +- 5 длинных VK-видео остаются `link_only / video too long` без Telegram + file_id и не публикуются, чтобы на сайте не было карточек без медиа; +- video pipeline загружает MP4 через Ghost `/media/upload/`, извлекает + WebP-обложку ffmpeg и хранит ролик в нативном Lexical `video` node; +- ролики больше 40 МБ или выше 720p перекодируются в H.264/AAC, + максимум 720p, target около 38 МБ, preset `fast`, `faststart`; +- после photo backfill каталог Ghost images занимал около 453 МБ; + актуальный общий расход проверяется отдельно с учётом MP4. + +Режим полного backfill: + +```bash +PYTHONPATH=/opt/vk-parser/src .venv/bin/python \ + scripts/publish_site_batch.py --all-ready +``` + +Он исключает уже опубликованные записи и ранжирует очередь так, чтобы +категории и источники реже повторялись подряд. + +### 23.6.1. Автоматический site-poster + +- worker: `src/vk_parser_app/workers/site_poster.py`; +- systemd: `site-poster.service`; +- двойное включение: `worker_controls.site-poster.enabled` и + `app_settings.site_poster_enabled`; +- интервал по умолчанию: 300 секунд; +- за проход: до 10 photo-ready и 1 video-only материала; +- worker вызывает общий проверенный pipeline `scripts/publish_site_batch.py`, + который защищён lock-файлом `/tmp/forma-n8-site-poster.lock` от параллельных + ручных и автоматических запусков; +- обычный автоматический запуск не повторяет `publish_failed`; для явного + ручного повтора используется флаг `--retry-failed`. + +### 23.7. Навигация Ghost + +- лента выводит 25 постов на страницу; +- H1 главной: `Новости тактического снаряжения и экипировки`; +- только на первой странице `/`, после ленты и пагинации, выводится + статический SEO-блок `О новостной ленте Forma N8` с описанием тематики, + категорий и производителей; на `/page/2/` и далее блок не дублируется; +- пагинация полностью русская; +- шапка: `Лента`, выпадающие `Категории`, `О проекте`, поиск; +- ссылки на каналы Forma N8 (`https://vk.com/forma_n8` и `https://t.me/forma_n8`) с SVG-иконками присутствуют в шапке, каждом превью, открытой статье, разделе `О проекте` и футере; +- описание проекта начинается с текста: «Ежедневно мы собираем новости снаряжения с сотен источников, отбираем лучшее и публикуем в одном месте!»; +- футер: `RSS`, `Карта сайта`; +- старые `/tag/{slug}/` перенаправляются на `/category/{slug}/`; +- `/about/` содержит описание проекта; +- ссылка `Первоисточник` удалена из существующих статей и не добавляется + новым site-poster batch. + +### 23.8. SEO Ghost + +- название сайта: `Forma N8`; +- слоган главной: `Forma N8 — все новости снаряжения в одном месте`; +- description: `Forma N8 — все новости снаряжения в одном месте. Новинки, обзоры и события производителей тактического, туристического и специального снаряжения.`; +- title статьи: `Forma N8 — Название статьи`; +- title раздела: `Forma N8 — Название раздела`; +- meta keywords и «SEO-хештеги» не используются: категории оформляются + обычными Ghost tags, формирующими структуру сайта и посадочные страницы; +- тема должна сохранять один H1, canonical, Open Graph, Twitter Cards, + JSON-LD, RSS и sitemap, которые формирует Ghost через `ghost_head`; +- SSH-доступ и изменения на серверах `139.100.234.51` и + `109.120.156.205` выполняются через Python-библиотеку `paramiko`. + +доступ к серверу по ssh root@109.120.156.205 -p 22 +пароль OU8gUSCtQYcf diff --git a/RAA.md b/RAA.md new file mode 100644 index 0000000..a936477 --- /dev/null +++ b/RAA.md @@ -0,0 +1,34 @@ +# RAA Parser/Poster + +RAA is a separate deployment copied from the FN-8 parser/poster codebase. + +## Runtime + +- Local workspace: `D:\DEVELOPMENT\raa-parser_poster` +- Public admin domain: `https://raa.panel.f-n8.ru` +- Site poster must stay disabled until a site adapter is configured. +- The current site-poster adapter boundary is `site_poster_provider`; only `ghost` + exists in code today. WordPress should be added as a code adapter, not as + arbitrary executable code stored in settings. + +## First-Run Settings + +Set real secrets in Coolify/env, not in git: + +- `APP_SECRET_KEY` +- `ADMIN_BOOTSTRAP_LOGIN` +- `ADMIN_BOOTSTRAP_PASSWORD` +- `DB_*` +- `TG_BOT_TOKEN` +- `TELEGRAM_API_ID` +- `TELEGRAM_API_HASH` +- `VK_ACCESS_TOKEN` or `VK_GROUP_ACCESS_TOKEN` + +Set runtime settings in the admin UI/DB: + +- `TG_MEDIA_CHANNEL_ID` +- `tg_poster_bot_token` +- `tg_poster_chat_id` +- `vk_poster_owner_id` +- `site_poster_enabled=false` +- `site_poster_provider=""` diff --git a/README.md b/README.md new file mode 100644 index 0000000..874cb65 --- /dev/null +++ b/README.md @@ -0,0 +1,199 @@ +# VK Parser New Core + +Первый чистый слой проекта: + +- админка с источниками, пользователями, воркерами и raw-лентой; +- VK parser worker; +- VK storage uploader worker; +- PostgreSQL job queue. + +Старый проект лежит в `original_project/` как справочник. Новый код живёт в `src/vk_parser_app/`. + +## Что нужно на сервере + +Ubuntu 22.04/24.04: + +```bash +sudo apt update +sudo apt install -y python3 python3-venv python3-pip postgresql postgresql-contrib nginx git curl +``` + +Для будущей работы с видео: + +```bash +sudo apt install -y ffmpeg +``` + +## БД + +Пример: + +```bash +sudo -u postgres psql +``` + +```sql +CREATE DATABASE vk_parser; +CREATE USER vk_parser_user WITH PASSWORD 'CHANGE_ME'; +GRANT ALL PRIVILEGES ON DATABASE vk_parser TO vk_parser_user; +\c vk_parser +GRANT ALL ON SCHEMA public TO vk_parser_user; +``` + +## Установка проекта + +```bash +cd /opt +sudo git clone vk-parser +sudo chown -R $USER:$USER /opt/vk-parser +cd /opt/vk-parser + +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +cp .env.example .env +``` + +Заполнить `.env`: + +```env +APP_SECRET_KEY=long-random-string +ADMIN_BOOTSTRAP_LOGIN=admin +ADMIN_BOOTSTRAP_PASSWORD=long-admin-password + +DB_HOST=localhost +DB_PORT=5432 +DB_NAME=vk_parser +DB_USER=vk_parser_user +DB_PASSWORD=... + +VK_ACCESS_TOKEN=... +VK_STORAGE_GROUP_ID=239548476 +``` + +Важно: для `wall.post`, `wall.edit`, `photos.saveWallPhoto` нужен VK user token пользователя с правами в storage-группе. Community token для этого контура не подходит. + +## Миграции + +```bash +PYTHONPATH=src .venv/bin/python scripts/apply_migrations.py +``` + +Первый админ создаётся из `ADMIN_BOOTSTRAP_LOGIN` / `ADMIN_BOOTSTRAP_PASSWORD` при старте админки. + +## Проверка VK storage + +Без фото: + +```bash +PYTHONPATH=src .venv/bin/python scripts/vk_storage_spike.py +``` + +С фото: + +```bash +cp /path/to/photo.jpg sample_photo.jpg +PYTHONPATH=src .venv/bin/python scripts/vk_storage_spike.py +``` + +Скрипт должен вывести ссылку вида: + +```text +https://vk.com/wall-239548476_123 +``` + +После этого надо руками проверить: + +- пост создан от имени storage-группы; +- админ/редактор группы может открыть и отредактировать пост; +- вложенное фото видно в браузере. + +## Запуск локально + +Админка: + +```bash +PYTHONPATH=src .venv/bin/uvicorn vk_parser_app.admin:app --host 0.0.0.0 --port 8080 +``` + +Парсер: + +```bash +PYTHONPATH=src .venv/bin/python -m vk_parser_app.workers.parser +``` + +VK storage uploader: + +```bash +PYTHONPATH=src .venv/bin/python -m vk_parser_app.workers.vk_storage_uploader +``` + +## systemd units + +Минимальный `vk-admin.service`: + +```ini +[Unit] +Description=VK Parser Admin +After=network.target postgresql.service + +[Service] +WorkingDirectory=/opt/vk-parser +Environment=PYTHONPATH=/opt/vk-parser/src +ExecStart=/opt/vk-parser/.venv/bin/uvicorn vk_parser_app.admin:app --host 0.0.0.0 --port 8080 +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target +``` + +`vk-parser.service`: + +```ini +[Unit] +Description=VK Parser Worker +After=network.target postgresql.service + +[Service] +WorkingDirectory=/opt/vk-parser +Environment=PYTHONPATH=/opt/vk-parser/src +ExecStart=/opt/vk-parser/.venv/bin/python -m vk_parser_app.workers.parser +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target +``` + +`vk-storage-uploader.service`: + +```ini +[Unit] +Description=VK Storage Uploader Worker +After=network.target postgresql.service + +[Service] +WorkingDirectory=/opt/vk-parser +Environment=PYTHONPATH=/opt/vk-parser/src +ExecStart=/opt/vk-parser/.venv/bin/python -m vk_parser_app.workers.vk_storage_uploader +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target +``` + +## Текущий pipeline + +```text +sources(vk) + -> vk-parser + -> raw_posts + raw_post_media + -> jobs(type='vk.storage.copy') + -> vk-storage-uploader + -> wall.post в storage-группе + -> raw_posts.storage_post_url +``` + +Фото копируются в storage-группу. Видео в первой версии сохраняются как `link_only`/исходное VK-вложение, чтобы сначала стабилизировать основной контур. diff --git a/STATE.md b/STATE.md new file mode 100644 index 0000000..97d0f57 --- /dev/null +++ b/STATE.md @@ -0,0 +1,142 @@ +# N8 Parser: current state + +Last updated: 2026-08-03 + +This file is the short handoff state for future Codex threads. Read this first before touching the project, so the whole chat history does not need to be carried forward. + +## Runtime / Deploy + +Current runtime is local infrastructure, not the old VPS. + +- Infrastructure source of truth: `D:\DEVELOPMENT\.infra.md`. +- Git deploy remote: `gitea` -> `http://192.168.1.135:3000/exostring/new_vk_parser.git`. +- Deploy path: push to `gitea/main` -> Gitea webhook -> Coolify -> Docker build/deploy. +- Coolify + Gitea LXC: `192.168.1.135` (CTID 102). +- Coolify app UUID: `n6cnr60anmruiwkiey0pukye` (`FN-8 admin`). API token is only in `D:\DEVELOPMENT\.infra.md`; do not copy it into repo docs. +- Application LXC: `192.168.1.105` (CTID 105, `new-vk-parser`). +- Public admin URL: `https://panel.f-n8.ru` (also `https://admin.f-n8.ru` when DNS is current). +- Coolify repository URL must be reachable from the deployment container; use `http://192.168.1.135:3000/exostring/new_vk_parser.git` or another resolvable Gitea URL, not `http://coolify.lan:3000/...`. +- Do not deploy this project through `sw.exostring.xyz`, `109.120.156.205`, `/opt/vk-parser`, or manual `systemctl` unless the user explicitly says the old VPS is being restored. +- Old `sw.exostring.xyz` / `109.120.156.205` notes are stale for this project. + +## Local Project + +- Local workspace: `D:\DEVELOPMENT\new_vk_parser` +- Main app package: `src/vk_parser_app` +- Admin routes: `src/vk_parser_app/admin.py` +- Templates: `src/vk_parser_app/templates` +- Workers: + - VK parser: `src/vk_parser_app/workers/parser.py` + - Telegram media/storage uploader: `src/vk_parser_app/workers/vk_storage_uploader.py` + - AI qualifier: `src/vk_parser_app/workers/ai_qualifier.py` + - AI writer: `src/vk_parser_app/workers/ai_writer.py` +- DB migrations: `db/migrations` +- Extended project documentation: `PROJECT.md` + +## Pipeline + +1. Sources are configured in admin. +2. VK parser reads enabled sources and stores raw posts. +3. Media/storage uploader sends raw post copies to Telegram storage and stores media metadata. +4. AI qualifier scores raw posts and marks accepted/rejected/maybe. +5. AI writer rewrites accepted posts into editor-ready drafts. +6. Editor page allows review, editing, category/source tag selection, media upload/removal, accept/reject. +7. Site poster automatically publishes accepted media-ready posts to Ghost. + - As of 2026-08-03, site publishing is provider-based: `site_poster_provider=ghost` + for FN-8, with `site_poster_base_url` and `site_poster_secret_file` copied from + legacy Ghost settings by migration `040_site_poster_provider_settings.sql`. + - New installs default `site-poster` to disabled and must explicitly set provider, + base URL, secret, and `site_poster_enabled=true`. + +## Important Text Rules + +- AI writer output text must be clean: no physical hashtags at the end. +- `raw_posts.final_text` must also stay clean. +- Category and source hashtags are separate fields: + - category tag: `final_category_tag` / `rewrite_category_tag` + - source tag: `final_source_tag` / `rewrite_source_tag` / `sources.tag` +- Publication preview may show hashtags by composing: + - clean text + - blank line + - `#category #source` +- The editor textarea must show only clean editable text. +- The list preview in `/editor` should show the publication preview, including composed hashtags. +- `build_publication_text()` in `src/vk_parser_app/text_utils.py` is display/composition helper only. Do not use it before writing `final_text` to DB. + +## Recent Hotfix: Hashtags + +On 2026-06-17 the hashtag flow was fixed: + +- `ai_writer.py` now stores clean `final_text=$2`, not text with hashtags. +- `admin.py` prepares: + - `review_text`: clean text for textarea + - `publication_preview_text`: composed display text with hashtags +- editor accept/save store clean text. +- existing DB rows with trailing `#category #source` in `final_text` were cleaned. +- server check after fix: `remaining_hashtag_tail: 0`. + +## Admin Pages + +- `/sources`: source management. +- `/raw`: raw post feed. +- `/editor`: editor queue. +- `/workers`: worker controls and settings. +- `/prompt-test`: AI writer prompt test page. +- `/users`: admin users. +- `/logs`: audit/log page if enabled. + +## Recent Admin UI State + +As of 2026-07-29: + +- Admin title comes from env `ADMIN_APP_TITLE`; fallback is `Редакторская`. The old hardcoded `Parser Admin` title and lightning SVG are gone. +- Browser/page title comes from env `ADMIN_SITE_TITLE`; fallback is `Редакторская`. +- Admin sidebar is collapsible and collapsed by default; collapsed state shows icons only. +- Theme toggle supports dark/light themes. Tailwind `app.*` colors use RGB-channel CSS variables (`rgb(var(--app-...)/)`), so do not revert them to hex variables or opacity utilities will break. Light-theme hover contrast is patched in base CSS. +- `/editor` desktop filter panel is collapsible and collapsed by default; the feed expands into the freed width. +- Mobile editor status tabs scroll horizontally instead of overflowing the viewport. `/sources` and `/raw` have mobile layout fixes to avoid page-level horizontal scroll. +- In editor cards, post meta/status/source info is above the post, not in a side column. +- `Оригинальный текст` is a spoiler directly above `Финальный текст`. +- `AI заметки / Резолюция` is a separate spoiler. It contains AI notes/resolution and an AI rewrite preview below it, so editors can compare their changes with the original AI rewrite. +- `Финальный текст` textarea must autosize to fully fit text without internal scroll. Current implementation uses Alpine `ResizeObserver`, trims trailing whitespace in the browser/template, and keeps `min-h-[150px]`. +- Media hiding is handled by the editor media X button: it adds hidden `delete_media_ids`; backend marks media `editor_hidden=TRUE`, so hidden media is not published. +- Reject action has no confirmation popup. +- Save/accept/return actions use targeted HTMX partial updates for status/actions where possible, so open spoilers and the post body should not collapse after clicking action buttons. +- `/editor/{post_id}/return` returns accepted/rejected posts to review. The return button should exist on both accepted and rejected tabs. + +## Recent AI Writer State + +- Current live writer model setting checked on 2026-07-29: `ai_writer_model = anthropic/claude-sonnet-4-6`. +- Sonnet 4.6 can return `rewrites` as a JSON string instead of a list. `validate_rewrites()` in `src/vk_parser_app/workers/ai_writer.py` now parses stringified `rewrites` before validation. +- 2026-07-30 incident: post `2348` was retried 983 times by AI writer because automatic claim included `rewrite_status='failed'`; estimated internal cost was `$16.465977`. Live setting `ai_writer_enabled` was switched to `false` manually to stop spend. Automatic AI qualifier/writer claims must not include `failed`; failed posts require an explicit manual reset to `pending`. AI writer/qualifier keep running after a failed batch, show the error on `/workers`, and send a Telegram alert via daily-report/TG-poster bot recipients. +- Sonnet 4.6 may return `category_id=18` (`Не целевой контент`) with empty `text` and the explanation in `notes`. This is valid AI rejection, not a writer error; `validate_rewrites()` allows short/empty text only for non-target category. +- AI qualifier has exactly one automatic retry: after the first failed qualification batch, affected posts are returned to `qualification_status='pending'`; after the second failed batch for the same post, they stay `failed` and are not claimed again automatically. + +## Prompt Architecture + +- Prompts are split into editable human part and technical contract part. +- User should be able to edit human prompt safely without breaking JSON schema. +- Contracts should contain JSON-only/output-schema requirements. +- Writer should return category ID, not free-form tag, where possible. +- Categories are managed in worker/admin settings and have stable IDs. + +## Security Notes + +- Secrets should not be committed. +- `.env` should contain only secrets and deploy-specific values. +- Runtime settings that are not secrets should live in DB/admin settings. +- Dangerous FN-8 defaults were removed from runtime defaults on 2026-08-03: + no default TG publication chat, no default VK storage group, no default Ghost URL/key path. +- Docker image now copies `db/migrations`, and `docker-entrypoint.sh` runs + `python /app/scripts/apply_migrations.py` before starting the app. +- Admin has login protection and server has firewall/fail2ban hardening from previous iterations. + +## Working Rules For Future Codex Runs + +- Before any deploy/server operation, read `D:\DEVELOPMENT\.infra.md` first. +- Before pushing or deploying, run `git pull --rebase gitea main`. +- For this project, push deployable changes to `gitea/main`; Coolify handles deployment. +- Prefer small targeted file reads with `rg` and narrow ranges. +- Avoid dumping long post texts from DB unless explicitly needed. +- For runtime checks, use the current Coolify/Gitea/LXC 105 path from `D:\DEVELOPMENT\.infra.md`. +- Do not `git reset --hard` or revert user/server hotfixes. diff --git a/db/migrations/001_initial.sql b/db/migrations/001_initial.sql new file mode 100644 index 0000000..8e02d1b --- /dev/null +++ b/db/migrations/001_initial.sql @@ -0,0 +1,194 @@ +CREATE TABLE IF NOT EXISTS admin_users ( + id BIGSERIAL PRIMARY KEY, + login TEXT NOT NULL UNIQUE, + password_hash TEXT NOT NULL, + role TEXT NOT NULL DEFAULT 'admin' CHECK (role IN ('admin','editor','viewer')), + is_active BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS admin_sessions ( + token_hash TEXT PRIMARY KEY, + user_id BIGINT NOT NULL REFERENCES admin_users(id) ON DELETE CASCADE, + csrf_token TEXT NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_admin_sessions_user_id ON admin_sessions(user_id); +CREATE INDEX IF NOT EXISTS idx_admin_sessions_expires_at ON admin_sessions(expires_at); + +CREATE TABLE IF NOT EXISTS sources ( + id BIGSERIAL PRIMARY KEY, + platform TEXT NOT NULL DEFAULT 'vk', + name TEXT NOT NULL, + url TEXT NOT NULL, + external_id TEXT, + external_owner_id BIGINT, + active BOOLEAN NOT NULL DEFAULT TRUE, + status TEXT NOT NULL DEFAULT 'new', + status_msg TEXT, + parse_from TIMESTAMPTZ, + last_checked_at TIMESTAMPTZ, + last_parsed_at TIMESTAMPTZ, + priority INTEGER NOT NULL DEFAULT 100, + tags TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[], + settings_json JSONB NOT NULL DEFAULT '{}'::jsonb, + created_by BIGINT REFERENCES admin_users(id) ON DELETE SET NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + archived_at TIMESTAMPTZ +); + +CREATE UNIQUE INDEX IF NOT EXISTS ux_sources_platform_url_active +ON sources(platform, lower(url)) +WHERE archived_at IS NULL; + +CREATE INDEX IF NOT EXISTS idx_sources_active_platform ON sources(active, platform, priority, id) +WHERE archived_at IS NULL; + +CREATE TABLE IF NOT EXISTS raw_posts ( + id BIGSERIAL PRIMARY KEY, + source_id BIGINT NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + platform TEXT NOT NULL, + external_post_id TEXT NOT NULL, + external_owner_id BIGINT, + original_url TEXT NOT NULL, + storage_owner_id BIGINT, + storage_post_id BIGINT, + storage_post_url TEXT, + raw_text TEXT NOT NULL DEFAULT '', + raw_json JSONB NOT NULL DEFAULT '{}'::jsonb, + text_hash TEXT NOT NULL, + content_hash TEXT NOT NULL, + posted_at TIMESTAMPTZ, + copied_at TIMESTAMPTZ, + status TEXT NOT NULL DEFAULT 'raw_saved', + skip_reason TEXT, + error_reason TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE(source_id, external_post_id) +); + +CREATE INDEX IF NOT EXISTS idx_raw_posts_status_created ON raw_posts(status, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_raw_posts_source_posted ON raw_posts(source_id, posted_at DESC); +CREATE INDEX IF NOT EXISTS idx_raw_posts_content_hash ON raw_posts(content_hash); + +CREATE TABLE IF NOT EXISTS raw_post_media ( + id BIGSERIAL PRIMARY KEY, + raw_post_id BIGINT NOT NULL REFERENCES raw_posts(id) ON DELETE CASCADE, + platform TEXT NOT NULL, + media_type TEXT NOT NULL CHECK (media_type IN ('photo','video','link','doc','unknown')), + original_url TEXT, + original_attachment_id TEXT, + storage_attachment_id TEXT, + storage_url TEXT, + preview_url TEXT, + width INTEGER, + height INTEGER, + duration_sec INTEGER, + sort_order INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'pending', + attempts INTEGER NOT NULL DEFAULT 0, + error TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_raw_post_media_post ON raw_post_media(raw_post_id, sort_order, id); +CREATE INDEX IF NOT EXISTS idx_raw_post_media_status ON raw_post_media(status); + +CREATE TABLE IF NOT EXISTS jobs ( + id BIGSERIAL PRIMARY KEY, + type TEXT NOT NULL, + entity_type TEXT NOT NULL, + entity_id BIGINT NOT NULL, + payload_json JSONB NOT NULL DEFAULT '{}'::jsonb, + status TEXT NOT NULL DEFAULT 'pending', + attempts INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 5, + next_run_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + locked_by TEXT, + locked_at TIMESTAMPTZ, + last_error TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE UNIQUE INDEX IF NOT EXISTS ux_jobs_open_entity_type +ON jobs(type, entity_type, entity_id) +WHERE status IN ('pending','retry','in_progress'); + +CREATE INDEX IF NOT EXISTS idx_jobs_claim ON jobs(type, status, next_run_at, id); + +CREATE TABLE IF NOT EXISTS worker_controls ( + name TEXT PRIMARY KEY, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + settings_json JSONB NOT NULL DEFAULT '{}'::jsonb, + updated_by BIGINT REFERENCES admin_users(id) ON DELETE SET NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS worker_heartbeats ( + name TEXT PRIMARY KEY, + heartbeat_at TIMESTAMPTZ NOT NULL, + status TEXT NOT NULL DEFAULT 'running', + current_job_id BIGINT, + meta_json JSONB NOT NULL DEFAULT '{}'::jsonb, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS app_settings ( + key TEXT PRIMARY KEY, + value_json JSONB NOT NULL, + value_type TEXT NOT NULL DEFAULT 'str', + title TEXT NOT NULL DEFAULT '', + description TEXT NOT NULL DEFAULT '', + category TEXT NOT NULL DEFAULT 'General', + updated_by BIGINT REFERENCES admin_users(id) ON DELETE SET NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS audit_log ( + id BIGSERIAL PRIMARY KEY, + actor_id BIGINT REFERENCES admin_users(id) ON DELETE SET NULL, + action TEXT NOT NULL, + entity_type TEXT NOT NULL, + entity_id BIGINT, + before_json JSONB, + after_json JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +INSERT INTO worker_controls(name, enabled, settings_json) +VALUES + ('vk-parser', TRUE, '{}'::jsonb), + ('vk-storage-uploader', TRUE, '{}'::jsonb) +ON CONFLICT (name) DO NOTHING; + +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('parser_interval_sec', '300'::jsonb, 'int', 'Parser interval, sec', 'Pause between parser cycles.', 'Parser'), + ('parser_new_source_lookback_days', '14'::jsonb, 'int', 'New source lookback, days', 'How far back to read a new source.', 'Parser'), + ('parser_reparse_overlap_minutes', '120'::jsonb, 'int', 'Reparse overlap, min', 'Overlap window for known sources.', 'Parser'), + ('parser_min_text_length', '50'::jsonb, 'int', 'Minimum text length', 'Posts with shorter text are treated as junk.', 'Parser'), + ('parser_skip_reposts', 'true'::jsonb, 'bool', 'Skip reposts', 'Do not save reposts from VK copy_history.', 'Parser'), + ('parser_skip_empty_text', 'true'::jsonb, 'bool', 'Skip empty text posts', 'Treat posts without text as junk.', 'Parser'), + ('parser_skip_no_media', 'true'::jsonb, 'bool', 'Skip posts without media', 'Treat posts without media as junk.', 'Parser'), + ('parser_skip_text_too_short', 'true'::jsonb, 'bool', 'Skip short text posts', 'Treat posts shorter than parser_min_text_length as junk.', 'Parser'), + ('parser_store_skipped_posts', 'false'::jsonb, 'bool', 'Store skipped posts', 'Keep junk posts in raw_posts instead of dropping them.', 'Parser'), + ('parser_dedupe_content_hash', 'true'::jsonb, 'bool', 'Dedupe by content hash', 'Skip posts whose text and media hash already exists in raw_posts.', 'Parser'), + ('parser_source_pause_sec', '0'::jsonb, 'float', 'Source pause, sec', 'Pause between VK sources inside one parser cycle.', 'Parser'), + ('vk_requests_per_second', '3'::jsonb, 'int', 'VK requests per second', 'Global VK API rate limit per worker.', 'VK'), + ('vk_wall_page_size', '50'::jsonb, 'int', 'VK wall page size', 'wall.get count per page.', 'VK'), + ('vk_api_timeout_total_sec', '15'::jsonb, 'int', 'VK total timeout, sec', 'Total timeout for one VK API request.', 'VK'), + ('vk_api_timeout_connect_sec', '5'::jsonb, 'int', 'VK connect timeout, sec', 'Connection timeout for VK API.', 'VK'), + ('vk_rate_limit_sleep_sec', '1'::jsonb, 'float', 'VK rate limit sleep, sec', 'Sleep after VK API rate limit error code 6.', 'VK'), + ('vk_api_retry_attempts', '3'::jsonb, 'int', 'VK retry attempts', 'Attempts for temporary VK API/network errors.', 'VK'), + ('vk_api_retry_min_delay_sec', '2'::jsonb, 'float', 'VK retry min delay, sec', 'Initial retry delay for temporary VK errors.', 'VK'), + ('vk_api_retry_max_delay_sec', '10'::jsonb, 'float', 'VK retry max delay, sec', 'Maximum retry delay for temporary VK errors.', 'VK'), + ('uploader_interval_sec', '5'::jsonb, 'int', 'Uploader idle interval, sec', 'Pause when no jobs are available.', 'Uploader'), + ('uploader_download_timeout_sec', '45'::jsonb, 'int', 'Download timeout, sec', 'Timeout for source media download.', 'Uploader') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/002_telegram_storage.sql b/db/migrations/002_telegram_storage.sql new file mode 100644 index 0000000..13d5fdb --- /dev/null +++ b/db/migrations/002_telegram_storage.sql @@ -0,0 +1,27 @@ +ALTER TABLE raw_posts + ADD COLUMN IF NOT EXISTS tg_storage_chat_id BIGINT, + ADD COLUMN IF NOT EXISTS tg_storage_thread_id BIGINT, + ADD COLUMN IF NOT EXISTS tg_storage_message_ids BIGINT[] NOT NULL DEFAULT ARRAY[]::BIGINT[], + ADD COLUMN IF NOT EXISTS tg_storage_meta_message_id BIGINT; + +ALTER TABLE raw_post_media + ADD COLUMN IF NOT EXISTS tg_file_id TEXT, + ADD COLUMN IF NOT EXISTS tg_file_unique_id TEXT; + +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('tg_media_channel_id', '""'::jsonb, 'str', 'Telegram media storage chat', 'Chat id or chat_id:topic_id for raw media storage.', 'Telegram'), + ('local_bot_api_url', '""'::jsonb, 'str', 'Local Telegram Bot API URL', 'Optional local Bot API server URL.', 'Telegram'), + ('telegram_media_group_max_items', '10'::jsonb, 'int', 'Telegram media group max items', 'Maximum media items per Telegram media group.', 'Telegram'), + ('telegram_media_upload_delay_sec', '1'::jsonb, 'float', 'Telegram media upload delay, sec', 'Pause between Telegram upload operations.', 'Telegram'), + ('telegram_retry_max_attempts', '4'::jsonb, 'int', 'Telegram retry attempts', 'Retry attempts for Telegram calls.', 'Telegram'), + ('telegram_retry_backoff_max_sec', '15'::jsonb, 'int', 'Telegram retry max backoff, sec', 'Maximum retry backoff for Telegram calls.', 'Telegram'), + ('telegram_caption_limit', '1024'::jsonb, 'int', 'Telegram caption limit', 'Caption character limit for media messages.', 'Telegram'), + ('telegram_message_limit', '4096'::jsonb, 'int', 'Telegram message limit', 'Text message character limit.', 'Telegram'), + ('telegram_text_overflow_marker', '"Продолжение следующим сообщением."'::jsonb, 'str', 'Telegram overflow marker', 'Caption marker when text does not fit.', 'Telegram'), + ('media_post_job_pause_sec', '0.2'::jsonb, 'float', 'Post job pause, sec', 'Pause after each processed media storage job.', 'Uploader'), + ('video_max_size_mb', '49'::jsonb, 'int', 'Video max size, MB', 'Maximum video size to upload to Telegram.', 'Uploader'), + ('video_max_duration_sec', '300'::jsonb, 'int', 'Video max duration, sec', 'Maximum video duration to upload to Telegram.', 'Uploader'), + ('uploader_yt_dlp_timeout_sec', '300'::jsonb, 'int', 'yt-dlp timeout, sec', 'Timeout for VK video downloads.', 'Uploader'), + ('uploader_max_media_attempts', '3'::jsonb, 'int', 'Media max attempts', 'Maximum attempts per media item.', 'Uploader') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/003_parser_hardening_settings.sql b/db/migrations/003_parser_hardening_settings.sql new file mode 100644 index 0000000..c407a91 --- /dev/null +++ b/db/migrations/003_parser_hardening_settings.sql @@ -0,0 +1,16 @@ +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('parser_skip_reposts', 'true'::jsonb, 'bool', 'Skip reposts', 'Do not save reposts from VK copy_history.', 'Parser'), + ('parser_skip_empty_text', 'true'::jsonb, 'bool', 'Skip empty text posts', 'Treat posts without text as junk.', 'Parser'), + ('parser_skip_no_media', 'true'::jsonb, 'bool', 'Skip posts without media', 'Treat posts without media as junk.', 'Parser'), + ('parser_skip_text_too_short', 'true'::jsonb, 'bool', 'Skip short text posts', 'Treat posts shorter than parser_min_text_length as junk.', 'Parser'), + ('parser_store_skipped_posts', 'false'::jsonb, 'bool', 'Store skipped posts', 'Keep junk posts in raw_posts instead of dropping them.', 'Parser'), + ('parser_dedupe_content_hash', 'true'::jsonb, 'bool', 'Dedupe by content hash', 'Skip posts whose text and media hash already exists in raw_posts.', 'Parser'), + ('parser_source_pause_sec', '0'::jsonb, 'float', 'Source pause, sec', 'Pause between VK sources inside one parser cycle.', 'Parser'), + ('vk_api_timeout_total_sec', '15'::jsonb, 'int', 'VK total timeout, sec', 'Total timeout for one VK API request.', 'VK'), + ('vk_api_timeout_connect_sec', '5'::jsonb, 'int', 'VK connect timeout, sec', 'Connection timeout for VK API.', 'VK'), + ('vk_rate_limit_sleep_sec', '1'::jsonb, 'float', 'VK rate limit sleep, sec', 'Sleep after VK API rate limit error code 6.', 'VK'), + ('vk_api_retry_attempts', '3'::jsonb, 'int', 'VK retry attempts', 'Attempts for temporary VK API/network errors.', 'VK'), + ('vk_api_retry_min_delay_sec', '2'::jsonb, 'float', 'VK retry min delay, sec', 'Initial retry delay for temporary VK errors.', 'VK'), + ('vk_api_retry_max_delay_sec', '10'::jsonb, 'float', 'VK retry max delay, sec', 'Maximum retry delay for temporary VK errors.', 'VK') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/004_junk_post_policy.sql b/db/migrations/004_junk_post_policy.sql new file mode 100644 index 0000000..792a3a8 --- /dev/null +++ b/db/migrations/004_junk_post_policy.sql @@ -0,0 +1,28 @@ +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('parser_skip_empty_text', 'true'::jsonb, 'bool', 'Skip empty text posts', 'Treat posts without text as junk.', 'Parser') +ON CONFLICT (key) DO NOTHING; + +UPDATE app_settings +SET value_json='50'::jsonb, + description='Posts with shorter text are treated as junk.', + updated_at=NOW() +WHERE key='parser_min_text_length' + AND value_json='0'::jsonb; + +UPDATE app_settings +SET value_json='false'::jsonb, + description='Keep junk posts in raw_posts instead of dropping them.', + updated_at=NOW() +WHERE key='parser_store_skipped_posts' + AND value_json='true'::jsonb; + +UPDATE app_settings +SET description='Treat posts without media as junk.', + updated_at=NOW() +WHERE key='parser_skip_no_media'; + +UPDATE app_settings +SET description='Treat posts shorter than parser_min_text_length as junk.', + updated_at=NOW() +WHERE key='parser_skip_text_too_short'; diff --git a/db/migrations/005_ai_qualifier.sql b/db/migrations/005_ai_qualifier.sql new file mode 100644 index 0000000..2acab71 --- /dev/null +++ b/db/migrations/005_ai_qualifier.sql @@ -0,0 +1,50 @@ +ALTER TABLE raw_posts + ADD COLUMN IF NOT EXISTS qualification_status TEXT, + ADD COLUMN IF NOT EXISTS qualification_score INTEGER, + ADD COLUMN IF NOT EXISTS qualification_decision TEXT, + ADD COLUMN IF NOT EXISTS qualification_reason TEXT, + ADD COLUMN IF NOT EXISTS qualification_model TEXT, + ADD COLUMN IF NOT EXISTS qualification_prompt_hash TEXT, + ADD COLUMN IF NOT EXISTS qualification_batch_id BIGINT, + ADD COLUMN IF NOT EXISTS qualified_at TIMESTAMPTZ; + +CREATE INDEX IF NOT EXISTS idx_raw_posts_qualification_claim +ON raw_posts(status, qualification_status, created_at) +WHERE status='storage_ready'; + +CREATE TABLE IF NOT EXISTS ai_qualification_batches ( + id BIGSERIAL PRIMARY KEY, + status TEXT NOT NULL DEFAULT 'processing', + provider TEXT NOT NULL, + model TEXT NOT NULL, + prompt_hash TEXT NOT NULL, + posts_count INTEGER NOT NULL DEFAULT 0, + accepted_count INTEGER NOT NULL DEFAULT 0, + rejected_count INTEGER NOT NULL DEFAULT 0, + maybe_count INTEGER NOT NULL DEFAULT 0, + request_json JSONB NOT NULL DEFAULT '{}'::jsonb, + response_json JSONB, + error TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + completed_at TIMESTAMPTZ +); + +INSERT INTO worker_controls(name, enabled, settings_json) +VALUES ('ai-qualifier', FALSE, '{}'::jsonb) +ON CONFLICT (name) DO NOTHING; + +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('ai_qualifier_enabled', 'false'::jsonb, 'bool', 'AI qualifier enabled', 'Enable qualification requests to AI API.', 'AI Qualifier'), + ('ai_qualifier_provider', '"openrouter"'::jsonb, 'str', 'Provider', 'LiteLLM provider: openrouter, openai, anthropic, gemini, openai_compatible.', 'AI Qualifier'), + ('ai_qualifier_model', '"openrouter/openai/gpt-4.1-mini"'::jsonb, 'str', 'Model', 'Model name in LiteLLM format.', 'AI Qualifier'), + ('ai_qualifier_api_key', '""'::jsonb, 'secret', 'API key', 'AI provider API key.', 'AI Qualifier'), + ('ai_qualifier_api_base', '""'::jsonb, 'str', 'API base', 'Optional base URL for OpenAI-compatible providers.', 'AI Qualifier'), + ('ai_qualifier_batch_size', '30'::jsonb, 'int', 'Batch size', 'How many raw posts to classify in one request.', 'AI Qualifier'), + ('ai_qualifier_min_score', '7'::jsonb, 'int', 'Minimum accepted score', 'Posts with score >= this value are accepted.', 'AI Qualifier'), + ('ai_qualifier_interval_sec', '60'::jsonb, 'int', 'Idle interval, sec', 'Pause between qualifier checks.', 'AI Qualifier'), + ('ai_qualifier_max_text_chars', '2000'::jsonb, 'int', 'Max text chars per post', 'Trim raw text before sending it to AI.', 'AI Qualifier'), + ('ai_qualifier_temperature', '0'::jsonb, 'float', 'Temperature', 'AI sampling temperature for stable classification.', 'AI Qualifier'), + ('ai_qualifier_timeout_sec', '120'::jsonb, 'int', 'Timeout, sec', 'AI request timeout.', 'AI Qualifier'), + ('ai_qualifier_prompt', '"Ты квалификатор постов для проекта, который собирает посты производителей и магазинов милитари, тактической и airsoft-снаряги.\\n\\nОцени каждый raw-пост по пригодности для дальнейшего рерайта и публикации.\\n\\nПодходит: товарные посты, новинки, обзоры, производство, экипировка, тактическая одежда, броня, подсумки, рюкзаки, разгрузки, ремни, патчи, аксессуары, airsoft/страйкбольная снаряга, полезные анонсы производителя.\\n\\nМусор: мемы, поздравления, вакансии, политика, бытовуха, пустые посты, посты без конкретного товара/снаряжения, розыгрыши без предметного контента, чистая реклама скидки без понятного товара, нерелевантные новости.\\n\\nВерни только JSON-объект по схеме: {\\\"results\\\":[{\\\"id\\\":123,\\\"score\\\":8,\\\"decision\\\":\\\"accepted\\\",\\\"reason\\\":\\\"короткая причина\\\"}]}. score — целое число от 1 до 10. decision: accepted, rejected или maybe. Не добавляй markdown и пояснения вне JSON."'::jsonb, 'text', 'Prompt', 'Main qualification prompt.', 'AI Qualifier') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/006_russian_settings_and_ai_catalog.sql b/db/migrations/006_russian_settings_and_ai_catalog.sql new file mode 100644 index 0000000..21cdc73 --- /dev/null +++ b/db/migrations/006_russian_settings_and_ai_catalog.sql @@ -0,0 +1,47 @@ +UPDATE app_settings SET title='Интервал парсера, сек', description='Пауза между полными циклами парсинга источников.' WHERE key='parser_interval_sec'; +UPDATE app_settings SET title='Глубина нового источника, дней', description='Сколько дней назад читать посты при первом запуске источника.' WHERE key='parser_new_source_lookback_days'; +UPDATE app_settings SET title='Перекрытие повторного чтения, мин', description='Насколько назад перечитывать источник, чтобы не пропустить посты.' WHERE key='parser_reparse_overlap_minutes'; +UPDATE app_settings SET title='Минимальная длина текста', description='Посты с текстом короче этого значения считаются мусором.' WHERE key='parser_min_text_length'; +UPDATE app_settings SET title='Пропускать репосты', description='Не брать VK-посты с copy_history.' WHERE key='parser_skip_reposts'; +UPDATE app_settings SET title='Пропускать пустой текст', description='Не брать посты без текста.' WHERE key='parser_skip_empty_text'; +UPDATE app_settings SET title='Пропускать без медиа', description='Не брать посты без фото/видео.' WHERE key='parser_skip_no_media'; +UPDATE app_settings SET title='Пропускать короткий текст', description='Не брать посты короче минимальной длины текста.' WHERE key='parser_skip_text_too_short'; +UPDATE app_settings SET title='Сохранять мусорные посты', description='Если включено, мусорные посты попадут в raw_posts со статусом skipped. Обычно выключено.' WHERE key='parser_store_skipped_posts'; +UPDATE app_settings SET title='Дедупликация по контенту', description='Не брать пост, если такой текст и набор медиа уже есть в raw_posts.' WHERE key='parser_dedupe_content_hash'; +UPDATE app_settings SET title='Пауза между источниками, сек', description='Задержка между чтением разных VK-источников внутри одного цикла.' WHERE key='parser_source_pause_sec'; + +UPDATE app_settings SET title='VK запросов в секунду', description='Ограничение частоты запросов к VK API.' WHERE key='vk_requests_per_second'; +UPDATE app_settings SET title='Размер страницы VK wall.get', description='Сколько постов запрашивать за один вызов wall.get.' WHERE key='vk_wall_page_size'; +UPDATE app_settings SET title='VK общий таймаут, сек', description='Общий таймаут одного запроса к VK API.' WHERE key='vk_api_timeout_total_sec'; +UPDATE app_settings SET title='VK таймаут соединения, сек', description='Таймаут установки соединения с VK API.' WHERE key='vk_api_timeout_connect_sec'; +UPDATE app_settings SET title='Пауза при VK rate limit, сек', description='Сколько ждать после ошибки VK API error_code=6.' WHERE key='vk_rate_limit_sleep_sec'; +UPDATE app_settings SET title='VK retry попытки', description='Сколько раз повторять временно упавший VK-запрос.' WHERE key='vk_api_retry_attempts'; +UPDATE app_settings SET title='VK retry минимум, сек', description='Начальная задержка между повторными VK-запросами.' WHERE key='vk_api_retry_min_delay_sec'; +UPDATE app_settings SET title='VK retry максимум, сек', description='Максимальная задержка между повторными VK-запросами.' WHERE key='vk_api_retry_max_delay_sec'; + +UPDATE app_settings SET title='Интервал аплоадера, сек', description='Пауза, когда нет задач на загрузку raw-постов в Telegram.' WHERE key='uploader_interval_sec'; +UPDATE app_settings SET title='Таймаут скачивания, сек', description='Таймаут скачивания исходного медиа.' WHERE key='uploader_download_timeout_sec'; +UPDATE app_settings SET title='Telegram канал raw', description='ID Telegram-канала/топика для хранения raw-постов.' WHERE key='tg_media_channel_id'; +UPDATE app_settings SET title='Локальный Bot API URL', description='Опциональный URL локального Telegram Bot API сервера.' WHERE key='local_bot_api_url'; +UPDATE app_settings SET title='Задержка на медиа, сек', description='Пауза после отправки медиа, умножается на количество файлов.' WHERE key='media_upload_delay_sec'; +UPDATE app_settings SET title='Пауза после поста, сек', description='Короткая пауза после полной обработки одного raw-поста.' WHERE key='media_post_job_pause_sec'; +UPDATE app_settings SET title='Максимум медиа в группе', description='Максимум файлов в одном Telegram media group.' WHERE key='media_group_max_items'; +UPDATE app_settings SET title='Максимум попыток медиа', description='Сколько раз пытаться скачать/загрузить отдельное медиа.' WHERE key='max_media_attempts'; +UPDATE app_settings SET title='Telegram retry попытки', description='Сколько раз повторять временно упавший Telegram-запрос.' WHERE key='tg_retry_attempts'; +UPDATE app_settings SET title='Telegram backoff максимум, сек', description='Максимальная задержка между retry Telegram-запросов.' WHERE key='tg_retry_backoff_max_sec'; +UPDATE app_settings SET title='Максимум видео, МБ', description='Максимальный размер видео для загрузки в Telegram.' WHERE key='video_max_size_mb'; +UPDATE app_settings SET title='Максимум видео, сек', description='Видео длиннее этого значения помечается как link_only.' WHERE key='video_max_duration_sec'; +UPDATE app_settings SET title='yt-dlp таймаут, сек', description='Сколько ждать скачивания VK-видео через yt-dlp.' WHERE key='uploader_yt_dlp_timeout_sec'; + +UPDATE app_settings SET title='Включить AI-квалификатор', description='Разрешить воркеру отправлять raw-посты в AI API.' WHERE key='ai_qualifier_enabled'; +UPDATE app_settings SET title='Провайдер LLM', description='Провайдер для LiteLLM. Для OpenRouter список моделей подгружается live.' WHERE key='ai_qualifier_provider'; +UPDATE app_settings SET title='Модель LLM', description='Модель для квалификации. Можно выбрать из списка или вписать вручную.' WHERE key='ai_qualifier_model'; +UPDATE app_settings SET title='API токен', description='Токен выбранного LLM-провайдера. Сохраняется в настройках.' WHERE key='ai_qualifier_api_key'; +UPDATE app_settings SET title='API base URL', description='Опциональный base URL для OpenAI-compatible провайдера.' WHERE key='ai_qualifier_api_base'; +UPDATE app_settings SET title='Размер пачки', description='Сколько raw-постов отправлять в один AI-запрос.' WHERE key='ai_qualifier_batch_size'; +UPDATE app_settings SET title='Минимальная оценка', description='Пост принимается, если score больше или равен этому значению.' WHERE key='ai_qualifier_min_score'; +UPDATE app_settings SET title='Интервал проверки, сек', description='Пауза между проверками новых raw-постов.' WHERE key='ai_qualifier_interval_sec'; +UPDATE app_settings SET title='Максимум символов текста', description='Сколько символов raw_text отправлять в AI на один пост.' WHERE key='ai_qualifier_max_text_chars'; +UPDATE app_settings SET title='Температура', description='Температура генерации. Для классификации обычно 0.' WHERE key='ai_qualifier_temperature'; +UPDATE app_settings SET title='AI таймаут, сек', description='Таймаут одного AI-запроса.' WHERE key='ai_qualifier_timeout_sec'; +UPDATE app_settings SET title='Промпт квалификатора', description='Основная инструкция для оценки пригодности raw-постов.' WHERE key='ai_qualifier_prompt'; diff --git a/db/migrations/007_ai_writer.sql b/db/migrations/007_ai_writer.sql new file mode 100644 index 0000000..0c2053d --- /dev/null +++ b/db/migrations/007_ai_writer.sql @@ -0,0 +1,57 @@ +ALTER TABLE ai_qualification_batches + ADD COLUMN IF NOT EXISTS prompt_tokens INTEGER, + ADD COLUMN IF NOT EXISTS completion_tokens INTEGER, + ADD COLUMN IF NOT EXISTS total_tokens INTEGER, + ADD COLUMN IF NOT EXISTS estimated_cost_usd NUMERIC(12, 6); + +ALTER TABLE raw_posts + ADD COLUMN IF NOT EXISTS rewrite_status TEXT, + ADD COLUMN IF NOT EXISTS rewritten_text TEXT, + ADD COLUMN IF NOT EXISTS rewrite_notes TEXT, + ADD COLUMN IF NOT EXISTS rewrite_model TEXT, + ADD COLUMN IF NOT EXISTS rewrite_prompt_hash TEXT, + ADD COLUMN IF NOT EXISTS rewrite_batch_id BIGINT, + ADD COLUMN IF NOT EXISTS rewritten_at TIMESTAMPTZ; + +CREATE INDEX IF NOT EXISTS idx_raw_posts_writer_claim +ON raw_posts(qualification_status, rewrite_status, qualified_at, id) +WHERE qualification_status='accepted'; + +CREATE TABLE IF NOT EXISTS ai_writer_batches ( + id BIGSERIAL PRIMARY KEY, + status TEXT NOT NULL DEFAULT 'processing', + provider TEXT NOT NULL, + model TEXT NOT NULL, + prompt_hash TEXT NOT NULL, + posts_count INTEGER NOT NULL DEFAULT 0, + ready_count INTEGER NOT NULL DEFAULT 0, + failed_count INTEGER NOT NULL DEFAULT 0, + request_json JSONB NOT NULL DEFAULT '{}'::jsonb, + response_json JSONB, + error TEXT, + prompt_tokens INTEGER, + completion_tokens INTEGER, + total_tokens INTEGER, + estimated_cost_usd NUMERIC(12, 6), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + completed_at TIMESTAMPTZ +); + +INSERT INTO worker_controls(name, enabled, settings_json) +VALUES ('ai-writer', FALSE, '{}'::jsonb) +ON CONFLICT (name) DO NOTHING; + +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('ai_writer_enabled', 'false'::jsonb, 'bool', 'AI-райтер включён', 'Разрешает отправлять прошедшие квалификацию посты на рерайт.', 'AI Writer'), + ('ai_writer_provider', '"anthropic"'::jsonb, 'str', 'Провайдер LLM', 'Через какого провайдера LiteLLM отправлять запросы райтера.', 'AI Writer'), + ('ai_writer_model', '"claude-haiku-4-5-20251001"'::jsonb, 'str', 'Модель', 'Модель для рерайта. Для отладки используем дешёвую Haiku.', 'AI Writer'), + ('ai_writer_api_key', '""'::jsonb, 'secret', 'API-ключ', 'Ключ провайдера для AI-райтера. Можно использовать тот же Anthropic-ключ, что и у квалификатора.', 'AI Writer'), + ('ai_writer_api_base', '""'::jsonb, 'str', 'API base URL', 'Необязательный base URL для OpenAI-compatible провайдеров.', 'AI Writer'), + ('ai_writer_prompt', '"Ты редактор Telegram/VK-канала про милитари, тактическую и airsoft-снарягу. Перепиши исходный пост производителя в готовый к публикации пост на русском языке.\\n\\nСтиль: живой, понятный, без канцелярита, без чрезмерной рекламы, без выдуманных фактов. Сохраняй конкретику: модель, материал, цвета, размеры, назначение, цену и условия, если они есть в исходнике. Не добавляй то, чего нет в исходном посте.\\n\\nФормат результата: 1 короткий заголовок первой строкой, затем 2-5 абзацев или аккуратный список, если исходник товарный. В конце не добавляй ссылки и хэштеги.\\n\\nВерни только JSON-объект по схеме: {\\\"rewrites\\\":[{\\\"id\\\":123,\\\"text\\\":\\\"готовый текст\\\",\\\"notes\\\":\\\"короткая заметка для редактора\\\"}]}. Без markdown вне JSON."'::jsonb, 'text', 'Промпт райтера', 'Главный промпт рерайта. Поддерживает обычные переносы строк.', 'AI Writer'), + ('ai_writer_batch_size', '1'::jsonb, 'int', 'Размер batch', 'Сколько постов переписывать за один запрос. Для отладки лучше 1.', 'AI Writer'), + ('ai_writer_max_text_chars', '3500'::jsonb, 'int', 'Максимум символов исходника', 'Обрезает исходный raw-текст перед отправкой в LLM.', 'AI Writer'), + ('ai_writer_temperature', '0.4'::jsonb, 'float', 'Температура генерации', 'Ниже — стабильнее и суше, выше — свободнее и разнообразнее.', 'AI Writer'), + ('ai_writer_timeout_sec', '180'::jsonb, 'int', 'Таймаут, сек', 'Сколько ждать ответ модели.', 'AI Writer'), + ('ai_writer_interval_sec', '60'::jsonb, 'int', 'Интервал ожидания, сек', 'Пауза между проверками очереди writer, когда работы нет.', 'AI Writer') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/008_ai_prompt_text.sql b/db/migrations/008_ai_prompt_text.sql new file mode 100644 index 0000000..6d247ba --- /dev/null +++ b/db/migrations/008_ai_prompt_text.sql @@ -0,0 +1,5 @@ +ALTER TABLE ai_qualification_batches + ADD COLUMN IF NOT EXISTS prompt_text TEXT; + +ALTER TABLE ai_writer_batches + ADD COLUMN IF NOT EXISTS prompt_text TEXT; diff --git a/db/migrations/009_qualification_reject_tag.sql b/db/migrations/009_qualification_reject_tag.sql new file mode 100644 index 0000000..80baf63 --- /dev/null +++ b/db/migrations/009_qualification_reject_tag.sql @@ -0,0 +1,6 @@ +ALTER TABLE raw_posts + ADD COLUMN IF NOT EXISTS qualification_reject_tag TEXT; + +CREATE INDEX IF NOT EXISTS idx_raw_posts_qualification_reject_tag +ON raw_posts(qualification_reject_tag) +WHERE qualification_reject_tag IS NOT NULL; diff --git a/db/migrations/010_source_tags_and_rewrite_category.sql b/db/migrations/010_source_tags_and_rewrite_category.sql new file mode 100644 index 0000000..e9b7490 --- /dev/null +++ b/db/migrations/010_source_tags_and_rewrite_category.sql @@ -0,0 +1,53 @@ +ALTER TABLE sources + ADD COLUMN IF NOT EXISTS tag TEXT; + +UPDATE sources +SET tag = lower(regexp_replace(COALESCE(NULLIF(external_id, ''), name), '[^[:alnum:]_]+', '_', 'g')) +WHERE tag IS NULL OR btrim(tag) = ''; + +ALTER TABLE raw_posts + ADD COLUMN IF NOT EXISTS rewrite_category TEXT, + ADD COLUMN IF NOT EXISTS rewrite_category_tag TEXT, + ADD COLUMN IF NOT EXISTS rewrite_source_tag TEXT; + +CREATE INDEX IF NOT EXISTS idx_sources_tag ON sources(lower(tag)) +WHERE archived_at IS NULL; + +CREATE INDEX IF NOT EXISTS idx_raw_posts_rewrite_category ON raw_posts(rewrite_category) +WHERE rewrite_category IS NOT NULL; + +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('ai_writer_categories', '["защита","одежда","разгрузка","рюкзаки","airsoft","патчи","электроника","аксессуары","производство"]'::jsonb, 'text', 'Категории райтера', 'Список категорий, из которых AI-райтер выбирает одну. Можно писать через запятую или с новой строки.', 'AI Writer') +ON CONFLICT (key) DO NOTHING; + +UPDATE app_settings +SET value_json = to_jsonb($prompt$You are an editor of a Russian Telegram channel about military, tactical and airsoft gear. +Write on behalf of the channel — as an expert observer, not the manufacturer. +Source posts come from manufacturers/shops; your job is to reframe them as channel news. + +VOICE: concise, expert, matter-of-fact. Like telling a knowledgeable friend what dropped. +Example opener framing: "У [Producer] вышел...", "Wartech показали...", "[Producer] добавили в линейку..." + +RULES: +- Use only facts from source text. Never invent specs, properties or claims. +- If source is thin — write a short post, do not pad it. +- No hype, no exclamation spam, no manufacturer PR tone. +- Post length: 3-6 lines. Short is better than padded. +- No bullet lists unless source has 4+ distinct specs worth listing. +- Do not add hashtags. The system will add category and producer hashtags separately. + +CATEGORIES: pick exactly one from the categories field in input. + +INPUT fields used: id, producer_name, producer_tag, text. +qualification_score and media_count are metadata; ignore them for rewrite unless they help understand context. + +OUTPUT: JSON only, no markdown outside JSON.$prompt$::text), + description = 'Главный промпт рерайта. Модель выбирает category, но не пишет хэштеги.', + updated_at = NOW() +WHERE key = 'ai_writer_prompt' + AND ( + value_json::text ILIKE '%хэштег%' + OR value_json::text ILIKE '%hashtag%' + OR value_json::text ILIKE '%#CATEGORY%' + ); diff --git a/db/migrations/011_safer_ai_settings.sql b/db/migrations/011_safer_ai_settings.sql new file mode 100644 index 0000000..22150b2 --- /dev/null +++ b/db/migrations/011_safer_ai_settings.sql @@ -0,0 +1,39 @@ +UPDATE app_settings +SET title='Инструкция квалификатора', + description='Редактируемая человеческая инструкция: критерии оценки и мусора. JSON-контракт ответа добавляется кодом автоматически.' +WHERE key='ai_qualifier_prompt'; + +UPDATE app_settings +SET title='Инструкция райтера', + description='Редактируемая человеческая инструкция: стиль, факты, длина и голос канала. Категорию и JSON-контракт ответа контролирует код.' +WHERE key='ai_writer_prompt'; + +UPDATE app_settings +SET description='Модель для квалификации. Список берётся live из API провайдера, если доступен ключ; fallback помечается отдельно.' +WHERE key='ai_qualifier_model'; + +UPDATE app_settings +SET description='Модель для рерайта. Список берётся live из API провайдера, если доступен ключ; fallback помечается отдельно.' +WHERE key='ai_writer_model'; + +UPDATE app_settings +SET description='Токен выбранного LLM-провайдера. В админке показывается полностью, поэтому доступ к настройкам должен быть только у доверенных пользователей.' +WHERE key IN ('ai_qualifier_api_key', 'ai_writer_api_key'); + +UPDATE app_settings m +SET value_json = to_jsonb('anthropic/' || trim(both '"' from m.value_json::text)) +FROM app_settings p +WHERE p.key = replace(m.key, '_model', '_provider') + AND p.value_json = '"anthropic"'::jsonb + AND m.key IN ('ai_qualifier_model', 'ai_writer_model') + AND m.value_json::text NOT LIKE '"anthropic/%' + AND m.value_json::text NOT LIKE '"%/%"'; + +UPDATE app_settings m +SET value_json = to_jsonb('gemini/' || trim(both '"' from m.value_json::text)) +FROM app_settings p +WHERE p.key = replace(m.key, '_model', '_provider') + AND p.value_json = '"gemini"'::jsonb + AND m.key IN ('ai_qualifier_model', 'ai_writer_model') + AND m.value_json::text NOT LIKE '"gemini/%' + AND m.value_json::text NOT LIKE '"%/%"'; diff --git a/db/migrations/012_qualification_model_decision.sql b/db/migrations/012_qualification_model_decision.sql new file mode 100644 index 0000000..8346bb6 --- /dev/null +++ b/db/migrations/012_qualification_model_decision.sql @@ -0,0 +1,15 @@ +ALTER TABLE raw_posts + ADD COLUMN IF NOT EXISTS qualification_model_decision TEXT; + +UPDATE raw_posts rp +SET qualification_model_decision = item->>'decision' +FROM ai_qualification_batches b, + jsonb_array_elements(COALESCE(b.response_json->'results', '[]'::jsonb)) AS item +WHERE rp.qualification_batch_id = b.id + AND (item->>'id')::bigint = rp.id + AND rp.qualification_model_decision IS NULL + AND item ? 'decision'; + +CREATE INDEX IF NOT EXISTS idx_raw_posts_qualification_model_decision +ON raw_posts(qualification_model_decision) +WHERE qualification_model_decision IS NOT NULL; diff --git a/db/migrations/013_editorial_review.sql b/db/migrations/013_editorial_review.sql new file mode 100644 index 0000000..fb4fd45 --- /dev/null +++ b/db/migrations/013_editorial_review.sql @@ -0,0 +1,32 @@ +ALTER TABLE raw_posts + ADD COLUMN IF NOT EXISTS editorial_status TEXT, + ADD COLUMN IF NOT EXISTS final_text TEXT, + ADD COLUMN IF NOT EXISTS final_category TEXT, + ADD COLUMN IF NOT EXISTS final_category_tag TEXT, + ADD COLUMN IF NOT EXISTS final_source_tag TEXT, + ADD COLUMN IF NOT EXISTS editor_notes TEXT, + ADD COLUMN IF NOT EXISTS regeneration_prompt TEXT, + ADD COLUMN IF NOT EXISTS reviewed_by BIGINT REFERENCES admin_users(id) ON DELETE SET NULL, + ADD COLUMN IF NOT EXISTS reviewed_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS edited_at TIMESTAMPTZ; + +UPDATE raw_posts +SET editorial_status = 'review' +WHERE editorial_status IS NULL + AND rewrite_status = 'ready'; + +UPDATE raw_posts +SET final_text = rewritten_text, + final_category = rewrite_category, + final_category_tag = rewrite_category_tag, + final_source_tag = rewrite_source_tag +WHERE rewrite_status = 'ready' + AND final_text IS NULL; + +CREATE INDEX IF NOT EXISTS idx_raw_posts_editorial_status +ON raw_posts(editorial_status, rewritten_at DESC, id) +WHERE rewrite_status = 'ready'; + +CREATE INDEX IF NOT EXISTS idx_raw_posts_final_category +ON raw_posts(final_category) +WHERE final_category IS NOT NULL; diff --git a/db/migrations/014_prompt_contracts_and_admin_security.sql b/db/migrations/014_prompt_contracts_and_admin_security.sql new file mode 100644 index 0000000..f537cac --- /dev/null +++ b/db/migrations/014_prompt_contracts_and_admin_security.sql @@ -0,0 +1,43 @@ +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ( + 'ai_qualifier_contract', + '"OUTPUT SCHEMA (return array matching input order):\n{\"results\":[{\"id\":123,\"score\":8,\"decision\":\"accepted\",\"reason\":\"до 10 слов на русском\",\"reject_tag\":null}]}\n\nRules:\n- Input is a JSON array of posts.\n- Return JSON only. No markdown. No text outside JSON.\n- score must be integer 1..10.\n- decision must be exactly one of: \"accepted\", \"rejected\", \"maybe\".\n- reject_tag (rejected/maybe only) must be one of: \"meme\", \"no_product\", \"politics\", \"discount_only\", \"off_topic\", \"vacancy\", \"low_content\", \"wrong_language\", \"injection\", \"weapon\", null.\n- Return one result for every input post id."'::jsonb, + 'text', + 'Технический контракт квалификатора', + 'JSON-схема и строгие правила ответа. Обычно не меняется при правке смыслового промпта.', + 'AI Qualifier' + ), + ( + 'ai_writer_contract', + '"OUTPUT SCHEMA (return array matching input order):\n{\"rewrites\":[{\"id\":123,\"category\":\"разгрузка\",\"text\":\"готовый текст без хэштегов\",\"notes\":\"короткая заметка для редактора или null\"}]}\n\nRules:\n- Input is a JSON object with key \"posts\" containing accepted posts.\n- Each post includes producer_name and producer_tag. Use producer_name when it helps, but do not invent facts.\n- If post includes editor_regeneration_prompt, follow it as an additional editor instruction while still obeying facts and output schema.\n- Do not add hashtags to rewritten text. Pick exactly one category from the categories list in input.\n- Return JSON only. No markdown. No text outside JSON.\n- Return one rewrite for every input post id."'::jsonb, + 'text', + 'Технический контракт райтера', + 'JSON-схема, выбор категории и запрет хэштегов в тексте. Обычно не меняется при правке стиля.', + 'AI Writer' + ) +ON CONFLICT (key) DO NOTHING; + +UPDATE app_settings +SET title='Свободная инструкция квалификатора', + description='Роль, тон, критерии и примеры оценки. Технический JSON-контракт вынесен отдельно.' +WHERE key='ai_qualifier_prompt'; + +UPDATE app_settings +SET title='Свободная инструкция райтера', + description='Стиль, голос канала, правила фактов и примеры текста. Технический JSON-контракт вынесен отдельно.' +WHERE key='ai_writer_prompt'; + +CREATE TABLE IF NOT EXISTS admin_login_attempts ( + id BIGSERIAL PRIMARY KEY, + ip TEXT NOT NULL, + login TEXT NOT NULL DEFAULT '', + success BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_admin_login_attempts_ip_time +ON admin_login_attempts(ip, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_admin_login_attempts_login_time +ON admin_login_attempts(login, created_at DESC); diff --git a/db/migrations/015_editor_media_flags.sql b/db/migrations/015_editor_media_flags.sql new file mode 100644 index 0000000..39654e8 --- /dev/null +++ b/db/migrations/015_editor_media_flags.sql @@ -0,0 +1,9 @@ +ALTER TABLE raw_post_media + ADD COLUMN IF NOT EXISTS editor_hidden BOOLEAN NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS editor_added BOOLEAN NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS editor_uploaded_by BIGINT REFERENCES admin_users(id) ON DELETE SET NULL, + ADD COLUMN IF NOT EXISTS editor_uploaded_at TIMESTAMPTZ; + +CREATE INDEX IF NOT EXISTS idx_raw_post_media_editor_visible +ON raw_post_media(raw_post_id, sort_order, id) +WHERE editor_hidden = FALSE; diff --git a/db/migrations/016_remove_regeneration_and_writer_prompt.sql b/db/migrations/016_remove_regeneration_and_writer_prompt.sql new file mode 100644 index 0000000..b22a050 --- /dev/null +++ b/db/migrations/016_remove_regeneration_and_writer_prompt.sql @@ -0,0 +1,54 @@ +UPDATE raw_posts +SET rewrite_status='ready', + editorial_status='review', + regeneration_prompt=NULL, + updated_at=NOW() +WHERE COALESCE(editorial_status, '')='regenerating'; + +UPDATE app_settings +SET value_json = to_jsonb($prompt$ +Ты редактор русского Telegram-канала про тактическую, милитари и airsoft-экипировку. +Пиши от лица канала: как экспертный наблюдатель, а не как производитель или магазин. + +Задача: превратить исходный пост в короткую новостную публикацию для канала. + +Стиль: +- коротко, уверенно, по делу; +- без рекламного восторга и без выдуманных характеристик; +- только факты из исходного текста; +- если фактов мало, делай короткий пост, не растягивай; +- текст должен быть структурированным, не простынёй. + +Формат текста: +- 3-6 коротких строк или 2-4 компактных абзаца; +- не добавляй ссылки и хэштеги в text. + +Важное правило про источник: +- producer_name — это источник поста: производитель, магазин или площадка. +- Не утверждай, что producer_name «выпустил», «сделал», «представил» товар, если из текста не ясно, что это именно производитель этого товара. +- Если источник выглядит как магазин/площадка или в тексте упомянут другой бренд, формулируй нейтрально: «у [producer_name] появился...», «[producer_name] показали/добавили...», «в продаже появился...». +- Если из текста явно видно, что источник сам производит товар, можно писать: «[producer_name] выпустили...». + +Категория: +- выбери ровно одну category из списка categories во входных данных; +- если сомневаешься, выбирай ближайшую по назначению товара. + +OUTPUT: JSON only. +$prompt$::text), + updated_at = NOW() +WHERE key='ai_writer_prompt'; + +UPDATE app_settings +SET value_json = to_jsonb($contract$ +OUTPUT SCHEMA (return array matching input order): +{"rewrites":[{"id":123,"category":"разгрузка","text":"готовый текст без хэштегов","notes":"короткая заметка для редактора или null"}]} + +Rules: +- Input is a JSON object with key "posts" containing accepted posts. +- Each post includes producer_name and producer_tag. Use producer_name carefully: it can be a manufacturer, shop, or publishing source. +- Do not add hashtags to rewritten text. Pick exactly one category from the categories list in input. +- Return JSON only. No markdown. No text outside JSON. +- Return one rewrite for every input post id. +$contract$::text), + updated_at = NOW() +WHERE key='ai_writer_contract'; diff --git a/db/migrations/017_content_categories_and_media_cleanup.sql b/db/migrations/017_content_categories_and_media_cleanup.sql new file mode 100644 index 0000000..a213b81 --- /dev/null +++ b/db/migrations/017_content_categories_and_media_cleanup.sql @@ -0,0 +1,24 @@ +CREATE TABLE IF NOT EXISTS content_categories ( + id BIGSERIAL PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + tag TEXT NOT NULL, + is_active BOOLEAN NOT NULL DEFAULT TRUE, + sort_order INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_content_categories_active_order +ON content_categories(is_active, sort_order, name); + +INSERT INTO content_categories(name, tag, sort_order) +SELECT value, regexp_replace(lower(value), '[^[:alnum:]_а-яё]+', '_', 'g'), ordinality::int +FROM app_settings s +CROSS JOIN LATERAL jsonb_array_elements_text( + CASE WHEN jsonb_typeof(s.value_json) = 'array' THEN s.value_json ELSE '[]'::jsonb END +) WITH ORDINALITY AS items(value, ordinality) +WHERE s.key = 'ai_writer_categories' +ON CONFLICT (name) DO UPDATE +SET tag = EXCLUDED.tag, + sort_order = LEAST(content_categories.sort_order, EXCLUDED.sort_order), + updated_at = NOW(); diff --git a/db/migrations/018_writer_prompt_contract_cleanup.sql b/db/migrations/018_writer_prompt_contract_cleanup.sql new file mode 100644 index 0000000..df004d1 --- /dev/null +++ b/db/migrations/018_writer_prompt_contract_cleanup.sql @@ -0,0 +1,25 @@ +UPDATE app_settings +SET value_json = to_jsonb( + trim( + regexp_replace( + value_json #>> '{}', + E'\\n?OUTPUT:[[:space:]]*JSON only\\.[[:space:]]*', + E'\\n', + 'gi' + ) + ) +), +updated_at = NOW() +WHERE key = 'ai_writer_prompt' + AND value_json #>> '{}' ~* 'OUTPUT:[[:space:]]*JSON only\.'; + +UPDATE app_settings +SET value_json = to_jsonb( + CASE + WHEN value_json #>> '{}' ~* 'Return JSON only\\.' + THEN value_json #>> '{}' + ELSE trim(value_json #>> '{}') || E'\n- Return JSON only. No markdown. No text outside JSON.' + END +), +updated_at = NOW() +WHERE key = 'ai_writer_contract'; diff --git a/db/migrations/019_writer_prompt_profile_style.sql b/db/migrations/019_writer_prompt_profile_style.sql new file mode 100644 index 0000000..0377aef --- /dev/null +++ b/db/migrations/019_writer_prompt_profile_style.sql @@ -0,0 +1,74 @@ +UPDATE app_settings +SET value_json = to_jsonb($prompt$ +РОЛЬ +Ты редактор профильного русского Telegram-канала про тактическую, милитари и airsoft-экипировку. +Ты эксперт в тактическом, военном и страйкбольном снаряжении. Последние 10 лет ты следишь за новостями про новинки тактического снаряжения. +Ты профессиональный копирайтер и эксперт по SEO-оптимизации текстов для сайтов и соцсетей. +Пиши от лица канала: как экспертный наблюдатель, а не как производитель или магазин. + +ТВОИ ЗНАНИЯ +1. Ты профессионально разбираешься в материалах и технологиях, которые используются в производстве военного и тактического снаряжения: типы тканей, кордуры, стропы, фурнитура, интерфейсы креплений, системы быстрого сброса, мембраны, корсетные системы, баллистическая и противоосколочная защита. +2. Ты разбираешься в практическом применении тактического, военного и страйкбольного снаряжения. Можешь отличить, что подходит для страйкбола, что для гражданских стрелков, что для военных, а что универсально для всех направлений. +3. Ты профессионально разбираешься в истории тактического и военного снаряжения. По конструкции можешь отличить классические и популярные модели от новых разработок последних лет. + +ЗАДАЧА +Превратить исходный пост в короткую новостную публикацию для канала. + +РЕЗУЛЬТАТ +Создать грамотный, понятный, структурированный и оптимизированный для русскоязычной аудитории текст для публикации в Telegram и ВКонтакте. +Текст должен быть адаптирован для военных, гражданских стрелков и людей, которые интересуются военным, тактическим и страйкбольным снаряжением. + +СТРУКТУРА ТЕКСТА +1. SEO-заголовок с обязательным названием компании/источника и названием продукта или предмета новости. Пример: "Новый подсумок XXX от компании YYY". +2. Черта-разделитель. +3. Краткое объяснение, чем интересна новость или новинка. Указывай конкретику, избегай общих слов. Пример: "Арс Арма выпустила подсумок" лучше, чем "Российский производитель снаряжения выпустил подсумок". +4. Тело новости: основные свойства, назначение, технические особенности. Лаконично, используя факты исходного текста, опиши все важные свойства или тезисы. Если блок получился слишком объёмным, сократи его без потери смысла. +5. Если есть в исходнике, кратко укажи материалы и цвета, в которых производится новинка. +6. Если есть в исходнике, укажи стоимость новинки. +7. Черта-разделитель. +8. Фраза: "Подписывайтесь на нас в МАКС, ТГ, ВК". + +ФОРМАТ ТЕКСТА +- 3-6 коротких строк или 2-5 компактных абзацев; +- не добавляй ссылки и хэштеги в text. + +СТИЛЬ +- коротко, уверенно, по делу; +- без рекламного восторга и без выдуманных характеристик; +- нельзя использовать абстрактные слова типа "успех", "секрет", "мотивация", "справишься" и подобные; +- только факты из исходного текста; +- нельзя придумывать факты; +- если фактов мало, делай короткий пост, не растягивай; +- текст должен быть структурированным, не простынёй; +- используй подход "Пиши, сокращай": конкретика лучше общих и абстрактных формулировок. + +ВАЖНОЕ ПРАВИЛО ПРО ИСТОЧНИК +- producer_name — это источник поста: производитель, магазин или площадка. +- Не утверждай, что producer_name "выпустил", "сделал", "представил" товар, если из текста не ясно, что это именно производитель этого товара. +- Если источник выглядит как магазин/площадка или в тексте упомянут другой бренд, формулируй нейтрально: "у [producer_name] появился...", "[producer_name] показали/добавили...", "в продаже появился...". +- Если из текста явно видно, что источник сам производит товар, можно писать: "[producer_name] выпустили...". + +КАТЕГОРИЯ +- выбери ровно одну category из списка categories во входных данных; +- если сомневаешься, выбирай ближайшую по назначению товара. +$prompt$::text), + updated_at = NOW() +WHERE key='ai_writer_prompt'; + +UPDATE app_settings +SET value_json = to_jsonb($contract$ +OUTPUT SCHEMA (return array matching input order): +{"rewrites":[{"id":123,"category":"разгрузка","text":"готовый текст без хэштегов","notes":"короткая заметка для редактора или null"}]} + +Rules: +- Input is a JSON object with key "posts" containing accepted posts. +- Each post includes producer_name and producer_tag. producer_name can be a manufacturer, shop, or publishing source. +- Pick exactly one category from the categories list in input. +- Do not add links or hashtags to rewritten text. +- Return JSON only. No markdown. No text outside JSON. +- Return one rewrite for every input post id. +- The "text" field must be a ready-to-publish Russian post with normal paragraph line breaks encoded as JSON string newlines. +- The "notes" field must be a short editor note in Russian or null. +$contract$::text), + updated_at = NOW() +WHERE key='ai_writer_contract'; diff --git a/db/migrations/020_writer_category_ids.sql b/db/migrations/020_writer_category_ids.sql new file mode 100644 index 0000000..d8801ee --- /dev/null +++ b/db/migrations/020_writer_category_ids.sql @@ -0,0 +1,74 @@ +ALTER TABLE raw_posts + ADD COLUMN IF NOT EXISTS rewrite_category_id INTEGER, + ADD COLUMN IF NOT EXISTS final_category_id INTEGER; + +WITH duplicate_categories AS ( + SELECT id, + ROW_NUMBER() OVER (PARTITION BY sort_order ORDER BY is_active DESC, id) AS duplicate_rank + FROM content_categories +), +renumber AS ( + SELECT id, + (SELECT COALESCE(MAX(sort_order), 0) FROM content_categories) + + ROW_NUMBER() OVER (ORDER BY id) AS new_sort_order + FROM duplicate_categories + WHERE duplicate_rank > 1 +) +UPDATE content_categories c +SET sort_order = r.new_sort_order, + updated_at = NOW() +FROM renumber r +WHERE c.id = r.id; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_content_categories_public_id +ON content_categories(sort_order); + +UPDATE raw_posts rp +SET rewrite_category_id = c.sort_order +FROM content_categories c +WHERE rp.rewrite_category_id IS NULL + AND rp.rewrite_category IS NOT NULL + AND ( + lower(c.name) = lower(rp.rewrite_category) + OR lower(c.tag) = lower(COALESCE(rp.rewrite_category_tag, '')) + ); + +UPDATE raw_posts rp +SET final_category_id = c.sort_order +FROM content_categories c +WHERE rp.final_category_id IS NULL + AND rp.final_category IS NOT NULL + AND ( + lower(c.name) = lower(rp.final_category) + OR lower(c.tag) = lower(COALESCE(rp.final_category_tag, '')) + ); + +UPDATE app_settings +SET value_json = to_jsonb( + replace( + value_json #>> '{}', + E'- выбери ровно одну category из списка categories во входных данных;\n- если сомневаешься, выбирай ближайшую по назначению товара.', + E'- выбери ровно один category_id из списка categories во входных данных;\n- categories приходит списком объектов: id, name, tag;\n- если сомневаешься, выбирай ближайший id по назначению товара.' + ) +), +updated_at = NOW() +WHERE key='ai_writer_prompt'; + +UPDATE app_settings +SET value_json = to_jsonb($contract$ +OUTPUT SCHEMA (return array matching input order): +{"rewrites":[{"id":123,"category_id":2,"text":"готовый текст без хэштегов","notes":"короткая заметка для редактора или null"}]} + +Rules: +- Input is a JSON object with key "posts" containing accepted posts. +- Input has key "categories" with objects: id, name, tag. +- Each post includes producer_name and producer_tag. producer_name can be a manufacturer, shop, or publishing source. +- Pick exactly one category from the categories list in input and return its numeric id as category_id. +- Do not add links or hashtags to rewritten text. +- Return JSON only. No markdown. No text outside JSON. +- Return one rewrite for every input post id. +- The "text" field must be a ready-to-publish Russian post with normal paragraph line breaks encoded as JSON string newlines. +- The "notes" field must be a short editor note in Russian or null. +$contract$::text), +updated_at = NOW() +WHERE key='ai_writer_contract'; diff --git a/db/migrations/021_writer_contract_no_empty_response.sql b/db/migrations/021_writer_contract_no_empty_response.sql new file mode 100644 index 0000000..3f49cc0 --- /dev/null +++ b/db/migrations/021_writer_contract_no_empty_response.sql @@ -0,0 +1,19 @@ +UPDATE app_settings +SET value_json = to_jsonb($contract$ +OUTPUT SCHEMA (return array matching input order): +{"rewrites":[{"id":123,"category_id":2,"text":"готовый текст без хэштегов","notes":"короткая заметка для редактора или null"}]} + +Rules: +- Input is a JSON object with key "posts" containing accepted posts. +- Input has key "categories" with objects: id, name, tag. +- Each post includes producer_name and producer_tag. producer_name can be a manufacturer, shop, or publishing source. +- Pick exactly one category from the categories list in input and return its numeric id as category_id. +- Do not add links or hashtags to rewritten text. +- Return JSON only. No markdown. No text outside JSON. +- Never return an empty object. Never omit the "rewrites" key. +- Return one rewrite for every input post id. +- The "text" field must be a ready-to-publish Russian post with normal paragraph line breaks encoded as JSON string newlines. +- The "notes" field must be a short editor note in Russian or null. +$contract$::text), +updated_at = NOW() +WHERE key='ai_writer_contract'; diff --git a/db/migrations/022_audit_log_retention_and_indexes.sql b/db/migrations/022_audit_log_retention_and_indexes.sql new file mode 100644 index 0000000..9712a9c --- /dev/null +++ b/db/migrations/022_audit_log_retention_and_indexes.sql @@ -0,0 +1,15 @@ +DELETE FROM audit_log +WHERE created_at < NOW() - INTERVAL '30 days'; + +CREATE INDEX IF NOT EXISTS idx_audit_log_created_at +ON audit_log(created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_audit_log_action_time +ON audit_log(action, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_audit_log_actor_time +ON audit_log(actor_id, created_at DESC) +WHERE actor_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_audit_log_entity_time +ON audit_log(entity_type, entity_id, created_at DESC); diff --git a/db/migrations/022_sync_ai_worker_enabled.sql b/db/migrations/022_sync_ai_worker_enabled.sql new file mode 100644 index 0000000..52cdc62 --- /dev/null +++ b/db/migrations/022_sync_ai_worker_enabled.sql @@ -0,0 +1,15 @@ +UPDATE app_settings s +SET value_json = to_jsonb(wc.enabled), + updated_at = NOW() +FROM worker_controls wc +WHERE (wc.name, s.key) IN ( + ('ai-qualifier', 'ai_qualifier_enabled'), + ('ai-writer', 'ai_writer_enabled') +); + +UPDATE raw_posts +SET rewrite_status='failed', + rewrite_notes=COALESCE(rewrite_notes, 'recovered stale writer processing state'), + updated_at=NOW() +WHERE rewrite_status='processing' + AND updated_at < NOW() - INTERVAL '30 minutes'; diff --git a/db/migrations/023_writer_strict_separator.sql b/db/migrations/023_writer_strict_separator.sql new file mode 100644 index 0000000..96e6199 --- /dev/null +++ b/db/migrations/023_writer_strict_separator.sql @@ -0,0 +1,21 @@ +-- Keep the AI-writer divider deterministic. +-- Target separator length: 27 characters. + +UPDATE app_settings +SET value_json = to_jsonb( + replace( + value_json #>> '{}', + '2. Черта с точным количеством символов ━━━━━━━━━━━━━━━━━━━━━━━━━━━', + '2. Черта-разделитель: отдельная строка ровно из 27 символов. Используй строго эту строку без изменений: ━━━━━━━━━━━━━━━━━━━━━━━━━━━' + ) +), +updated_at = NOW() +WHERE key = 'ai_writer_prompt' + AND (value_json #>> '{}') LIKE '%Черта с точным количеством символов%'; + +UPDATE raw_posts +SET rewritten_text = regexp_replace(rewritten_text, E'(^━{3,}$)', '━━━━━━━━━━━━━━━━━━━━━━━━━━━', 'gm'), + final_text = regexp_replace(final_text, E'(^━{3,}$)', '━━━━━━━━━━━━━━━━━━━━━━━━━━━', 'gm'), + updated_at = NOW() +WHERE COALESCE(rewritten_text, '') LIKE '%━%' + OR COALESCE(final_text, '') LIKE '%━%'; diff --git a/db/migrations/024_tg_poster.sql b/db/migrations/024_tg_poster.sql new file mode 100644 index 0000000..965507f --- /dev/null +++ b/db/migrations/024_tg_poster.sql @@ -0,0 +1,93 @@ +ALTER TABLE raw_posts + ADD COLUMN IF NOT EXISTS publication_status TEXT NOT NULL DEFAULT 'pending', + ADD COLUMN IF NOT EXISTS publication_error TEXT, + ADD COLUMN IF NOT EXISTS published_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS tg_publication_chat_id BIGINT, + ADD COLUMN IF NOT EXISTS tg_publication_thread_id BIGINT, + ADD COLUMN IF NOT EXISTS tg_publication_message_ids BIGINT[], + ADD COLUMN IF NOT EXISTS tg_publication_url TEXT; + +CREATE INDEX IF NOT EXISTS idx_raw_posts_publication_queue +ON raw_posts(publication_status, editorial_status, rewrite_status, published_at, reviewed_at, id); + +CREATE INDEX IF NOT EXISTS idx_raw_posts_publication_rank +ON raw_posts(final_category_tag, final_source_tag, published_at DESC); + +CREATE TABLE IF NOT EXISTS publication_runs ( + id BIGSERIAL PRIMARY KEY, + poster TEXT NOT NULL, + schedule_id TEXT NOT NULL, + scheduled_for DATE NOT NULL, + scheduled_time TEXT NOT NULL, + planned_count INTEGER NOT NULL DEFAULT 0, + published_count INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'started', + error TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE(poster, schedule_id, scheduled_for) +); + +CREATE INDEX IF NOT EXISTS idx_publication_runs_poster_date +ON publication_runs(poster, scheduled_for DESC, scheduled_time); + +INSERT INTO worker_controls(name, enabled, settings_json) +VALUES ('tg-poster', FALSE, '{}'::jsonb) +ON CONFLICT (name) DO NOTHING; + +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('tg_poster_enabled', 'false'::jsonb, 'bool', 'TG poster enabled', 'Double safety flag for publishing accepted posts to Telegram.', 'TG Poster'), + ('tg_poster_bot_token', '""'::jsonb, 'secret', 'TG poster bot token', 'Bot token for final Telegram publication. For test can be the same as upload bot token.', 'TG Poster'), + ('tg_poster_chat_id', '""'::jsonb, 'str', 'TG publication chat id', 'Target Telegram channel/group id. Supports chat_id:thread_id for topics.', 'TG Poster'), + ('tg_poster_schedule_json', '[{"id":"morning","time":"10:00","count":1,"enabled":true},{"id":"evening","time":"18:00","count":1,"enabled":true}]'::jsonb, 'json', 'TG publication schedule', 'EKB time schedule rows: id, time HH:MM, count, enabled.', 'TG Poster'), + ('tg_poster_interval_sec', '60'::jsonb, 'int', 'TG poster interval, sec', 'Pause between poster checks.', 'TG Poster'), + ('tg_poster_media_group_max_items', '10'::jsonb, 'int', 'TG media group max items', 'Telegram media group supports 2..10 media items.', 'TG Poster'), + ('tg_poster_caption_limit', '1024'::jsonb, 'int', 'TG caption limit', 'Telegram caption limit for media messages.', 'TG Poster'), + ('tg_poster_message_limit', '4096'::jsonb, 'int', 'TG message limit', 'Telegram text message limit.', 'TG Poster'), + ('tg_poster_max_attempts', '3'::jsonb, 'int', 'TG post retry attempts', 'Retries for Telegram API errors.', 'TG Poster'), + ('tg_poster_retry_backoff_max_sec', '30'::jsonb, 'int', 'TG retry backoff max, sec', 'Max retry sleep for Telegram API errors.', 'TG Poster'), + ('tg_poster_send_delay_sec', '1'::jsonb, 'float', 'TG send delay, sec', 'Delay between Telegram sends.', 'TG Poster'), + ('tg_poster_text_overflow_caption', '"⬇️ Описание под медиа"'::jsonb, 'str', 'TG overflow caption', 'Caption used when full text is longer than Telegram media caption limit.', 'TG Poster'), + ('tg_poster_recent_window', '20'::jsonb, 'int', 'Ranking recent window', 'How many published posts are checked to reduce repeated categories and sources.', 'TG Poster'), + ('tg_poster_category_repeat_penalty', '3'::jsonb, 'float', 'Category repeat penalty', 'Ranking penalty for recently published same category.', 'TG Poster'), + ('tg_poster_source_repeat_penalty', '4'::jsonb, 'float', 'Source repeat penalty', 'Ranking penalty for recently published same source.', 'TG Poster') +ON CONFLICT (key) DO NOTHING; + +UPDATE app_settings +SET description='Обрезает исходный raw-текст перед отправкой в LLM. Итоговый rewrite должен быть компактным: Telegram caption 1024 символа минус будущая строка хэштегов.' +WHERE key='ai_writer_max_text_chars'; + +UPDATE app_settings +SET value_json = to_jsonb($contract$ +OUTPUT SCHEMA (return array matching input order): +{"rewrites":[{"id":123,"category_id":2,"text":"готовый текст без хэштегов","notes":"короткая заметка для редактора или null"}]} + +Rules: +- Input is a JSON object with key "posts" containing accepted posts. +- Input has key "categories" with objects: id, name, tag. +- Each post includes producer_name and producer_tag. producer_name can be a manufacturer, shop, or publishing source. +- Pick exactly one category from the categories list in input and return its numeric id as category_id. +- Do not add links or hashtags to rewritten text. +- Keep text compact enough for Telegram media caption: target 850 characters, hard maximum 900 characters. +- Remember the system appends category and source hashtags after text, so leave space for them. +- Return JSON only. No markdown. No text outside JSON. +- Never return an empty object. Never omit the "rewrites" key. +- Return one rewrite for every input post id. +- The "text" field must be a ready-to-publish Russian post with normal paragraph line breaks encoded as JSON string newlines. +- The "notes" field must be a short editor note in Russian or null. +$contract$::text), +updated_at = NOW() +WHERE key='ai_writer_contract'; + +UPDATE app_settings +SET value_json = to_jsonb( + regexp_replace( + value_json #>> '{}', + '- 3-6 коротких строк или 2-5 компактных абзацев;', + '- 3-6 коротких строк или 2-4 компактных абзаца; итоговый text должен целиться в 850 символов и не превышать 900 символов, потому что система добавит хэштеги и Telegram caption ограничен 1024 символами;', + 'g' + ) +) +WHERE key='ai_writer_prompt' + AND (value_json #>> '{}') LIKE '%3-6 коротких строк или 2-5 компактных абзацев%'; diff --git a/db/migrations/025_tg_poster_hourly_schedule.sql b/db/migrations/025_tg_poster_hourly_schedule.sql new file mode 100644 index 0000000..880f022 --- /dev/null +++ b/db/migrations/025_tg_poster_hourly_schedule.sql @@ -0,0 +1,46 @@ +UPDATE app_settings +SET value_json = '"⬇️ Описание"'::jsonb, + updated_at = NOW() +WHERE key='tg_poster_text_overflow_caption'; + +UPDATE app_settings +SET value_json = $schedule$[ + {"id":"hour-00","time":"00:00","count":4,"enabled":true}, + {"id":"hour-01","time":"01:00","count":4,"enabled":true}, + {"id":"hour-02","time":"02:00","count":4,"enabled":true}, + {"id":"hour-03","time":"03:00","count":4,"enabled":true}, + {"id":"hour-04","time":"04:00","count":4,"enabled":true}, + {"id":"hour-05","time":"05:00","count":4,"enabled":true}, + {"id":"hour-06","time":"06:00","count":4,"enabled":true}, + {"id":"hour-07","time":"07:00","count":4,"enabled":true}, + {"id":"hour-08","time":"08:00","count":4,"enabled":true}, + {"id":"hour-09","time":"09:00","count":4,"enabled":true}, + {"id":"hour-10","time":"10:00","count":4,"enabled":true}, + {"id":"hour-11","time":"11:00","count":4,"enabled":true}, + {"id":"hour-12","time":"12:00","count":4,"enabled":true}, + {"id":"hour-13","time":"13:00","count":4,"enabled":true}, + {"id":"hour-14","time":"14:00","count":4,"enabled":true}, + {"id":"hour-15","time":"15:00","count":4,"enabled":true}, + {"id":"hour-16","time":"16:00","count":4,"enabled":true}, + {"id":"hour-17","time":"17:00","count":4,"enabled":true}, + {"id":"hour-18","time":"18:00","count":4,"enabled":true}, + {"id":"hour-19","time":"19:00","count":4,"enabled":true}, + {"id":"hour-20","time":"20:00","count":4,"enabled":true}, + {"id":"hour-21","time":"21:00","count":4,"enabled":true}, + {"id":"hour-22","time":"22:00","count":4,"enabled":true}, + {"id":"hour-23","time":"23:00","count":4,"enabled":true} +]$schedule$::jsonb, + updated_at = NOW() +WHERE key='tg_poster_schedule_json'; + +UPDATE raw_posts +SET editorial_status='published', + updated_at=NOW() +WHERE publication_status='published' + AND editorial_status='accepted'; + +UPDATE raw_posts +SET editorial_status='publish_failed', + updated_at=NOW() +WHERE publication_status='publish_failed' + AND editorial_status='accepted'; diff --git a/db/migrations/026_writer_separator_17_chars.sql b/db/migrations/026_writer_separator_17_chars.sql new file mode 100644 index 0000000..a415f53 --- /dev/null +++ b/db/migrations/026_writer_separator_17_chars.sql @@ -0,0 +1,44 @@ +-- Keep the AI-writer divider deterministic. +-- Target separator length: 17 characters. + +UPDATE app_settings +SET value_json = to_jsonb( + regexp_replace( + value_json #>> '{}', + 'Черта-разделитель:[^\n]*', + 'Черта-разделитель: отдельная строка ровно из 17 символов. Используй строго эту строку без изменений: ━━━━━━━━━━━━━━━━━', + 'g' + ) +), +updated_at = NOW() +WHERE key = 'ai_writer_prompt' + AND (value_json #>> '{}') LIKE '%Черта-разделитель%'; + +UPDATE app_settings +SET value_json = to_jsonb( + CASE + WHEN (value_json #>> '{}') LIKE '%Separator line must be exactly%' + THEN regexp_replace( + value_json #>> '{}', + 'Separator line must be exactly[^\n]*', + 'Separator line must be exactly this 17-character line when used: ━━━━━━━━━━━━━━━━━', + 'g' + ) + ELSE (value_json #>> '{}') || E'\n- Separator line must be exactly this 17-character line when used: ━━━━━━━━━━━━━━━━━' + END +), +updated_at = NOW() +WHERE key = 'ai_writer_contract'; + +UPDATE raw_posts +SET rewritten_text = CASE + WHEN rewritten_text IS NULL THEN NULL + ELSE regexp_replace(rewritten_text, E'^[━─—-]{3,}\\s*$', '━━━━━━━━━━━━━━━━━', 'gm') + END, + final_text = CASE + WHEN final_text IS NULL THEN NULL + ELSE regexp_replace(final_text, E'^[━─—-]{3,}\\s*$', '━━━━━━━━━━━━━━━━━', 'gm') + END, + updated_at = NOW() +WHERE COALESCE(rewritten_text, '') ~ E'(?m)^[━─—-]{3,}\\s*$' + OR COALESCE(final_text, '') ~ E'(?m)^[━─—-]{3,}\\s*$'; diff --git a/db/migrations/027_raw_manual_rewrite_and_nck_reject.sql b/db/migrations/027_raw_manual_rewrite_and_nck_reject.sql new file mode 100644 index 0000000..714f855 --- /dev/null +++ b/db/migrations/027_raw_manual_rewrite_and_nck_reject.sql @@ -0,0 +1,34 @@ +-- Manual Raw-to-rewrite flow is implemented in admin.py. +-- Non-target category (sort_order/category_id 18, tag НЦК) must not enter editorial review. + +UPDATE app_settings +SET value_json = to_jsonb( + CASE + WHEN (value_json #>> '{}') LIKE '%category_id 18%' + THEN value_json #>> '{}' + ELSE (value_json #>> '{}') || E'\n- If the post is non-target content, choose category_id 18 ("Не целевой контент", tag "НЦК"); the app will reject it automatically.' + END +), +updated_at = NOW() +WHERE key = 'ai_writer_contract'; + +UPDATE raw_posts +SET editorial_status='rejected', + qualification_status='rejected', + qualification_decision='rejected', + qualification_model_decision='rejected', + qualification_reason=COALESCE(NULLIF(qualification_reason, ''), 'AI writer selected non-target category'), + qualification_reject_tag=COALESCE(NULLIF(qualification_reject_tag, ''), 'non_target_content'), + editor_notes=COALESCE(NULLIF(editor_notes, ''), 'Отклонено AI: Не целевой контент'), + reviewed_at=COALESCE(reviewed_at, NOW()), + updated_at=NOW() +WHERE rewrite_status='ready' + AND COALESCE(editorial_status, 'review')='review' + AND ( + rewrite_category_id=18 + OR final_category_id=18 + OR lower(COALESCE(rewrite_category_tag, '')) IN ('нцк', 'nck') + OR lower(COALESCE(final_category_tag, '')) IN ('нцк', 'nck') + OR lower(COALESCE(rewrite_category, '')) = lower('Не целевой контент') + OR lower(COALESCE(final_category, '')) = lower('Не целевой контент') + ); diff --git a/db/migrations/028_vk_poster_and_platform_publications.sql b/db/migrations/028_vk_poster_and_platform_publications.sql new file mode 100644 index 0000000..08706a1 --- /dev/null +++ b/db/migrations/028_vk_poster_and_platform_publications.sql @@ -0,0 +1,67 @@ +CREATE TABLE IF NOT EXISTS post_publications ( + id BIGSERIAL PRIMARY KEY, + raw_post_id BIGINT NOT NULL REFERENCES raw_posts(id) ON DELETE CASCADE, + platform TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + target_id TEXT, + external_id TEXT, + url TEXT, + error TEXT, + attachments_json JSONB NOT NULL DEFAULT '[]'::jsonb, + published_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE(raw_post_id, platform) +); + +CREATE INDEX IF NOT EXISTS idx_post_publications_platform_status +ON post_publications(platform, status, target_id, published_at DESC); + +CREATE INDEX IF NOT EXISTS idx_post_publications_raw_post +ON post_publications(raw_post_id); + +INSERT INTO post_publications(raw_post_id, platform, status, target_id, external_id, url, error, attachments_json, published_at, updated_at) +SELECT + id, + 'tg', + publication_status, + tg_publication_chat_id::text, + CASE WHEN tg_publication_message_ids IS NULL THEN NULL ELSE array_to_string(tg_publication_message_ids, ',') END, + tg_publication_url, + publication_error, + COALESCE(to_jsonb(tg_publication_message_ids), '[]'::jsonb), + published_at, + NOW() +FROM raw_posts +WHERE COALESCE(publication_status, 'pending') <> 'pending' +ON CONFLICT (raw_post_id, platform) DO UPDATE +SET status=EXCLUDED.status, + target_id=EXCLUDED.target_id, + external_id=EXCLUDED.external_id, + url=EXCLUDED.url, + error=EXCLUDED.error, + attachments_json=EXCLUDED.attachments_json, + published_at=EXCLUDED.published_at, + updated_at=NOW(); + +INSERT INTO worker_controls(name, enabled, settings_json) +VALUES ('vk-poster', FALSE, '{}'::jsonb) +ON CONFLICT (name) DO NOTHING; + +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('vk_poster_enabled', 'false'::jsonb, 'bool', 'VK poster enabled', 'Double safety flag for publishing accepted posts to VK.', 'VK Poster'), + ('vk_poster_access_token', '""'::jsonb, 'secret', 'VK poster access token', 'Group/user token with wall.post and photos permissions. If empty, env VK_GROUP_ACCESS_TOKEN/VK_ACCESS_TOKEN is used.', 'VK Poster'), + ('vk_poster_owner_id', '0'::jsonb, 'int', 'VK target owner_id', 'Target VK group owner_id, usually negative: -123456789.', 'VK Poster'), + ('vk_poster_from_group', 'true'::jsonb, 'bool', 'Post from group', 'Use from_group=1 when publishing to a community wall.', 'VK Poster'), + ('vk_poster_schedule_json', '[{"id":"vk-morning","time":"10:05","count":1,"enabled":true},{"id":"vk-evening","time":"18:05","count":1,"enabled":true}]'::jsonb, 'json', 'VK publication schedule', 'EKB time schedule rows: id, time HH:MM, count, enabled.', 'VK Poster'), + ('vk_poster_interval_sec', '60'::jsonb, 'int', 'VK poster interval, sec', 'Pause between poster checks.', 'VK Poster'), + ('vk_poster_timeout_sec', '90'::jsonb, 'int', 'VK API timeout, sec', 'Total timeout for VK API calls and media uploads.', 'VK Poster'), + ('vk_poster_max_attempts', '3'::jsonb, 'int', 'VK post retry attempts', 'Retries for VK API/network errors.', 'VK Poster'), + ('vk_poster_retry_backoff_max_sec', '30'::jsonb, 'int', 'VK retry backoff max, sec', 'Max retry sleep for VK API/network errors.', 'VK Poster'), + ('vk_poster_send_delay_sec', '1'::jsonb, 'float', 'VK send delay, sec', 'Delay between VK posts.', 'VK Poster'), + ('vk_poster_recent_window', '20'::jsonb, 'int', 'Ranking recent window', 'How many VK-published posts are checked to reduce repeated categories and sources.', 'VK Poster'), + ('vk_poster_category_repeat_penalty', '3'::jsonb, 'float', 'Category repeat penalty', 'Ranking penalty for recently published same category.', 'VK Poster'), + ('vk_poster_source_repeat_penalty', '4'::jsonb, 'float', 'Source repeat penalty', 'Ranking penalty for recently published same source.', 'VK Poster'), + ('vk_poster_dry_run', 'false'::jsonb, 'bool', 'VK dry run', 'If true, prepares attachments but does not call wall.post. Keep false for production.', 'VK Poster') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/029_vk_poster_oauth_settings.sql b/db/migrations/029_vk_poster_oauth_settings.sql new file mode 100644 index 0000000..e24de00 --- /dev/null +++ b/db/migrations/029_vk_poster_oauth_settings.sql @@ -0,0 +1,5 @@ +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('vk_poster_app_id', '""'::jsonb, 'str', 'VK OAuth app id', 'VK application id used for user OAuth code flow.', 'VK Poster'), + ('vk_poster_client_secret', '""'::jsonb, 'secret', 'VK OAuth client secret', 'Protected key from VK application settings. Used server-side to exchange code for user access token.', 'VK Poster') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/030_vk_poster_refresh_token.sql b/db/migrations/030_vk_poster_refresh_token.sql new file mode 100644 index 0000000..db4b6f3 --- /dev/null +++ b/db/migrations/030_vk_poster_refresh_token.sql @@ -0,0 +1,6 @@ +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('vk_poster_refresh_token', '""'::jsonb, 'secret', 'VK OAuth refresh token', 'Refresh token from VK ID code flow. Used by VK poster to renew access token automatically.', 'VK Poster'), + ('vk_poster_token_device_id', '""'::jsonb, 'str', 'VK OAuth device id', 'VK ID device_id returned with authorization code/token exchange.', 'VK Poster'), + ('vk_poster_token_expires_at', '""'::jsonb, 'str', 'VK access token expires at', 'UTC ISO timestamp used by VK poster to refresh the access token before expiry.', 'VK Poster') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/031_writer_require_producer_name.sql b/db/migrations/031_writer_require_producer_name.sql new file mode 100644 index 0000000..94ca386 --- /dev/null +++ b/db/migrations/031_writer_require_producer_name.sql @@ -0,0 +1,18 @@ +-- Make producer mention mandatory in every AI writer result. + +UPDATE app_settings +SET value_json = to_jsonb( + CASE + WHEN (value_json #>> '{}') LIKE '%explicitly mention producer_name%' + OR (value_json #>> '{}') LIKE '%явно упоминает producer_name%' + THEN value_json #>> '{}' + ELSE replace( + value_json #>> '{}', + '- Each post includes producer_name and producer_tag. producer_name can be a manufacturer, shop, or publishing source.', + '- Each post includes producer_name and producer_tag. producer_name can be a manufacturer, shop, or publishing source.' || E'\n' || + '- The "text" field for every rewrite must explicitly mention producer_name at least once. Do not satisfy this only through producer_tag, hashtags, notes, or metadata.' + ) + END +), +updated_at = NOW() +WHERE key = 'ai_writer_contract'; diff --git a/db/migrations/032_tg_reactor.sql b/db/migrations/032_tg_reactor.sql new file mode 100644 index 0000000..b99e6a6 --- /dev/null +++ b/db/migrations/032_tg_reactor.sql @@ -0,0 +1,39 @@ +CREATE TABLE IF NOT EXISTS post_reactions ( + id BIGSERIAL PRIMARY KEY, + publication_id BIGINT NOT NULL REFERENCES post_publications(id) ON DELETE CASCADE, + raw_post_id BIGINT NOT NULL REFERENCES raw_posts(id) ON DELETE CASCADE, + platform TEXT NOT NULL, + reactor_key TEXT NOT NULL, + reaction TEXT NOT NULL, + target_id TEXT NOT NULL, + message_id TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + error TEXT, + attempts INTEGER NOT NULL DEFAULT 0, + reacted_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE(publication_id, reactor_key, message_id) +); + +CREATE INDEX IF NOT EXISTS idx_post_reactions_status +ON post_reactions(platform, status, updated_at); + +CREATE INDEX IF NOT EXISTS idx_post_reactions_raw_post +ON post_reactions(raw_post_id); + +INSERT INTO worker_controls(name, enabled, settings_json) +VALUES ('tg-reactor', FALSE, '{}'::jsonb) +ON CONFLICT (name) DO NOTHING; + +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('tg_reactor_enabled', 'false'::jsonb, 'bool', 'TG reactor enabled', 'Double safety flag for Telegram reaction worker.', 'TG Reactor'), + ('tg_reactor_bot_tokens', '""'::jsonb, 'secret', 'TG reactor bot tokens', 'One or more Telegram bot tokens separated by comma/newline. If empty, uses tg_poster_bot_token/TG_BOT_TOKEN.', 'TG Reactor'), + ('tg_reactor_reactions_json', '["👍"]'::jsonb, 'json', 'TG reactions', 'Emoji reactions. One bot can set one reaction; multiple tokens can set multiple reactions.', 'TG Reactor'), + ('tg_reactor_delay_sec', '60'::jsonb, 'int', 'Delay after publication, sec', 'How long to wait after TG publication before reacting.', 'TG Reactor'), + ('tg_reactor_interval_sec', '60'::jsonb, 'int', 'TG reactor interval, sec', 'Pause between reaction worker checks.', 'TG Reactor'), + ('tg_reactor_react_all_messages', 'false'::jsonb, 'bool', 'React to all media messages', 'If false, reacts only to the first message of each published post.', 'TG Reactor'), + ('tg_reactor_max_attempts', '3'::jsonb, 'int', 'TG reaction retry attempts', 'Retries per publication/message/reactor.', 'TG Reactor'), + ('tg_reactor_retry_backoff_max_sec', '30'::jsonb, 'int', 'TG reaction retry backoff max, sec', 'Max sleep for Telegram reaction retry errors.', 'TG Reactor') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/033_tg_reactor_since.sql b/db/migrations/033_tg_reactor_since.sql new file mode 100644 index 0000000..b96ae14 --- /dev/null +++ b/db/migrations/033_tg_reactor_since.sql @@ -0,0 +1,4 @@ +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('tg_reactor_since', '""'::jsonb, 'str', 'React only since', 'UTC ISO timestamp. If set, TG reactor ignores publications older than this moment.', 'TG Reactor') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/034_tg_reactor_reaction_pause.sql b/db/migrations/034_tg_reactor_reaction_pause.sql new file mode 100644 index 0000000..ff6098e --- /dev/null +++ b/db/migrations/034_tg_reactor_reaction_pause.sql @@ -0,0 +1,4 @@ +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('tg_reactor_reaction_pause_sec', '2'::jsonb, 'int', 'Pause between TG reactions, sec', 'Small pause after each successful reaction to avoid a burst of Telegram API calls.', 'TG Reactor') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/035_uploader_video_format_limits.sql b/db/migrations/035_uploader_video_format_limits.sql new file mode 100644 index 0000000..67a2ac3 --- /dev/null +++ b/db/migrations/035_uploader_video_format_limits.sql @@ -0,0 +1,14 @@ +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('video_max_height', '720'::jsonb, 'int', 'Video max height', 'Maximum VK video height selected by yt-dlp before uploading to Telegram.', 'Uploader') +ON CONFLICT (key) DO NOTHING; + +UPDATE app_settings +SET value_json='600'::jsonb, + updated_at=NOW() +WHERE key='uploader_yt_dlp_timeout_sec' + AND CASE + WHEN jsonb_typeof(value_json) = 'number' THEN (value_json #>> '{}')::int + WHEN jsonb_typeof(value_json) = 'string' AND (value_json #>> '{}') ~ '^\d+$' THEN (value_json #>> '{}')::int + ELSE 0 + END < 600; diff --git a/db/migrations/036_daily_report.sql b/db/migrations/036_daily_report.sql new file mode 100644 index 0000000..fe0a41d --- /dev/null +++ b/db/migrations/036_daily_report.sql @@ -0,0 +1,13 @@ +INSERT INTO worker_controls(name, enabled, settings_json) +VALUES ('daily-report', TRUE, '{}'::jsonb) +ON CONFLICT (name) DO NOTHING; + +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('daily_report_enabled', 'true'::jsonb, 'bool', 'Daily report enabled', 'Double safety flag for daily Telegram operational report.', 'Daily Report'), + ('daily_report_bot_token', '""'::jsonb, 'secret', 'Daily report bot token', 'Telegram bot token for reports. If empty, uses tg_poster_bot_token or TG_BOT_TOKEN.', 'Daily Report'), + ('daily_report_recipient_ids', '[442509142]'::jsonb, 'json', 'Daily report recipients', 'Telegram user/chat IDs that receive the daily report.', 'Daily Report'), + ('daily_report_time', '"00:04"'::jsonb, 'str', 'Daily report time', 'EKB time HH:MM when yesterday report is sent.', 'Daily Report'), + ('daily_report_interval_sec', '60'::jsonb, 'int', 'Daily report interval, sec', 'Pause between checks.', 'Daily Report'), + ('daily_report_last_sent_date', '""'::jsonb, 'str', 'Last sent report date', 'Internal YYYY-MM-DD report date already sent automatically.', 'Daily Report') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/037_site_category_fields.sql b/db/migrations/037_site_category_fields.sql new file mode 100644 index 0000000..26f7f00 --- /dev/null +++ b/db/migrations/037_site_category_fields.sql @@ -0,0 +1,57 @@ +ALTER TABLE content_categories + ADD COLUMN IF NOT EXISTS site_name TEXT, + ADD COLUMN IF NOT EXISTS site_slug TEXT, + ADD COLUMN IF NOT EXISTS site_enabled BOOLEAN NOT NULL DEFAULT TRUE; + +UPDATE content_categories +SET site_name = CASE sort_order + WHEN 1 THEN 'Одежда' + WHEN 2 THEN 'Снаряжение' + WHEN 3 THEN 'Броня' + WHEN 4 THEN 'Аксессуары' + WHEN 5 THEN 'Связь' + WHEN 6 THEN 'Патчи' + WHEN 7 THEN 'Навигация' + WHEN 8 THEN 'Прицелы' + WHEN 9 THEN 'Ночные приборы' + WHEN 10 THEN 'Медицина' + WHEN 11 THEN 'Свет' + WHEN 12 THEN 'Электроника' + WHEN 13 THEN 'Ножи' + WHEN 14 THEN 'Оружейный обвес' + WHEN 15 THEN 'Практика' + WHEN 16 THEN 'Теория' + WHEN 17 THEN 'Пушки' + WHEN 18 THEN 'НЦК' + ELSE initcap(replace(tag, '_', ' ')) + END, + site_slug = CASE sort_order + WHEN 1 THEN 'odezhda' + WHEN 2 THEN 'snaryazhenie' + WHEN 3 THEN 'bronya' + WHEN 4 THEN 'aksessuary' + WHEN 5 THEN 'svyaz' + WHEN 6 THEN 'patchi' + WHEN 7 THEN 'navigatsiya' + WHEN 8 THEN 'pricely' + WHEN 9 THEN 'nochnye-pribory' + WHEN 10 THEN 'medicina' + WHEN 11 THEN 'svet' + WHEN 12 THEN 'elektronika' + WHEN 13 THEN 'nozhi' + WHEN 14 THEN 'oruzheynyy-obves' + WHEN 15 THEN 'praktika' + WHEN 16 THEN 'teoriya' + WHEN 17 THEN 'pushki' + WHEN 18 THEN 'nck' + ELSE site_slug + END, + site_enabled = sort_order <> 18, + updated_at = NOW(); + +ALTER TABLE content_categories + ALTER COLUMN site_name SET NOT NULL, + ALTER COLUMN site_slug SET NOT NULL; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_content_categories_site_slug +ON content_categories(site_slug); diff --git a/db/migrations/038_site_poster.sql b/db/migrations/038_site_poster.sql new file mode 100644 index 0000000..3edd067 --- /dev/null +++ b/db/migrations/038_site_poster.sql @@ -0,0 +1,15 @@ +INSERT INTO worker_controls(name, enabled, settings_json) +VALUES ('site-poster', FALSE, '{}'::jsonb) +ON CONFLICT (name) DO NOTHING; + +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('site_poster_enabled', 'false'::jsonb, 'bool', 'Site poster enabled', 'Double safety flag for automatic site publishing.', 'Site Poster'), + ('site_poster_provider', '""'::jsonb, 'str', 'Site provider', 'Publish adapter: ghost now; wordpress can be added as a code adapter later.', 'Site Poster'), + ('site_poster_base_url', '""'::jsonb, 'str', 'Site API base URL', 'Public/admin base URL for the selected site provider.', 'Site Poster'), + ('site_poster_secret_file', '""'::jsonb, 'str', 'Site secret file', 'Server path to the selected provider API secret. Keep empty in git.', 'Site Poster'), + ('site_poster_fallback_title', '"Новость"'::jsonb, 'str', 'Fallback article title', 'Title used when post text has no first line.', 'Site Poster'), + ('site_poster_interval_sec', '300'::jsonb, 'int', 'Site poster interval, sec', 'Pause between automatic queue checks.', 'Site Poster'), + ('site_poster_photo_batch_size', '10'::jsonb, 'int', 'Photo batch size', 'Maximum photo-ready posts per automatic run.', 'Site Poster'), + ('site_poster_video_batch_size', '1'::jsonb, 'int', 'Video batch size', 'Maximum video-only posts per automatic run.', 'Site Poster') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/039_max_poster.sql b/db/migrations/039_max_poster.sql new file mode 100644 index 0000000..d36ab8c --- /dev/null +++ b/db/migrations/039_max_poster.sql @@ -0,0 +1,28 @@ +INSERT INTO worker_controls(name, enabled, settings_json) +VALUES ('max-poster', FALSE, '{}'::jsonb) +ON CONFLICT (name) DO NOTHING; + +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('max_poster_enabled', 'false'::jsonb, 'bool', 'MAX poster enabled', 'Double safety flag for publishing accepted posts to MAX.', 'MAX Poster'), + ('max_poster_bot_token', '""'::jsonb, 'secret', 'MAX poster bot token', 'Bot access token for MAX API. Keep empty in git and set on server.', 'MAX Poster'), + ('max_poster_chat_id', '""'::jsonb, 'str', 'MAX target chat id', 'Target MAX channel/group chat_id.', 'MAX Poster'), + ('max_poster_api_base_url', '"https://platform-api2.max.ru"'::jsonb, 'str', 'MAX API base URL', 'MAX Bot API base URL. Since 2026 use platform-api2.max.ru.', 'MAX Poster'), + ('max_poster_schedule_json', '[{"id":"hour-00","time":"00:00","count":4,"enabled":true},{"id":"hour-01","time":"01:00","count":4,"enabled":true},{"id":"hour-02","time":"02:00","count":4,"enabled":true},{"id":"hour-03","time":"03:00","count":4,"enabled":true},{"id":"hour-04","time":"04:00","count":4,"enabled":true},{"id":"hour-05","time":"05:00","count":4,"enabled":true},{"id":"hour-06","time":"06:00","count":4,"enabled":true},{"id":"hour-07","time":"07:00","count":4,"enabled":true},{"id":"hour-08","time":"08:00","count":4,"enabled":true},{"id":"hour-09","time":"09:00","count":4,"enabled":true},{"id":"hour-10","time":"10:00","count":4,"enabled":true},{"id":"hour-11","time":"11:00","count":4,"enabled":true},{"id":"hour-12","time":"12:00","count":4,"enabled":true},{"id":"hour-13","time":"13:00","count":4,"enabled":true},{"id":"hour-14","time":"14:00","count":4,"enabled":true},{"id":"hour-15","time":"15:00","count":4,"enabled":true},{"id":"hour-16","time":"16:00","count":4,"enabled":true},{"id":"hour-17","time":"17:00","count":4,"enabled":true},{"id":"hour-18","time":"18:00","count":4,"enabled":true},{"id":"hour-19","time":"19:00","count":4,"enabled":true},{"id":"hour-20","time":"20:00","count":4,"enabled":true},{"id":"hour-21","time":"21:00","count":4,"enabled":true},{"id":"hour-22","time":"22:00","count":4,"enabled":true},{"id":"hour-23","time":"23:00","count":4,"enabled":true}]'::jsonb, 'json', 'MAX publication schedule', 'EKB time schedule rows: id, time HH:MM, count, enabled. Defaults to the TG hourly schedule.', 'MAX Poster'), + ('max_poster_interval_sec', '60'::jsonb, 'int', 'MAX poster interval, sec', 'Pause between poster checks.', 'MAX Poster'), + ('max_poster_message_limit', '4000'::jsonb, 'int', 'MAX message limit', 'MAX text message limit.', 'MAX Poster'), + ('max_poster_media_max_items', '10'::jsonb, 'int', 'MAX media max items', 'Maximum media attachments per MAX message.', 'MAX Poster'), + ('max_poster_media_process_delay_sec', '3'::jsonb, 'float', 'MAX media process delay, sec', 'Pause after media upload before sending the message.', 'MAX Poster'), + ('max_poster_video_ready_attempts', '5'::jsonb, 'int', 'MAX video ready attempts', 'Retries for sending a message while MAX is still processing uploaded video.', 'MAX Poster'), + ('max_poster_video_ready_delay_sec', '10'::jsonb, 'float', 'MAX video ready delay, sec', 'Initial retry delay while MAX is processing uploaded video. Delay doubles after each attempt.', 'MAX Poster'), + ('max_poster_max_attempts', '3'::jsonb, 'int', 'MAX post retry attempts', 'Retries for MAX API/network errors and attachment processing.', 'MAX Poster'), + ('max_poster_retry_backoff_max_sec', '30'::jsonb, 'int', 'MAX retry backoff max, sec', 'Max retry sleep for MAX API/network errors.', 'MAX Poster'), + ('max_poster_send_delay_sec', '1'::jsonb, 'float', 'MAX send delay, sec', 'Delay between MAX sends.', 'MAX Poster'), + ('max_poster_recent_window', '20'::jsonb, 'int', 'Ranking recent window', 'How many MAX-published posts are checked to reduce repeated categories and sources.', 'MAX Poster'), + ('max_poster_category_repeat_penalty', '3'::jsonb, 'float', 'Category repeat penalty', 'Ranking penalty for recently published same category.', 'MAX Poster'), + ('max_poster_source_repeat_penalty', '4'::jsonb, 'float', 'Source repeat penalty', 'Ranking penalty for recently published same source.', 'MAX Poster'), + ('max_poster_auto_reaction_enabled', 'true'::jsonb, 'bool', 'MAX auto reaction enabled', 'Try to set a reaction after publishing. If MAX API rejects it, publication remains successful.', 'MAX Poster'), + ('max_poster_auto_reaction', '"👍"'::jsonb, 'str', 'MAX auto reaction', 'Emoji reaction to try after publishing.', 'MAX Poster'), + ('max_poster_reaction_path_template', '"/messages/{message_id}/reactions"'::jsonb, 'str', 'MAX reaction path template', 'Relative API path for reaction call if supported by MAX API.', 'MAX Poster'), + ('max_poster_dry_run', 'false'::jsonb, 'bool', 'MAX dry run', 'If true, prepares publication but does not call MAX send message.', 'MAX Poster') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/040_site_poster_provider_settings.sql b/db/migrations/040_site_poster_provider_settings.sql new file mode 100644 index 0000000..9972bd7 --- /dev/null +++ b/db/migrations/040_site_poster_provider_settings.sql @@ -0,0 +1,36 @@ +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('site_poster_provider', '""'::jsonb, 'str', 'Site provider', 'Publish adapter: ghost now; wordpress can be added as a code adapter later.', 'Site Poster'), + ('site_poster_base_url', '""'::jsonb, 'str', 'Site API base URL', 'Public/admin base URL for the selected site provider.', 'Site Poster'), + ('site_poster_secret_file', '""'::jsonb, 'str', 'Site secret file', 'Server path to the selected provider API secret. Keep empty in git.', 'Site Poster'), + ('site_poster_fallback_title', '"Новость"'::jsonb, 'str', 'Fallback article title', 'Title used when post text has no first line.', 'Site Poster') +ON CONFLICT (key) DO NOTHING; + +UPDATE app_settings +SET value_json='"ghost"'::jsonb, + updated_at=NOW() +WHERE key='site_poster_provider' + AND COALESCE(value_json #>> '{}', '') = '' + AND EXISTS ( + SELECT 1 FROM app_settings legacy + WHERE legacy.key='site_poster_ghost_url' + AND COALESCE(legacy.value_json #>> '{}', '') <> '' + ); + +UPDATE app_settings target +SET value_json=to_jsonb(legacy.value_json #>> '{}'), + updated_at=NOW() +FROM app_settings legacy +WHERE target.key='site_poster_base_url' + AND legacy.key='site_poster_ghost_url' + AND COALESCE(target.value_json #>> '{}', '') = '' + AND COALESCE(legacy.value_json #>> '{}', '') <> ''; + +UPDATE app_settings target +SET value_json=to_jsonb(legacy.value_json #>> '{}'), + updated_at=NOW() +FROM app_settings legacy +WHERE target.key='site_poster_secret_file' + AND legacy.key='site_poster_key_file' + AND COALESCE(target.value_json #>> '{}', '') = '' + AND COALESCE(legacy.value_json #>> '{}', '') <> ''; diff --git a/deploy/max-poster.service b/deploy/max-poster.service new file mode 100644 index 0000000..d1a8c30 --- /dev/null +++ b/deploy/max-poster.service @@ -0,0 +1,15 @@ +[Unit] +Description=VK Parser MAX Poster +After=network.target postgresql.service + +[Service] +Type=simple +WorkingDirectory=/opt/vk-parser +Environment=PYTHONPATH=/opt/vk-parser/src +ExecStart=/opt/vk-parser/.venv/bin/python -m vk_parser_app.workers.max_poster +Restart=always +RestartSec=10 +User=root + +[Install] +WantedBy=multi-user.target diff --git a/deploy/site-poster.service b/deploy/site-poster.service new file mode 100644 index 0000000..1874803 --- /dev/null +++ b/deploy/site-poster.service @@ -0,0 +1,14 @@ +[Unit] +Description=VK Parser Site Poster +After=network.target postgresql.service + +[Service] +Type=simple +WorkingDirectory=/opt/vk-parser +Environment=PYTHONPATH=/opt/vk-parser/src +ExecStart=/opt/vk-parser/.venv/bin/python -m vk_parser_app.workers.site_poster +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100644 index 0000000..53c4150 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env sh +set -eu + +tg_pid="" + +stop_children() { + if [ -n "$tg_pid" ] && kill -0 "$tg_pid" 2>/dev/null; then + kill "$tg_pid" + wait "$tg_pid" 2>/dev/null || true + fi +} + +trap stop_children INT TERM EXIT + +if [ -n "${TELEGRAM_API_ID:-}" ] && [ -n "${TELEGRAM_API_HASH:-}" ]; then + mkdir -p "${TELEGRAM_WORK_DIR:-/var/lib/telegram-bot-api}" "${TELEGRAM_TEMP_DIR:-/tmp/telegram-bot-api}" + telegram-bot-api \ + --api-id="${TELEGRAM_API_ID}" \ + --api-hash="${TELEGRAM_API_HASH}" \ + --dir="${TELEGRAM_WORK_DIR:-/var/lib/telegram-bot-api}" \ + --temp-dir="${TELEGRAM_TEMP_DIR:-/tmp/telegram-bot-api}" \ + --http-port="${TELEGRAM_HTTP_PORT:-8081}" \ + --local & + tg_pid="$!" + echo "Started local Telegram Bot API on 127.0.0.1:${TELEGRAM_HTTP_PORT:-8081}" +else + echo "TELEGRAM_API_ID/TELEGRAM_API_HASH are not set; local Telegram Bot API is disabled" +fi + +python /app/scripts/apply_migrations.py + +exec "$@" diff --git a/favi.png b/favi.png new file mode 100644 index 0000000..1b552f1 Binary files /dev/null and b/favi.png differ diff --git a/forma-n8-social-preview-v2.jpg b/forma-n8-social-preview-v2.jpg new file mode 100644 index 0000000..d06cbd2 Binary files /dev/null and b/forma-n8-social-preview-v2.jpg differ diff --git a/forma_n8_site/.env.example b/forma_n8_site/.env.example new file mode 100644 index 0000000..e075a5d --- /dev/null +++ b/forma_n8_site/.env.example @@ -0,0 +1,8 @@ +DATABASE_URL=postgresql://forma_n8:change-me@127.0.0.1:5432/forma_n8 +SITE_BASE_URL=http://127.0.0.1:8090 +SITE_NAME=Forma N8 +SITE_TAGLINE=Новости снаряжения +SITE_INDEXING_ENABLED=false +PUBLISH_API_TOKEN=change-me +UPLOAD_DIR=/var/lib/forma-n8/uploads +PAGE_SIZE=12 diff --git a/forma_n8_site/README.md b/forma_n8_site/README.md new file mode 100644 index 0000000..2435175 --- /dev/null +++ b/forma_n8_site/README.md @@ -0,0 +1,55 @@ +# Forma N8 Site + +Отдельное SSR-приложение публичного сайта Forma N8. Контент и постоянные URL не зависят от будущего дизайна. + +## Возможности + +- лента новостей, страницы статей, категорий и производителей; +- PostgreSQL со стабильными `external_id` и `slug`; +- idempotent API публикации и отдельный upload API; +- RSS 2.0, sitemap, robots.txt; +- canonical, Open Graph и JSON-LD `NewsArticle`; +- системный деплой через nginx + systemd. + +## Локальный запуск + +```bash +python -m venv .venv +.venv/bin/pip install -r requirements.txt +export DATABASE_URL=postgresql://forma_n8:forma_n8@127.0.0.1:5432/forma_n8 +PYTHONPATH=src .venv/bin/python scripts/migrate.py +PYTHONPATH=src .venv/bin/uvicorn forma_n8_site.app:app --port 8090 +``` + +## Публикационный API + +Сначала медиа загружается через `POST /api/v1/media` с заголовком `X-Forma-Token`, затем полученный относительный URL передаётся в статью. + +`POST /api/v1/articles`: + +```json +{ + "external_id": "parser:518", + "title": "Заголовок новости", + "lead": "Краткое описание", + "body": "Основной текст без HTML", + "category_name": "Снаряжение", + "category_tag": "snaryazhenie", + "producer_name": "Производитель", + "producer_tag": "producer", + "source_url": "https://example.com/source", + "status": "published", + "published_at": "2026-06-21T10:00:00+05:00", + "media": [ + {"type": "image", "url": "/media/ab/hash.jpg", "alt": "Описание изображения"} + ] +} +``` + +Повторный запрос с тем же `external_id` обновляет материал, но сохраняет первоначальные slug и дату первой публикации. Это предотвращает дубли URL и потерю поисковой истории. + +Публичное название категории берётся из таблицы `categories` по нормализованному `category_tag`. Описательные названия категорий из AI-контура не используются в URL, H1 и SEO-метаданных. + +После подключения домена меняется только `SITE_BASE_URL` в `/etc/forma-n8.env`; затем настраивается TLS и отправляются sitemap в Google Search Console и Яндекс Вебмастер. + +До подключения домена `SITE_INDEXING_ENABLED=false`: страницы получают `noindex`, а `robots.txt` запрещает обход IP-адреса. После настройки домена, HTTPS и 301-редиректа с IP значение меняется на `true`. diff --git a/forma_n8_site/db/migrations/001_initial.sql b/forma_n8_site/db/migrations/001_initial.sql new file mode 100644 index 0000000..cf55ccf --- /dev/null +++ b/forma_n8_site/db/migrations/001_initial.sql @@ -0,0 +1,46 @@ +CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS articles ( + id BIGSERIAL PRIMARY KEY, + external_id TEXT NOT NULL UNIQUE, + slug TEXT NOT NULL UNIQUE, + title TEXT NOT NULL, + lead TEXT NOT NULL DEFAULT '', + body TEXT NOT NULL, + category_name TEXT NOT NULL, + category_tag TEXT NOT NULL, + producer_name TEXT NOT NULL, + producer_tag TEXT NOT NULL, + source_url TEXT, + seo_title TEXT NOT NULL DEFAULT '', + seo_description TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'draft' CHECK (status IN ('draft', 'published')), + published_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_articles_feed + ON articles(status, published_at DESC, id DESC); +CREATE INDEX IF NOT EXISTS idx_articles_category + ON articles(category_tag, status, published_at DESC); +CREATE INDEX IF NOT EXISTS idx_articles_producer + ON articles(producer_tag, status, published_at DESC); + +CREATE TABLE IF NOT EXISTS article_media ( + id BIGSERIAL PRIMARY KEY, + article_id BIGINT NOT NULL REFERENCES articles(id) ON DELETE CASCADE, + media_type TEXT NOT NULL CHECK (media_type IN ('image', 'video')), + url TEXT NOT NULL, + alt_text TEXT NOT NULL DEFAULT '', + width INTEGER, + height INTEGER, + sort_order INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_article_media_order + ON article_media(article_id, sort_order, id); diff --git a/forma_n8_site/db/migrations/002_categories.sql b/forma_n8_site/db/migrations/002_categories.sql new file mode 100644 index 0000000..fa547b0 --- /dev/null +++ b/forma_n8_site/db/migrations/002_categories.sql @@ -0,0 +1,35 @@ +CREATE TABLE IF NOT EXISTS categories ( + tag TEXT PRIMARY KEY, + name TEXT NOT NULL, + sort_order INTEGER NOT NULL DEFAULT 100, + is_active BOOLEAN NOT NULL DEFAULT TRUE +); + +INSERT INTO categories(tag, name, sort_order) VALUES + ('odezhda', 'Одежда', 1), + ('snaryazhenie', 'Снаряжение', 2), + ('bronya', 'Броня', 3), + ('aksessuary', 'Аксессуары', 4), + ('svyaz', 'Связь', 5), + ('patchi', 'Патчи', 6), + ('navigaciya', 'Навигация', 7), + ('pricely', 'Прицелы', 8), + ('nochnye-pribory', 'Ночные приборы', 9), + ('medicina', 'Медицина', 10), + ('svet', 'Свет', 11), + ('elektronika', 'Электроника', 12), + ('nozhi', 'Ножи и мультитулы', 13), + ('oruzheynyy-obves', 'Оружейный обвес', 14), + ('praktika', 'Практика', 15), + ('teoriya', 'Теория', 16), + ('pushki', 'Оружие', 17) +ON CONFLICT(tag) DO UPDATE SET + name=EXCLUDED.name, + sort_order=EXCLUDED.sort_order; + +UPDATE articles a +SET category_name=c.name, + updated_at=NOW() +FROM categories c +WHERE a.category_tag=c.tag + AND a.category_name<>c.name; diff --git a/forma_n8_site/deploy/forma-n8.service b/forma_n8_site/deploy/forma-n8.service new file mode 100644 index 0000000..7f7d14b --- /dev/null +++ b/forma_n8_site/deploy/forma-n8.service @@ -0,0 +1,23 @@ +[Unit] +Description=Forma N8 public website +After=network.target postgresql.service +Wants=postgresql.service + +[Service] +Type=simple +User=forma-n8 +Group=forma-n8 +WorkingDirectory=/opt/forma-n8 +EnvironmentFile=/etc/forma-n8.env +Environment=PYTHONPATH=/opt/forma-n8/src +ExecStart=/opt/forma-n8/.venv/bin/uvicorn forma_n8_site.app:app --host 127.0.0.1 --port 8090 --proxy-headers +Restart=always +RestartSec=3 +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/forma-n8 + +[Install] +WantedBy=multi-user.target diff --git a/forma_n8_site/deploy/nginx.conf b/forma_n8_site/deploy/nginx.conf new file mode 100644 index 0000000..55a63f5 --- /dev/null +++ b/forma_n8_site/deploy/nginx.conf @@ -0,0 +1,31 @@ +server { + listen 80 default_server; + listen [::]:80 default_server; + server_name _; + + client_max_body_size 52m; + + location /static/ { + alias /opt/forma-n8/static/; + expires 30d; + add_header Cache-Control "public, immutable"; + access_log off; + } + + location /media/ { + alias /var/lib/forma-n8/uploads/; + expires 30d; + add_header Cache-Control "public"; + add_header X-Content-Type-Options "nosniff"; + } + + location / { + proxy_pass http://127.0.0.1:8090; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 60s; + } +} diff --git a/forma_n8_site/design/concept-home.png b/forma_n8_site/design/concept-home.png new file mode 100644 index 0000000..cb19625 Binary files /dev/null and b/forma_n8_site/design/concept-home.png differ diff --git a/forma_n8_site/ghost-theme/forma-n8/assets/css/screen.css b/forma_n8_site/ghost-theme/forma-n8/assets/css/screen.css new file mode 100644 index 0000000..fa2f0d4 --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/assets/css/screen.css @@ -0,0 +1,431 @@ +@font-face { + font-family: "Roboto Condensed"; + src: url("../fonts/roboto-condensed-cyrillic-400.woff2") format("woff2"); + font-weight: 400; + font-display: swap; +} + +@font-face { + font-family: "Roboto Condensed"; + src: url("../fonts/roboto-condensed-cyrillic-700.woff2") format("woff2"); + font-weight: 700; + font-display: swap; +} + +:root { + color-scheme: dark; + --bg: #090c0d; + --surface: #101415; + --surface-strong: #151a1b; + --text: #eceee8; + --muted: #9ba19b; + --faint: #686f6b; + --border: #29302e; + --border-strong: #3a423f; + --olive: #a5ad62; + --red: #df4a40; + --max: 1240px; + --gutter: 24px; + --font: "Roboto Condensed", "Arial Narrow", Arial, sans-serif; +} + +* { box-sizing: border-box; } + +html { background: var(--bg); scroll-behavior: smooth; } + +body { + margin: 0; + min-width: 320px; + background: var(--bg); + color: var(--text); + font-family: var(--font); + font-size: 18px; + line-height: 1.55; + letter-spacing: 0; + -webkit-font-smoothing: antialiased; +} + +body::before { + content: ""; + position: fixed; + inset: 0; + pointer-events: none; + opacity: .22; + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='120' height='120' viewBox='0 0 120 120'%3E%3Cfilter id='n'%3E%3CfeTurbulence type='fractalNoise' baseFrequency='.8' numOctaves='2' stitchTiles='stitch'/%3E%3C/filter%3E%3Crect width='100%25' height='100%25' filter='url(%23n)' opacity='.08'/%3E%3C/svg%3E"); +} + +a { color: inherit; text-decoration: none; } +a:hover { color: var(--olive); } +img, video { display: block; max-width: 100%; } +button, input { font: inherit; letter-spacing: 0; } + +:focus-visible { outline: 2px solid var(--olive); outline-offset: 4px; } + +.skip-link { + position: fixed; + z-index: 100; + top: 10px; + left: 10px; + transform: translateY(-160%); + padding: 10px 14px; + background: var(--text); + color: var(--bg); +} +.skip-link:focus { transform: none; } + +.site-header { + position: sticky; + z-index: 20; + top: 0; + border-bottom: 1px solid var(--border); + background: rgba(9, 12, 13, .96); + backdrop-filter: blur(12px); +} + +.header-inner { + width: min(var(--max), calc(100% - var(--gutter) * 2)); + min-height: 76px; + margin: 0 auto; + display: grid; + grid-template-columns: 170px 1fr auto; + align-items: center; + gap: 32px; +} + +.brand { + display: inline-flex; + width: fit-content; + align-items: center; + gap: 11px; + font-size: 15px; + font-weight: 700; + line-height: .95; +} +.brand img { width: 42px; height: 42px; border: 1px solid var(--border-strong); } + +.main-nav { display: flex; justify-content: center; align-self: stretch; gap: 38px; } +.main-nav .nav { display: contents; margin: 0; padding: 0; list-style: none; } +.main-nav .nav li { display: contents; } +.nav-dropdown { position: relative; display: flex; align-items: center; } +.nav-dropdown-toggle { display: inline-flex; height: 100%; align-items: center; gap: 8px; padding: 0; border: 0; background: transparent; color: var(--muted); cursor: pointer; } +.nav-dropdown-toggle:hover, .nav-dropdown-toggle[aria-expanded="true"] { color: var(--text); } +.nav-dropdown-icon { width: 7px; height: 7px; border-right: 1.5px solid currentColor; border-bottom: 1.5px solid currentColor; transform: translateY(-2px) rotate(45deg); transition: transform .18s ease; } +.nav-dropdown-toggle[aria-expanded="true"] .nav-dropdown-icon { transform: translateY(2px) rotate(225deg); } +.category-menu { position: absolute; z-index: 30; top: calc(100% - 1px); left: 50%; display: none; grid-template-columns: repeat(2, minmax(170px, 1fr)); width: 430px; padding: 12px; border: 1px solid var(--border-strong); background: var(--bg); transform: translateX(-50%); box-shadow: 0 18px 44px rgba(0,0,0,.38); } +.category-menu.is-open { display: grid; } +.category-menu a { min-height: 40px; padding: 9px 11px; border-bottom: 1px solid var(--border); color: var(--muted); } +.category-menu a:hover { background: var(--surface-strong); color: var(--text); } +.main-nav a { + position: relative; + display: inline-flex; + align-items: center; + color: var(--muted); + font-size: 16px; + white-space: nowrap; +} +.main-nav a:hover, .main-nav a[aria-current="page"] { color: var(--text); } +.main-nav a[aria-current="page"]::after { + content: ""; + position: absolute; + right: 0; + bottom: -1px; + left: 0; + height: 3px; + background: var(--olive); +} + +.header-actions { display: flex; gap: 8px; } +.header-actions > .social-links { margin-right: 5px; padding-right: 10px; border-right: 1px solid var(--border); } +.icon-button { + display: grid; + width: 40px; + height: 40px; + padding: 0; + place-items: center; + border: 0; + background: transparent; + color: var(--text); + cursor: pointer; +} +.icon-button:hover { background: var(--surface-strong); color: var(--olive); } +.icon-button svg { width: 23px; height: 23px; fill: none; stroke: currentColor; stroke-width: 1.8; stroke-linecap: square; } +.menu-button { display: none; } + +.search-form { + width: min(760px, calc(100% - var(--gutter) * 2)); + margin: 0 auto; + padding: 16px 0 20px; +} +.search-form[hidden] { display: none; } +.search-form label { display: block; margin-bottom: 8px; color: var(--muted); font-size: 14px; } +.search-form > div { display: grid; grid-template-columns: 1fr auto; } +.search-form input { + min-width: 0; + height: 48px; + padding: 0 16px; + border: 1px solid var(--border-strong); + border-right: 0; + border-radius: 0; + background: var(--surface); + color: var(--text); + font-size: 17px; +} +.search-form button, .read-link, .empty-state a { + border: 1px solid var(--olive); + border-radius: 0; + background: transparent; + color: var(--text); + cursor: pointer; +} +.search-form button { padding: 0 24px; } +.search-form button:hover, .read-link:hover, .empty-state a:hover { background: var(--olive); color: var(--bg); } + +.feed-shell, .article-page, .related-section, .site-footer { + position: relative; + width: min(var(--max), calc(100% - var(--gutter) * 2)); + margin: 0 auto; +} + +.page-heading { padding: 56px 0 46px; } +.page-heading h1 { + max-width: 900px; + margin: 0; + font-size: 68px; + font-weight: 700; + line-height: .98; +} +@media (min-width: 981px) { + .home-template .page-heading { + position: relative; + left: 50%; + width: min(1600px, calc(100vw - var(--gutter) * 2)); + transform: translateX(-50%); + } + .home-template .page-heading h1 { + max-width: none; + font-size: clamp(58px, 4vw, 68px); + text-align: center; + } +} +.page-heading > p { margin: 20px 0 0; color: var(--muted); } + +.lead-story { + display: grid; + grid-template-columns: minmax(0, 1.25fr) minmax(340px, .95fr); + min-height: 480px; + max-height: 680px; + border-bottom: 1px solid var(--border-strong); +} +.lead-media { min-height: 480px; max-height: 680px; overflow: hidden; background: var(--surface); } +.lead-media img, .lead-media video { width: 100%; height: 100%; object-fit: cover; transition: transform .45s ease; } +.lead-media:hover img { transform: scale(1.015); } +.lead-copy { display: flex; flex-direction: column; justify-content: center; padding: 34px 0 36px 44px; } +.lead-copy h2 { margin: 24px 0 22px; font-size: 40px; line-height: 1.14; } +.lead-copy h2 a:hover { color: var(--text); text-decoration: underline; text-decoration-color: var(--olive); text-underline-offset: 7px; } +.lead-copy > p { max-width: 580px; margin: 0; color: var(--muted); font-size: 20px; line-height: 1.5; } +.read-link { width: fit-content; margin-top: 30px; padding: 10px 15px; font-size: 15px; } +.read-link span { margin-left: 8px; } + +.article-meta { display: flex; align-items: center; flex-wrap: wrap; gap: 0; color: var(--muted); font-size: 14px; } +.article-meta > * { display: inline-flex; align-items: center; } +.article-meta > * + *::before { content: ""; width: 1px; height: 18px; margin: 0 16px; background: var(--border-strong); } +.article-meta .category-link { color: var(--olive); } +.article-meta .producer-link { color: var(--red); } + +.feed-list { width: 100%; } +.feed-list-top { border-top: 1px solid var(--border-strong); } +.feed-row { + display: grid; + grid-template-columns: 350px minmax(0, 1fr); + gap: 34px; + padding: 22px 0; + border-bottom: 1px solid var(--border); +} +.feed-row-media { aspect-ratio: 16 / 9; overflow: hidden; background: var(--surface); } +.feed-row-media img, .feed-row-media video { width: 100%; height: 100%; object-fit: cover; transition: transform .35s ease; } +.feed-row-media:hover img { transform: scale(1.02); } +.feed-row-copy { align-self: center; min-width: 0; } +.feed-row h2 { margin: 16px 0 8px; font-size: 25px; line-height: 1.2; } +.feed-row h2 a:hover { color: var(--text); text-decoration: underline; text-decoration-color: var(--olive); text-underline-offset: 5px; } +.feed-row p { max-width: 760px; margin: 0; color: var(--muted); font-size: 17px; line-height: 1.5; } +.preview-social { margin-top: 14px; } +.media-placeholder { display: grid; width: 100%; height: 100%; place-items: center; background: var(--surface); } +.media-placeholder img { width: 76px; height: 76px; opacity: .28; object-fit: contain; } +.feed-media { position: relative; } +.feed-media-feature, .feed-media-feature > img, .feed-media-feature > .media-placeholder { display: block; width: 100%; height: 100%; } +.feed-media-track { display: flex; width: 100%; height: 100%; overflow-x: auto; overflow-y: hidden; scroll-snap-type: x mandatory; scrollbar-width: none; } +.feed-media-track::-webkit-scrollbar { display: none; } +.feed-media-slide { flex: 0 0 100%; width: 100%; height: 100%; scroll-snap-align: start; } +.feed-media-slide > a, .feed-media-slide img, .feed-media-slide video { display: block; width: 100%; height: 100%; } +.feed-media-slide img, .feed-media-slide video { object-fit: cover; } +.feed-media-count { position: absolute; top: 10px; right: 10px; z-index: 2; padding: 4px 8px; border-radius: 999px; background: rgba(0,0,0,.68); color: #fff; font-size: 12px; line-height: 1; pointer-events: none; } + +.pagination { display: grid; grid-template-columns: 1fr auto 1fr; align-items: center; min-height: 90px; font-size: 15px; } +.pagination > :last-child { justify-self: end; } +.pagination span { color: var(--faint); } +.pagination .older-posts { grid-column: 3; justify-self: end; } +.pagination .newer-posts { grid-column: 1; } +.pagination .page-number { grid-column: 2; grid-row: 1; } + +.home-seo { + max-width: 960px; + margin: 54px auto 0; + padding: 40px 0 0; + border-top: 1px solid var(--border-strong); +} +.home-seo h2 { margin: 0 0 24px; font-size: 34px; line-height: 1.15; } +.home-seo p { margin: 0 0 18px; color: var(--muted); font-size: 17px; line-height: 1.7; } +.home-seo p:last-child { margin-bottom: 0; } + +.taxonomy-heading { padding-bottom: 34px; } +.taxonomy-heading > p { margin: 0 0 10px; color: var(--olive); font-size: 15px; text-transform: uppercase; } +.taxonomy-heading h1 { font-size: 56px; } + +.article-page { padding-top: 56px; } +.article-header { max-width: 960px; margin: 0 auto 42px; } +.article-breadcrumbs { display: flex; gap: 12px; margin-bottom: 28px; color: var(--muted); font-size: 14px; } +.article-breadcrumbs span { color: var(--faint); } +.article-header h1 { margin: 0; font-size: 58px; line-height: 1.06; } +.article-lead { max-width: 850px; margin: 26px 0 0; color: var(--muted); font-size: 23px; line-height: 1.45; } +.article-byline { display: flex; gap: 24px; margin-top: 28px; color: var(--muted); font-size: 14px; } +.article-byline a { color: var(--red); } +.article-media { display: grid; gap: 12px; margin-bottom: 46px; } +.article-media figure { margin: 0; background: var(--surface); } +.article-media img, .article-media video { width: 100%; max-height: 760px; object-fit: contain; } +.article-layout { display: grid; grid-template-columns: minmax(0, 760px) 260px; justify-content: center; gap: 78px; } +.article-body { font-size: 21px; line-height: 1.7; } +.article-body p { margin: 0 0 26px; } +.gh-content > * { max-width: 100%; } +.gh-content a { color: var(--olive); text-decoration: underline; text-underline-offset: 3px; } +.gh-content figure, .gh-content .kg-card { margin: 32px 0; } +.gh-content img, .gh-content video { width: 100%; height: auto; } +.gh-content blockquote { margin: 30px 0; padding-left: 24px; border-left: 3px solid var(--olive); color: var(--muted); } +.kg-width-wide, .kg-width-full { width: 100%; } +.kg-gallery-container { display: flex; flex-direction: column; gap: 10px; } +.kg-gallery-row { display: flex; gap: 10px; } +.kg-gallery-image { flex: 1 1 0; min-width: 0; margin: 0; overflow: hidden; background: var(--surface); } +.kg-gallery-image img { width: 100%; height: 100%; margin: 0; object-fit: cover; cursor: zoom-in; } +.media-lightbox { width: 100%; height: 100%; max-width: none; max-height: none; padding: 0; border: 0; background: rgba(0,0,0,.94); } +.media-lightbox::backdrop { background: rgba(0,0,0,.94); } +.media-lightbox img { width: 100%; height: 100%; object-fit: contain; } +.media-lightbox button { position: fixed; top: 18px; right: 18px; z-index: 2; width: 44px; height: 44px; border: 1px solid var(--border-strong); background: var(--bg); color: var(--text); font-size: 28px; cursor: pointer; } +.article-facts { align-self: start; border-top: 2px solid var(--olive); } +.article-facts > div { padding: 18px 0; border-bottom: 1px solid var(--border); } +.article-facts span { display: block; margin-bottom: 4px; color: var(--faint); font-size: 13px; text-transform: uppercase; } +.article-facts a { font-size: 17px; } +.article-facts .source-link { display: block; margin-top: 22px; color: var(--muted); font-size: 14px; } +.social-links { display: flex; align-items: center; flex-wrap: wrap; gap: 8px; margin-top: 13px; } +.social-link { display: inline-flex; min-height: 34px; align-items: center; gap: 7px; padding: 5px 10px; border: 1px solid var(--border); color: var(--muted); font-size: 14px; line-height: 1; transition: border-color .18s ease, color .18s ease, background .18s ease; } +.social-link:hover { border-color: var(--border-strong); background: var(--surface-strong); color: var(--text); } +.social-link svg { width: 18px; height: 18px; fill: currentColor; } +.social-link--vk:hover { color: #7ca8e8; } +.social-link--tg:hover { color: #58b9e8; } +.social-links--compact { gap: 4px; margin-top: 0; } +.social-links--compact .social-link { width: 34px; min-height: 34px; justify-content: center; padding: 0; } +.social-links--compact .social-link svg { width: 17px; height: 17px; } +.about-social { margin-top: 42px; padding-top: 24px; border-top: 1px solid var(--border-strong); } +.about-social > span { color: var(--faint); font-size: 13px; text-transform: uppercase; } +.related-section { margin-top: 90px; border-top: 1px solid var(--border-strong); } +.related-section > h2 { margin: 0; padding: 34px 0 18px; font-size: 32px; } + +.empty-state, .not-found { min-height: 56vh; display: flex; flex-direction: column; justify-content: center; align-items: flex-start; } +.empty-state h2, .not-found h1 { margin: 0; font-size: 42px; } +.empty-state p { color: var(--muted); } +.empty-state a { margin-top: 16px; padding: 10px 15px; } +.not-found > p { margin: 0; color: var(--red); font-size: 18px; } +.not-found a { margin-top: 24px; color: var(--olive); } + +.site-footer { + display: flex; + justify-content: space-between; + align-items: flex-end; + margin-top: 84px; + padding: 34px 0 46px; + border-top: 1px solid var(--border-strong); + color: var(--muted); + font-size: 14px; +} +.footer-brand { color: var(--text); font-size: 20px; font-weight: 700; } +.site-footer p { margin: 7px 0 0; } +.site-footer nav { display: flex; gap: 24px; } +.site-footer .social-links { margin-top: 18px; } + +@media (max-width: 980px) { + .header-inner { grid-template-columns: 150px 1fr auto; gap: 16px; } + .main-nav { gap: 22px; } + .main-nav a { font-size: 14px; } + .page-heading h1 { font-size: 56px; } + .lead-story { grid-template-columns: 1.1fr .9fr; min-height: 420px; } + .lead-media { min-height: 420px; } + .lead-copy { padding-left: 30px; } + .lead-copy h2 { font-size: 33px; } + .feed-row { grid-template-columns: 290px minmax(0, 1fr); gap: 26px; } + .article-layout { grid-template-columns: minmax(0, 680px) 220px; gap: 44px; } +} + +@media (max-width: 760px) { + :root { --gutter: 16px; } + body { font-size: 17px; } + .home-seo { margin-top: 38px; padding-top: 30px; } + .home-seo h2 { font-size: 28px; } + .home-seo p { font-size: 16px; line-height: 1.65; } + .header-inner { min-height: 64px; grid-template-columns: 1fr auto; } + .brand img { width: 38px; height: 38px; } + .main-nav { + position: absolute; + top: 64px; + right: 0; + left: 0; + display: none; + padding: 12px var(--gutter) 18px; + border-bottom: 1px solid var(--border); + background: var(--bg); + } + .main-nav.is-open { display: grid; } + .main-nav a { min-height: 44px; border-bottom: 1px solid var(--border); font-size: 16px; } + .nav-dropdown { display: block; } + .nav-dropdown-toggle { width: 100%; min-height: 44px; border-bottom: 1px solid var(--border); text-align: left; font-size: 16px; } + .category-menu { position: static; width: 100%; grid-template-columns: 1fr; padding: 0 0 8px 14px; border: 0; box-shadow: none; transform: none; } + .category-menu a { font-size: 15px; } + .main-nav a[aria-current="page"]::after { display: none; } + .menu-button { display: grid; } + .page-heading { padding: 38px 0 30px; } + .page-heading h1, .taxonomy-heading h1 { font-size: 44px; line-height: 1; } + .lead-story { display: block; min-height: 0; max-height: none; } + .lead-media { min-height: 0; max-height: min(72vh, 560px); aspect-ratio: 4 / 3; } + .lead-copy { padding: 25px 0 34px; } + .lead-copy h2 { margin: 20px 0 15px; font-size: 32px; } + .lead-copy > p { font-size: 18px; } + .feed-row { display: block; padding: 24px 0; } + .feed-row-media { width: 100%; max-height: min(72vh, 560px); aspect-ratio: 4 / 3; } + .feed-row-copy { padding-top: 16px; } + .feed-row h2 { margin: 10px 0 8px; font-size: 24px; } + .feed-row p { display: block; font-size: 16px; } + .preview-social { margin-top: 10px; } + .preview-social .social-link { width: 30px; min-height: 30px; } + .article-meta { font-size: 12px; } + .article-meta time { display: none; } + .article-meta > * + *::before { height: 14px; margin: 0 10px; } + .article-meta .category-link::before { display: none; } + .article-header { margin-bottom: 30px; } + .article-header h1 { font-size: 42px; } + .article-lead { font-size: 20px; } + .article-layout { display: block; } + .article-body { font-size: 19px; } + .article-facts { margin-top: 42px; } + .related-section { margin-top: 64px; } +} + +@media (max-width: 460px) { + .feed-row h2 { font-size: 22px; line-height: 1.2; } + .article-meta .producer-link::before { display: none; } + .article-meta .category-link { display: none; } + .article-header h1 { font-size: 36px; } + .article-byline { display: grid; gap: 7px; } + .site-footer { display: grid; gap: 24px; } +} + +@media (prefers-reduced-motion: reduce) { + html { scroll-behavior: auto; } + *, *::before, *::after { transition-duration: .01ms !important; } +} diff --git a/forma_n8_site/ghost-theme/forma-n8/assets/fonts/roboto-condensed-cyrillic-400.woff2 b/forma_n8_site/ghost-theme/forma-n8/assets/fonts/roboto-condensed-cyrillic-400.woff2 new file mode 100644 index 0000000..10138a0 Binary files /dev/null and b/forma_n8_site/ghost-theme/forma-n8/assets/fonts/roboto-condensed-cyrillic-400.woff2 differ diff --git a/forma_n8_site/ghost-theme/forma-n8/assets/fonts/roboto-condensed-cyrillic-700.woff2 b/forma_n8_site/ghost-theme/forma-n8/assets/fonts/roboto-condensed-cyrillic-700.woff2 new file mode 100644 index 0000000..c001da3 Binary files /dev/null and b/forma_n8_site/ghost-theme/forma-n8/assets/fonts/roboto-condensed-cyrillic-700.woff2 differ diff --git a/forma_n8_site/ghost-theme/forma-n8/assets/images/favi.png b/forma_n8_site/ghost-theme/forma-n8/assets/images/favi.png new file mode 100644 index 0000000..23a1268 Binary files /dev/null and b/forma_n8_site/ghost-theme/forma-n8/assets/images/favi.png differ diff --git a/forma_n8_site/ghost-theme/forma-n8/assets/js/site.js b/forma_n8_site/ghost-theme/forma-n8/assets/js/site.js new file mode 100644 index 0000000..af1bfa8 --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/assets/js/site.js @@ -0,0 +1,86 @@ +(() => { + document.querySelectorAll('[data-feed-media]').forEach((container) => { + const source = container.querySelector('.feed-media-source'); + const feature = container.querySelector('.feed-media-feature'); + if (!source || !feature) return; + + const videos = [...source.querySelectorAll('video')].map((item) => { + const video = item.cloneNode(true); + video.removeAttribute('poster'); + video.controls = true; + video.playsInline = true; + video.preload = 'metadata'; + return video; + }); + const images = [...source.querySelectorAll('.kg-gallery-card img, .kg-image-card img')].map((item) => { + const link = document.createElement('a'); + link.href = item.currentSrc || item.src; + link.setAttribute('aria-label', item.alt ? `Открыть: ${item.alt}` : 'Открыть изображение'); + link.appendChild(item.cloneNode(true)); + return link; + }); + + const items = videos.length ? [...videos, ...images] : [feature, ...images]; + if (items.length === 1 && !videos.length) return; + const track = document.createElement('div'); + track.className = 'feed-media-track'; + items.forEach((item) => { + const slide = document.createElement('div'); + slide.className = 'feed-media-slide'; + slide.appendChild(item); + track.appendChild(slide); + }); + container.replaceChildren(track, source); + if (items.length > 1) { + const count = document.createElement('span'); + count.className = 'feed-media-count'; + count.textContent = `1 / ${items.length}`; + container.appendChild(count); + track.addEventListener('scroll', () => { + const index = Math.round(track.scrollLeft / Math.max(track.clientWidth, 1)); + count.textContent = `${Math.min(index + 1, items.length)} / ${items.length}`; + }, { passive: true }); + } + }); +})(); + +(() => { + const button = document.querySelector('[data-menu-toggle]'); + const nav = document.querySelector('#main-nav'); + if (!button || !nav) return; + button.addEventListener('click', () => { + const open = nav.classList.toggle('is-open'); + button.setAttribute('aria-expanded', String(open)); + }); +})(); + +(() => { + const toggle = document.querySelector('[data-category-toggle]'); + const menu = document.querySelector('[data-category-menu]'); + if (!toggle || !menu) return; + const close = () => { menu.classList.remove('is-open'); toggle.setAttribute('aria-expanded', 'false'); }; + toggle.addEventListener('click', (event) => { + event.stopPropagation(); + const open = menu.classList.toggle('is-open'); + toggle.setAttribute('aria-expanded', String(open)); + }); + document.addEventListener('click', (event) => { if (!menu.contains(event.target)) close(); }); + document.addEventListener('keydown', (event) => { if (event.key === 'Escape') close(); }); +})(); + +(() => { + const images = document.querySelectorAll('.gh-content .kg-gallery-image img'); + if (!images.length || !('HTMLDialogElement' in window)) return; + const dialog = document.createElement('dialog'); + dialog.className = 'media-lightbox'; + dialog.innerHTML = ''; + const preview = dialog.querySelector('img'); + dialog.querySelector('button').addEventListener('click', () => dialog.close()); + dialog.addEventListener('click', (event) => { if (event.target === dialog) dialog.close(); }); + document.body.appendChild(dialog); + images.forEach((image) => image.addEventListener('click', () => { + preview.src = image.currentSrc || image.src; + preview.alt = image.alt || ''; + dialog.showModal(); + })); +})(); diff --git a/forma_n8_site/ghost-theme/forma-n8/default.hbs b/forma_n8_site/ghost-theme/forma-n8/default.hbs new file mode 100644 index 0000000..647acc2 --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/default.hbs @@ -0,0 +1,81 @@ + + + + + + + + + {{#is "home"}} + Forma N8 — все новости снаряжения в одном месте + {{else}} + {{#is "post"}} + {{#post}}Forma N8 — {{title}}{{/post}} + {{else}} + {{#is "tag"}} + {{#tag}}Forma N8 — {{name}}{{/tag}} + {{else}} + Forma N8 — {{meta_title}} + {{/is}} + {{/is}} + {{/is}} + + + {{ghost_head}} + + + + + + +
+
+ + + FORMA
N8 + + +
+ {{> "social-links" compact=true}} + + +
+
+
+
{{{body}}}
+ + + + + {{ghost_foot}} + + diff --git a/forma_n8_site/ghost-theme/forma-n8/error.hbs b/forma_n8_site/ghost-theme/forma-n8/error.hbs new file mode 100644 index 0000000..765c750 --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/error.hbs @@ -0,0 +1,2 @@ +{{!< default}} +

{{statusCode}}

{{message}}

Вернуться в ленту
diff --git a/forma_n8_site/ghost-theme/forma-n8/index.hbs b/forma_n8_site/ghost-theme/forma-n8/index.hbs new file mode 100644 index 0000000..bf08724 --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/index.hbs @@ -0,0 +1,38 @@ +{{!< default}} +
+

Новости тактического снаряжения и экипировки

+ {{#foreach posts}} + {{#if @first}} + {{#is "home"}} + + {{else}} + {{> "post-row"}} + {{/is}} + {{else}} + {{> "post-row"}} + {{/if}} + {{else}} +

Публикаций пока нет

Лента скоро наполнится новостями.

+ {{/foreach}} + {{> "pagination-ru"}} + {{#is "home"}} +
+

О новостной ленте Forma N8

+

Forma N8 — лента новостей о тактическом снаряжении, экипировке и EDC. Мы собираем новинки с десятков источников и публикуем в одном месте: плитоносцы и бронезащита, подсумки и разгрузочные системы, боевые брюки и рубашки, прицелы и оптика, тактические фонари, средства связи, медицина и ночные приборы.

+

В ленте — новинки российских и зарубежных производителей: Ars Arma, Wartech, SSO, Stich Profi, Giena Tactical, 5.45 Design, Red Cliff, VOIN, Sturmer, БАРС, SRVV, ANA Tactical, Mordor Tac, Centurion Gear, ENOTACTICAL, Black Buckle, MILGEAR, SKIF Armor, Atlant Armour, БронеЁж, R5 GEAR, VINDEX, JAEGER Equipment, Margul Gear, Gordeev Tactical, IGLA TAC, STRIX TAC, Колчуга, APEX, Пересвет, Гарсинг и многих других. Отслеживаем оптику Диполь, ARTELV и ARKON, гарнитуры VULCAN, Fidem и ОКБ Октава, фонари Nicron, патчи BAD GRINGO и RedWar Patch, ножи DAGGERR, Morpheus Knives и N.C.Custom.

+

Темы ленты охватывают весь комплект тактической экипировки: системы MOLLE и Molle-Minus, плиты стандартов Гранит, SAPI и ESAPI, ткани Cordura и NYCO, расцветки Multicam, Tiger Stripe и EMR. Новинки магазинов Академия Снаряжения, Semper Fi, ФИНИСТ и Империя ножей, обновления линеек, цены и отзывы пользователей — всё в одной новостной ленте о снаряжении.

+
+ {{/is}} +
diff --git a/forma_n8_site/ghost-theme/forma-n8/package.json b/forma_n8_site/ghost-theme/forma-n8/package.json new file mode 100644 index 0000000..4dc677a --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/package.json @@ -0,0 +1,18 @@ +{ + "name": "forma-n8", + "description": "Strict editorial theme for Forma N8", + "version": "1.0.0", + "author": { + "name": "Forma N8", + "email": "admin@f-n8.ru" + }, + "keywords": ["ghost-theme", "forma-n8", "news"], + "engines": { + "ghost": ">=6.0.0" + }, + "license": "UNLICENSED", + "config": { + "posts_per_page": 25, + "card_assets": true + } +} diff --git a/forma_n8_site/ghost-theme/forma-n8/page.hbs b/forma_n8_site/ghost-theme/forma-n8/page.hbs new file mode 100644 index 0000000..8c0d99d --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/page.hbs @@ -0,0 +1,2 @@ +{{!< default}} +{{#post}}
{{#if @page.show_title_and_feature_image}}

{{title}}

{{#if custom_excerpt}}

{{custom_excerpt}}

{{/if}}
{{#if feature_image}}
{{title}}
{{/if}}{{/if}}
{{content}}
{{/post}} diff --git a/forma_n8_site/ghost-theme/forma-n8/partials/pagination-ru.hbs b/forma_n8_site/ghost-theme/forma-n8/partials/pagination-ru.hbs new file mode 100644 index 0000000..4be7bf4 --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/partials/pagination-ru.hbs @@ -0,0 +1,9 @@ +{{#if pagination.pages}} + {{#match pagination.pages ">" 1}} + + {{/match}} +{{/if}} diff --git a/forma_n8_site/ghost-theme/forma-n8/partials/post-row.hbs b/forma_n8_site/ghost-theme/forma-n8/partials/post-row.hbs new file mode 100644 index 0000000..bf38e4a --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/partials/post-row.hbs @@ -0,0 +1,12 @@ + diff --git a/forma_n8_site/ghost-theme/forma-n8/partials/social-links.hbs b/forma_n8_site/ghost-theme/forma-n8/partials/social-links.hbs new file mode 100644 index 0000000..119b5f4 --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/partials/social-links.hbs @@ -0,0 +1,10 @@ + diff --git a/forma_n8_site/ghost-theme/forma-n8/post.hbs b/forma_n8_site/ghost-theme/forma-n8/post.hbs new file mode 100644 index 0000000..4108cdf --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/post.hbs @@ -0,0 +1,20 @@ +{{!< default}} +{{#post}} +
+
+
Лента{{#if primary_tag}}/{{primary_tag.name}}{{/if}}
+

{{title}}

+ {{#if custom_excerpt}}

{{custom_excerpt}}

{{/if}} +
+
+ {{#if feature_image}}
{{#if feature_image_alt}}{{feature_image_alt}}{{else}}{{title}}{{/if}}
{{/if}} +
+
{{content}}
+ +
+
+{{/post}} diff --git a/forma_n8_site/ghost-theme/forma-n8/tag.hbs b/forma_n8_site/ghost-theme/forma-n8/tag.hbs new file mode 100644 index 0000000..61e8ee2 --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/tag.hbs @@ -0,0 +1,6 @@ +{{!< default}} +
+ {{#tag}}

Тема

{{name}}

{{#if description}}

{{description}}

{{/if}}
{{/tag}} +
{{#foreach posts}}{{> "post-row"}}{{/foreach}}
+ {{> "pagination-ru"}} +
diff --git a/forma_n8_site/requirements.txt b/forma_n8_site/requirements.txt new file mode 100644 index 0000000..d02a6d6 --- /dev/null +++ b/forma_n8_site/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.115.12 +uvicorn[standard]==0.34.2 +asyncpg==0.30.0 +jinja2==3.1.6 +python-multipart==0.0.20 +pydantic-settings==2.9.1 diff --git a/forma_n8_site/scripts/migrate.py b/forma_n8_site/scripts/migrate.py new file mode 100644 index 0000000..a1ea41f --- /dev/null +++ b/forma_n8_site/scripts/migrate.py @@ -0,0 +1,28 @@ +import asyncio +import os +from pathlib import Path + +import asyncpg + + +async def main() -> None: + root = Path(__file__).resolve().parents[1] + conn = await asyncpg.connect(os.environ["DATABASE_URL"]) + try: + await conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW())" + ) + for path in sorted((root / "db" / "migrations").glob("*.sql")): + exists = await conn.fetchval("SELECT 1 FROM schema_migrations WHERE version=$1", path.name) + if exists: + continue + async with conn.transaction(): + await conn.execute(path.read_text(encoding="utf-8")) + await conn.execute("INSERT INTO schema_migrations(version) VALUES($1)", path.name) + print(f"applied {path.name}") + finally: + await conn.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/forma_n8_site/src/forma_n8_site/__init__.py b/forma_n8_site/src/forma_n8_site/__init__.py new file mode 100644 index 0000000..805a5c2 --- /dev/null +++ b/forma_n8_site/src/forma_n8_site/__init__.py @@ -0,0 +1 @@ +"""Forma N8 public website.""" diff --git a/forma_n8_site/src/forma_n8_site/app.py b/forma_n8_site/src/forma_n8_site/app.py new file mode 100644 index 0000000..069ef3f --- /dev/null +++ b/forma_n8_site/src/forma_n8_site/app.py @@ -0,0 +1,448 @@ +import hashlib +import hmac +import json +from contextlib import asynccontextmanager +from datetime import datetime, timezone +from pathlib import Path +from zoneinfo import ZoneInfo +from xml.etree import ElementTree as ET + +from fastapi import Depends, FastAPI, File, Header, HTTPException, Query, Request, UploadFile +from fastapi.responses import FileResponse, HTMLResponse, PlainTextResponse, Response +from fastapi.staticfiles import StaticFiles +from fastapi.templating import Jinja2Templates + +from .config import settings +from .db import connect, disconnect, get_pool +from .schemas import ArticleUpsert +from .text import excerpt, paragraphs, slugify + + +ROOT = Path(__file__).resolve().parents[2] +STATIC_DIR = ROOT / "static" +TEMPLATE_DIR = ROOT / "templates" +ALLOWED_UPLOADS = { + "image/jpeg": ".jpg", + "image/png": ".png", + "image/webp": ".webp", + "image/gif": ".gif", + "video/mp4": ".mp4", + "video/webm": ".webm", +} + + +@asynccontextmanager +async def lifespan(_: FastAPI): + settings.upload_dir.mkdir(parents=True, exist_ok=True) + await connect() + yield + await disconnect() + + +app = FastAPI(title=settings.site_name, docs_url=None, redoc_url=None, lifespan=lifespan) +app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static") +app.mount("/media", StaticFiles(directory=settings.upload_dir, check_dir=False), name="media") +templates = Jinja2Templates(directory=TEMPLATE_DIR) +templates.env.filters["paragraphs"] = paragraphs +templates.env.filters["excerpt"] = excerpt +LOCAL_TZ = ZoneInfo("Asia/Yekaterinburg") +templates.env.filters["ekb_date"] = lambda value: value.astimezone(LOCAL_TZ).strftime("%d.%m.%Y") if value else "" +templates.env.filters["ekb_time"] = lambda value: value.astimezone(LOCAL_TZ).strftime("%H:%M") if value else "" + + +def site_url(path: str = "") -> str: + return f"{settings.base_url}{path if path.startswith('/') else '/' + path if path else ''}" + + +def article_path(article: dict) -> str: + return f"/news/{article['slug']}" + + +templates.env.globals.update( + site_name=settings.site_name, + site_tagline=settings.site_tagline, + site_url=site_url, + article_path=article_path, +) + + +@app.middleware("http") +async def security_headers(request: Request, call_next): + response = await call_next(request) + response.headers.setdefault("X-Content-Type-Options", "nosniff") + response.headers.setdefault("X-Frame-Options", "DENY") + response.headers.setdefault("Referrer-Policy", "strict-origin-when-cross-origin") + response.headers.setdefault("Permissions-Policy", "camera=(), microphone=(), geolocation=()") + return response + + +def base_context(request: Request, **extra): + return { + "request": request, + "current_path": request.url.path, + "canonical_url": site_url(request.url.path), + "meta_title": f"{settings.site_tagline} — {settings.site_name}", + "meta_description": "Новости производителей тактического снаряжения, одежды, брони и аксессуаров.", + "noindex": not settings.site_indexing_enabled or bool(request.query_params.get("q")), + **extra, + } + + +ARTICLE_COLUMNS = """ + a.*, + COALESCE( + jsonb_agg( + jsonb_build_object( + 'type', m.media_type, + 'url', m.url, + 'alt', m.alt_text, + 'width', m.width, + 'height', m.height + ) ORDER BY m.sort_order, m.id + ) FILTER (WHERE m.id IS NOT NULL), + '[]'::jsonb + ) AS media +""" + + +def normalize_article(row) -> dict: + article = dict(row) + media = article.get("media") or [] + if isinstance(media, str): + try: + media = json.loads(media) + except json.JSONDecodeError: + media = [] + article["media"] = media if isinstance(media, list) else [] + return article + + +async def fetch_feed(*, page: int, q: str = "", category: str = "", producer: str = "", page_size: int | None = None): + pool = await get_pool() + where = ["a.status='published'", "a.published_at <= NOW()"] + args: list[object] = [] + if q: + args.append(f"%{q}%") + where.append(f"(a.title ILIKE ${len(args)} OR a.lead ILIKE ${len(args)} OR a.body ILIKE ${len(args)} OR a.producer_name ILIKE ${len(args)})") + if category: + args.append(category) + where.append(f"a.category_tag=${len(args)}") + if producer: + args.append(producer) + where.append(f"a.producer_tag=${len(args)}") + where_sql = " AND ".join(where) + total = int(await pool.fetchval(f"SELECT COUNT(*) FROM articles a WHERE {where_sql}", *args) or 0) + limit = max(1, min(50, page_size or settings.page_size)) + offset = (page - 1) * limit + rows = await pool.fetch( + f""" + SELECT {ARTICLE_COLUMNS} + FROM articles a + LEFT JOIN article_media m ON m.article_id=a.id + WHERE {where_sql} + GROUP BY a.id + ORDER BY a.published_at DESC, a.id DESC + LIMIT ${len(args) + 1} OFFSET ${len(args) + 2} + """, + *args, + limit, + offset, + ) + return [normalize_article(row) for row in rows], total, limit + + +@app.get("/", response_class=HTMLResponse) +async def feed(request: Request, page: int = Query(1, ge=1), q: str = Query("", max_length=100)): + articles, total, limit = await fetch_feed(page=page, q=q.strip()) + pages = max(1, (total + limit - 1) // limit) + canonical = site_url("/") if page == 1 else site_url(f"/?page={page}") + return templates.TemplateResponse( + "feed.html", + base_context( + request, + articles=articles, + page=page, + pages=pages, + total=total, + q=q.strip(), + canonical_url=canonical, + ), + ) + + +@app.get("/category/{tag}", response_class=HTMLResponse) +async def category_feed(request: Request, tag: str, page: int = Query(1, ge=1)): + pool = await get_pool() + category_name = await pool.fetchval("SELECT name FROM categories WHERE tag=$1 AND is_active=TRUE", tag) + if not category_name: + raise HTTPException(status_code=404) + articles, total, limit = await fetch_feed(page=page, category=tag) + name = str(category_name) + pages = max(1, (total + limit - 1) // limit) + return templates.TemplateResponse( + "taxonomy.html", + base_context( + request, + articles=articles, + page=page, + pages=pages, + heading=name, + taxonomy_label="Категория", + canonical_url=site_url(f"/category/{tag}" + (f"?page={page}" if page > 1 else "")), + meta_title=f"{name}: новости — {settings.site_name}", + meta_description=f"Последние новости в категории «{name}»: производители, новинки и обзоры снаряжения.", + noindex=total == 0, + ), + ) + + +@app.get("/brand/{tag}", response_class=HTMLResponse) +async def producer_feed(request: Request, tag: str, page: int = Query(1, ge=1)): + articles, total, limit = await fetch_feed(page=page, producer=tag) + if not articles and page == 1: + raise HTTPException(status_code=404) + name = articles[0]["producer_name"] if articles else tag + pages = max(1, (total + limit - 1) // limit) + return templates.TemplateResponse( + "taxonomy.html", + base_context( + request, + articles=articles, + page=page, + pages=pages, + heading=name, + taxonomy_label="Производитель", + canonical_url=site_url(f"/brand/{tag}" + (f"?page={page}" if page > 1 else "")), + meta_title=f"{name}: новости и новинки — {settings.site_name}", + meta_description=f"Новости и новинки производителя {name}: снаряжение, одежда и экипировка.", + ), + ) + + +@app.get("/news/{slug}", response_class=HTMLResponse) +async def article(request: Request, slug: str): + pool = await get_pool() + row = await pool.fetchrow( + f""" + SELECT {ARTICLE_COLUMNS} + FROM articles a + LEFT JOIN article_media m ON m.article_id=a.id + WHERE a.slug=$1 AND a.status='published' AND a.published_at <= NOW() + GROUP BY a.id + """, + slug, + ) + if not row: + raise HTTPException(status_code=404) + item = normalize_article(row) + related = await pool.fetch( + f""" + SELECT {ARTICLE_COLUMNS} + FROM articles a + LEFT JOIN article_media m ON m.article_id=a.id + WHERE a.status='published' AND a.id<>$1 + AND (a.category_tag=$2 OR a.producer_tag=$3) + GROUP BY a.id + ORDER BY (a.producer_tag=$3) DESC, a.published_at DESC + LIMIT 3 + """, + item["id"], + item["category_tag"], + item["producer_tag"], + ) + canonical = site_url(article_path(item)) + image = item["media"][0]["url"] if item["media"] else site_url("/static/favi.png") + if image.startswith("/"): + image = site_url(image) + json_ld = { + "@context": "https://schema.org", + "@type": "NewsArticle", + "headline": item["title"], + "description": item["seo_description"] or item["lead"] or excerpt(item["body"]), + "datePublished": item["published_at"].isoformat(), + "dateModified": item["updated_at"].isoformat(), + "mainEntityOfPage": canonical, + "image": [image], + "author": {"@type": "Organization", "name": settings.site_name}, + "publisher": { + "@type": "Organization", + "name": settings.site_name, + "logo": {"@type": "ImageObject", "url": site_url("/static/favi.png")}, + }, + } + return templates.TemplateResponse( + "article.html", + base_context( + request, + article=item, + related=[normalize_article(x) for x in related], + canonical_url=canonical, + meta_title=item["seo_title"] or f"{item['title']} — {settings.site_name}", + meta_description=item["seo_description"] or item["lead"] or excerpt(item["body"]), + og_image=image, + json_ld=json.dumps(json_ld, ensure_ascii=False), + ), + ) + + +def require_api_token(x_forma_token: str = Header(default="")) -> None: + if not settings.publish_api_token or not hmac.compare_digest(x_forma_token, settings.publish_api_token): + raise HTTPException(status_code=401, detail="invalid token") + + +@app.post("/api/v1/media", dependencies=[Depends(require_api_token)]) +async def upload_media(file: UploadFile = File(...)): + content_type = (file.content_type or "").split(";", 1)[0].lower() + if content_type not in ALLOWED_UPLOADS: + raise HTTPException(status_code=415, detail="unsupported media type") + data = await file.read(50 * 1024 * 1024 + 1) + if len(data) > 50 * 1024 * 1024: + raise HTTPException(status_code=413, detail="file too large") + digest = hashlib.sha256(data).hexdigest() + suffix = ALLOWED_UPLOADS[content_type] + target = settings.upload_dir / digest[:2] / f"{digest}{suffix}" + target.parent.mkdir(parents=True, exist_ok=True) + if not target.exists(): + target.write_bytes(data) + return {"url": f"/media/{digest[:2]}/{target.name}", "sha256": digest, "content_type": content_type} + + +@app.post("/api/v1/articles", dependencies=[Depends(require_api_token)]) +async def upsert_article(payload: ArticleUpsert): + pool = await get_pool() + existing = await pool.fetchrow("SELECT id, slug, published_at FROM articles WHERE external_id=$1", payload.external_id) + requested_slug = slugify(payload.slug or payload.title) + stable_slug = existing["slug"] if existing else f"{requested_slug}-{slugify(payload.external_id)[-24:]}" + published_at = payload.published_at or datetime.now(timezone.utc) + category_tag = slugify(payload.category_tag) + category_name = await pool.fetchval("SELECT name FROM categories WHERE tag=$1 AND is_active=TRUE", category_tag) + category_name = str(category_name or payload.category_name).strip() + async with pool.acquire() as conn: + async with conn.transaction(): + row = await conn.fetchrow( + """ + INSERT INTO articles( + external_id, slug, title, lead, body, category_name, category_tag, + producer_name, producer_tag, source_url, seo_title, seo_description, + status, published_at + ) + VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) + ON CONFLICT(external_id) DO UPDATE SET + title=EXCLUDED.title, + lead=EXCLUDED.lead, + body=EXCLUDED.body, + category_name=EXCLUDED.category_name, + category_tag=EXCLUDED.category_tag, + producer_name=EXCLUDED.producer_name, + producer_tag=EXCLUDED.producer_tag, + source_url=EXCLUDED.source_url, + seo_title=EXCLUDED.seo_title, + seo_description=EXCLUDED.seo_description, + status=EXCLUDED.status, + published_at=COALESCE(articles.published_at, EXCLUDED.published_at), + updated_at=NOW() + RETURNING id, slug, status, published_at, updated_at + """, + payload.external_id, + stable_slug, + payload.title.strip(), + payload.lead.strip(), + payload.body.strip(), + category_name, + category_tag, + payload.producer_name.strip(), + slugify(payload.producer_tag), + str(payload.source_url) if payload.source_url else None, + (payload.seo_title or payload.title).strip(), + (payload.seo_description or payload.lead or excerpt(payload.body)).strip(), + payload.status, + published_at, + ) + await conn.execute("DELETE FROM article_media WHERE article_id=$1", row["id"]) + for index, media in enumerate(payload.media): + await conn.execute( + """ + INSERT INTO article_media(article_id, media_type, url, alt_text, width, height, sort_order) + VALUES($1,$2,$3,$4,$5,$6,$7) + """, + row["id"], media.type, media.url, media.alt or payload.title, + media.width, media.height, media.sort_order or index, + ) + return { + "id": row["id"], + "external_id": payload.external_id, + "status": row["status"], + "url": site_url(f"/news/{row['slug']}") if row["status"] == "published" else None, + "published_at": row["published_at"], + "updated_at": row["updated_at"], + } + + +@app.get("/rss.xml") +async def rss(): + articles, _, _ = await fetch_feed(page=1, page_size=50) + root = ET.Element("rss", {"version": "2.0", "xmlns:media": "http://search.yahoo.com/mrss/"}) + channel = ET.SubElement(root, "channel") + ET.SubElement(channel, "title").text = f"{settings.site_name} — {settings.site_tagline}" + ET.SubElement(channel, "link").text = settings.base_url + ET.SubElement(channel, "description").text = "Новости производителей тактического снаряжения и экипировки." + ET.SubElement(channel, "language").text = "ru" + for item in articles[:50]: + node = ET.SubElement(channel, "item") + url = site_url(article_path(item)) + ET.SubElement(node, "title").text = item["title"] + ET.SubElement(node, "link").text = url + ET.SubElement(node, "guid", {"isPermaLink": "true"}).text = url + ET.SubElement(node, "description").text = item["lead"] or excerpt(item["body"]) + ET.SubElement(node, "pubDate").text = item["published_at"].strftime("%a, %d %b %Y %H:%M:%S +0000") + ET.SubElement(node, "category").text = item["category_name"] + if item["media"]: + ET.SubElement(node, "media:content", {"url": site_url(item["media"][0]["url"]) if item["media"][0]["url"].startswith("/") else item["media"][0]["url"], "medium": "image"}) + return Response(ET.tostring(root, encoding="utf-8", xml_declaration=True), media_type="application/rss+xml") + + +@app.get("/sitemap.xml") +async def sitemap(): + pool = await get_pool() + rows = await pool.fetch("SELECT slug, updated_at FROM articles WHERE status='published' ORDER BY id") + categories = await pool.fetch("SELECT category_tag AS tag, MAX(updated_at) AS updated_at FROM articles WHERE status='published' GROUP BY category_tag") + producers = await pool.fetch("SELECT producer_tag AS tag, MAX(updated_at) AS updated_at FROM articles WHERE status='published' GROUP BY producer_tag") + root = ET.Element("urlset", {"xmlns": "http://www.sitemaps.org/schemas/sitemap/0.9"}) + entries = [(site_url("/"), None)] + entries += [(site_url(f"/category/{row['tag']}"), row["updated_at"]) for row in categories] + entries += [(site_url(f"/brand/{row['tag']}"), row["updated_at"]) for row in producers] + entries += [(site_url(f"/news/{row['slug']}"), row["updated_at"]) for row in rows] + for loc, updated in entries: + node = ET.SubElement(root, "url") + ET.SubElement(node, "loc").text = loc + if updated: + ET.SubElement(node, "lastmod").text = updated.date().isoformat() + return Response(ET.tostring(root, encoding="utf-8", xml_declaration=True), media_type="application/xml") + + +@app.get("/robots.txt", response_class=PlainTextResponse) +async def robots(): + if not settings.site_indexing_enabled: + return "User-agent: *\nDisallow: /\n" + return f"User-agent: *\nAllow: /\nDisallow: /api/\nSitemap: {site_url('/sitemap.xml')}\n" + + +@app.get("/favicon.ico", include_in_schema=False) +async def favicon(): + return FileResponse(STATIC_DIR / "favi.png", media_type="image/png") + + +@app.get("/health") +async def health(): + pool = await get_pool() + await pool.fetchval("SELECT 1") + return {"status": "ok"} + + +@app.exception_handler(404) +async def not_found(request: Request, _): + return templates.TemplateResponse( + "404.html", + base_context(request, meta_title=f"Страница не найдена — {settings.site_name}", noindex=True), + status_code=404, + ) diff --git a/forma_n8_site/src/forma_n8_site/config.py b/forma_n8_site/src/forma_n8_site/config.py new file mode 100644 index 0000000..cbe716b --- /dev/null +++ b/forma_n8_site/src/forma_n8_site/config.py @@ -0,0 +1,23 @@ +from pathlib import Path + +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + database_url: str = "postgresql://forma_n8:forma_n8@127.0.0.1:5432/forma_n8" + site_base_url: str = "http://127.0.0.1:8090" + site_name: str = "Forma N8" + site_tagline: str = "Новости снаряжения" + site_indexing_enabled: bool = False + publish_api_token: str = "" + upload_dir: Path = Path("/var/lib/forma-n8/uploads") + page_size: int = 12 + + model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") + + @property + def base_url(self) -> str: + return self.site_base_url.rstrip("/") + + +settings = Settings() diff --git a/forma_n8_site/src/forma_n8_site/db.py b/forma_n8_site/src/forma_n8_site/db.py new file mode 100644 index 0000000..696ba9d --- /dev/null +++ b/forma_n8_site/src/forma_n8_site/db.py @@ -0,0 +1,24 @@ +import asyncpg + +from .config import settings + + +pool: asyncpg.Pool | None = None + + +async def connect() -> asyncpg.Pool: + global pool + if pool is None: + pool = await asyncpg.create_pool(settings.database_url, min_size=1, max_size=10, command_timeout=30) + return pool + + +async def disconnect() -> None: + global pool + if pool is not None: + await pool.close() + pool = None + + +async def get_pool() -> asyncpg.Pool: + return pool or await connect() diff --git a/forma_n8_site/src/forma_n8_site/schemas.py b/forma_n8_site/src/forma_n8_site/schemas.py new file mode 100644 index 0000000..d9945b4 --- /dev/null +++ b/forma_n8_site/src/forma_n8_site/schemas.py @@ -0,0 +1,31 @@ +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel, Field, HttpUrl + + +class MediaInput(BaseModel): + type: Literal["image", "video"] = "image" + url: str = Field(min_length=1, max_length=2000) + alt: str = Field(default="", max_length=500) + width: int | None = Field(default=None, ge=1) + height: int | None = Field(default=None, ge=1) + sort_order: int = Field(default=0, ge=0) + + +class ArticleUpsert(BaseModel): + external_id: str = Field(min_length=1, max_length=160) + slug: str | None = Field(default=None, max_length=220) + title: str = Field(min_length=5, max_length=300) + lead: str = Field(default="", max_length=1000) + body: str = Field(min_length=20, max_length=50000) + category_name: str = Field(min_length=1, max_length=120) + category_tag: str = Field(min_length=1, max_length=120) + producer_name: str = Field(min_length=1, max_length=200) + producer_tag: str = Field(min_length=1, max_length=160) + source_url: HttpUrl | None = None + seo_title: str | None = Field(default=None, max_length=300) + seo_description: str | None = Field(default=None, max_length=500) + status: Literal["draft", "published"] = "published" + published_at: datetime | None = None + media: list[MediaInput] = Field(default_factory=list, max_length=20) diff --git a/forma_n8_site/src/forma_n8_site/text.py b/forma_n8_site/src/forma_n8_site/text.py new file mode 100644 index 0000000..0dcbe29 --- /dev/null +++ b/forma_n8_site/src/forma_n8_site/text.py @@ -0,0 +1,37 @@ +import re +from html import escape + +from markupsafe import Markup + + +TRANSLIT = str.maketrans( + { + "а": "a", "б": "b", "в": "v", "г": "g", "д": "d", "е": "e", "ё": "e", + "ж": "zh", "з": "z", "и": "i", "й": "y", "к": "k", "л": "l", "м": "m", + "н": "n", "о": "o", "п": "p", "р": "r", "с": "s", "т": "t", "у": "u", + "ф": "f", "х": "h", "ц": "c", "ч": "ch", "ш": "sh", "щ": "sch", + "ъ": "", "ы": "y", "ь": "", "э": "e", "ю": "yu", "я": "ya", + } +) + + +def slugify(value: str) -> str: + value = value.strip().lower().translate(TRANSLIT) + value = re.sub(r"[^a-z0-9]+", "-", value).strip("-") + return value[:180] or "article" + + +def paragraphs(value: str) -> Markup: + blocks = [] + for part in re.split(r"\n\s*\n", value.strip()): + lines = "
".join(escape(line.strip()) for line in part.splitlines() if line.strip()) + if lines: + blocks.append(f"

{lines}

") + return Markup("".join(blocks)) + + +def excerpt(value: str, limit: int = 180) -> str: + compact = re.sub(r"\s+", " ", value or "").strip() + if len(compact) <= limit: + return compact + return compact[: limit - 1].rsplit(" ", 1)[0] + "…" diff --git a/forma_n8_site/static/favi.png b/forma_n8_site/static/favi.png new file mode 100644 index 0000000..23a1268 Binary files /dev/null and b/forma_n8_site/static/favi.png differ diff --git a/forma_n8_site/static/fonts/roboto-condensed-cyrillic-400.woff2 b/forma_n8_site/static/fonts/roboto-condensed-cyrillic-400.woff2 new file mode 100644 index 0000000..10138a0 Binary files /dev/null and b/forma_n8_site/static/fonts/roboto-condensed-cyrillic-400.woff2 differ diff --git a/forma_n8_site/static/fonts/roboto-condensed-cyrillic-700.woff2 b/forma_n8_site/static/fonts/roboto-condensed-cyrillic-700.woff2 new file mode 100644 index 0000000..c001da3 Binary files /dev/null and b/forma_n8_site/static/fonts/roboto-condensed-cyrillic-700.woff2 differ diff --git a/forma_n8_site/static/site.css b/forma_n8_site/static/site.css new file mode 100644 index 0000000..9c590af --- /dev/null +++ b/forma_n8_site/static/site.css @@ -0,0 +1,348 @@ +@font-face { + font-family: "Roboto Condensed"; + src: url("/static/fonts/roboto-condensed-cyrillic-400.woff2") format("woff2"); + font-weight: 400; + font-display: swap; +} + +@font-face { + font-family: "Roboto Condensed"; + src: url("/static/fonts/roboto-condensed-cyrillic-700.woff2") format("woff2"); + font-weight: 700; + font-display: swap; +} + +:root { + color-scheme: dark; + --bg: #090c0d; + --surface: #101415; + --surface-strong: #151a1b; + --text: #eceee8; + --muted: #9ba19b; + --faint: #686f6b; + --border: #29302e; + --border-strong: #3a423f; + --olive: #a5ad62; + --red: #df4a40; + --max: 1240px; + --gutter: 24px; + --font: "Roboto Condensed", "Arial Narrow", Arial, sans-serif; +} + +* { box-sizing: border-box; } + +html { background: var(--bg); scroll-behavior: smooth; } + +body { + margin: 0; + min-width: 320px; + background: var(--bg); + color: var(--text); + font-family: var(--font); + font-size: 18px; + line-height: 1.55; + letter-spacing: 0; + -webkit-font-smoothing: antialiased; +} + +body::before { + content: ""; + position: fixed; + inset: 0; + pointer-events: none; + opacity: .22; + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='120' height='120' viewBox='0 0 120 120'%3E%3Cfilter id='n'%3E%3CfeTurbulence type='fractalNoise' baseFrequency='.8' numOctaves='2' stitchTiles='stitch'/%3E%3C/filter%3E%3Crect width='100%25' height='100%25' filter='url(%23n)' opacity='.08'/%3E%3C/svg%3E"); +} + +a { color: inherit; text-decoration: none; } +a:hover { color: var(--olive); } +img, video { display: block; max-width: 100%; } +button, input { font: inherit; letter-spacing: 0; } + +:focus-visible { outline: 2px solid var(--olive); outline-offset: 4px; } + +.skip-link { + position: fixed; + z-index: 100; + top: 10px; + left: 10px; + transform: translateY(-160%); + padding: 10px 14px; + background: var(--text); + color: var(--bg); +} +.skip-link:focus { transform: none; } + +.site-header { + position: sticky; + z-index: 20; + top: 0; + border-bottom: 1px solid var(--border); + background: rgba(9, 12, 13, .96); + backdrop-filter: blur(12px); +} + +.header-inner { + width: min(var(--max), calc(100% - var(--gutter) * 2)); + min-height: 76px; + margin: 0 auto; + display: grid; + grid-template-columns: 170px 1fr auto; + align-items: center; + gap: 32px; +} + +.brand { + display: inline-flex; + width: fit-content; + align-items: center; + gap: 11px; + font-size: 15px; + font-weight: 700; + line-height: .95; +} +.brand img { width: 42px; height: 42px; border: 1px solid var(--border-strong); } + +.main-nav { display: flex; justify-content: center; align-self: stretch; gap: 38px; } +.main-nav a { + position: relative; + display: inline-flex; + align-items: center; + color: var(--muted); + font-size: 16px; + white-space: nowrap; +} +.main-nav a:hover, .main-nav a[aria-current="page"] { color: var(--text); } +.main-nav a[aria-current="page"]::after { + content: ""; + position: absolute; + right: 0; + bottom: -1px; + left: 0; + height: 3px; + background: var(--olive); +} + +.header-actions { display: flex; gap: 8px; } +.icon-button { + display: grid; + width: 40px; + height: 40px; + padding: 0; + place-items: center; + border: 0; + background: transparent; + color: var(--text); + cursor: pointer; +} +.icon-button:hover { background: var(--surface-strong); color: var(--olive); } +.icon-button svg { width: 23px; height: 23px; fill: none; stroke: currentColor; stroke-width: 1.8; stroke-linecap: square; } +.menu-button { display: none; } + +.search-form { + width: min(760px, calc(100% - var(--gutter) * 2)); + margin: 0 auto; + padding: 16px 0 20px; +} +.search-form[hidden] { display: none; } +.search-form label { display: block; margin-bottom: 8px; color: var(--muted); font-size: 14px; } +.search-form > div { display: grid; grid-template-columns: 1fr auto; } +.search-form input { + min-width: 0; + height: 48px; + padding: 0 16px; + border: 1px solid var(--border-strong); + border-right: 0; + border-radius: 0; + background: var(--surface); + color: var(--text); + font-size: 17px; +} +.search-form button, .read-link, .empty-state a { + border: 1px solid var(--olive); + border-radius: 0; + background: transparent; + color: var(--text); + cursor: pointer; +} +.search-form button { padding: 0 24px; } +.search-form button:hover, .read-link:hover, .empty-state a:hover { background: var(--olive); color: var(--bg); } + +.feed-shell, .article-page, .related-section, .site-footer { + position: relative; + width: min(var(--max), calc(100% - var(--gutter) * 2)); + margin: 0 auto; +} + +.page-heading { padding: 56px 0 46px; } +.page-heading h1 { + max-width: 900px; + margin: 0; + font-size: 68px; + font-weight: 700; + line-height: .98; +} +.page-heading > p { margin: 20px 0 0; color: var(--muted); } + +.lead-story { + display: grid; + grid-template-columns: minmax(0, 1.25fr) minmax(340px, .95fr); + min-height: 480px; + border-bottom: 1px solid var(--border-strong); +} +.lead-media { min-height: 480px; overflow: hidden; background: var(--surface); } +.lead-media img, .lead-media video { width: 100%; height: 100%; object-fit: cover; transition: transform .45s ease; } +.lead-media:hover img { transform: scale(1.015); } +.lead-copy { display: flex; flex-direction: column; justify-content: center; padding: 34px 0 36px 44px; } +.lead-copy h2 { margin: 24px 0 22px; font-size: 40px; line-height: 1.14; } +.lead-copy h2 a:hover { color: var(--text); text-decoration: underline; text-decoration-color: var(--olive); text-underline-offset: 7px; } +.lead-copy > p { max-width: 580px; margin: 0; color: var(--muted); font-size: 20px; line-height: 1.5; } +.read-link { width: fit-content; margin-top: 30px; padding: 10px 15px; font-size: 15px; } +.read-link span { margin-left: 8px; } + +.article-meta { display: flex; align-items: center; flex-wrap: wrap; gap: 0; color: var(--muted); font-size: 14px; } +.article-meta > * { display: inline-flex; align-items: center; } +.article-meta > * + *::before { content: ""; width: 1px; height: 18px; margin: 0 16px; background: var(--border-strong); } +.article-meta .category-link { color: var(--olive); } +.article-meta .producer-link { color: var(--red); } + +.feed-list { width: 100%; } +.feed-list-top { border-top: 1px solid var(--border-strong); } +.feed-row { + display: grid; + grid-template-columns: 350px minmax(0, 1fr); + gap: 34px; + padding: 22px 0; + border-bottom: 1px solid var(--border); +} +.feed-row-media { aspect-ratio: 16 / 9; overflow: hidden; background: var(--surface); } +.feed-row-media img, .feed-row-media video { width: 100%; height: 100%; object-fit: cover; transition: transform .35s ease; } +.feed-row-media:hover img { transform: scale(1.02); } +.feed-row-copy { align-self: center; min-width: 0; } +.feed-row h2 { margin: 16px 0 8px; font-size: 25px; line-height: 1.2; } +.feed-row h2 a:hover { color: var(--text); text-decoration: underline; text-decoration-color: var(--olive); text-underline-offset: 5px; } +.feed-row p { max-width: 760px; margin: 0; color: var(--muted); font-size: 17px; line-height: 1.5; } +.media-placeholder { display: grid; width: 100%; height: 100%; place-items: center; background: var(--surface); } +.media-placeholder img { width: 76px; height: 76px; opacity: .28; object-fit: contain; } + +.pagination { display: grid; grid-template-columns: 1fr auto 1fr; align-items: center; min-height: 90px; font-size: 15px; } +.pagination > :last-child { justify-self: end; } +.pagination span { color: var(--faint); } + +.taxonomy-heading { padding-bottom: 34px; } +.taxonomy-heading > p { margin: 0 0 10px; color: var(--olive); font-size: 15px; text-transform: uppercase; } +.taxonomy-heading h1 { font-size: 56px; } + +.article-page { padding-top: 56px; } +.article-header { max-width: 960px; margin: 0 auto 42px; } +.article-breadcrumbs { display: flex; gap: 12px; margin-bottom: 28px; color: var(--muted); font-size: 14px; } +.article-breadcrumbs span { color: var(--faint); } +.article-header h1 { margin: 0; font-size: 58px; line-height: 1.06; } +.article-lead { max-width: 850px; margin: 26px 0 0; color: var(--muted); font-size: 23px; line-height: 1.45; } +.article-byline { display: flex; gap: 24px; margin-top: 28px; color: var(--muted); font-size: 14px; } +.article-byline a { color: var(--red); } +.article-media { display: grid; gap: 12px; margin-bottom: 46px; } +.article-media figure { margin: 0; background: var(--surface); } +.article-media img, .article-media video { width: 100%; max-height: 760px; object-fit: contain; } +.article-layout { display: grid; grid-template-columns: minmax(0, 760px) 260px; justify-content: center; gap: 78px; } +.article-body { font-size: 21px; line-height: 1.7; } +.article-body p { margin: 0 0 26px; } +.article-facts { align-self: start; border-top: 2px solid var(--olive); } +.article-facts > div { padding: 18px 0; border-bottom: 1px solid var(--border); } +.article-facts span { display: block; margin-bottom: 4px; color: var(--faint); font-size: 13px; text-transform: uppercase; } +.article-facts a { font-size: 17px; } +.article-facts .source-link { display: block; margin-top: 22px; color: var(--muted); font-size: 14px; } +.related-section { margin-top: 90px; border-top: 1px solid var(--border-strong); } +.related-section > h2 { margin: 0; padding: 34px 0 18px; font-size: 32px; } + +.empty-state, .not-found { min-height: 56vh; display: flex; flex-direction: column; justify-content: center; align-items: flex-start; } +.empty-state h2, .not-found h1 { margin: 0; font-size: 42px; } +.empty-state p { color: var(--muted); } +.empty-state a { margin-top: 16px; padding: 10px 15px; } +.not-found > p { margin: 0; color: var(--red); font-size: 18px; } +.not-found a { margin-top: 24px; color: var(--olive); } + +.site-footer { + display: flex; + justify-content: space-between; + align-items: flex-end; + margin-top: 84px; + padding: 34px 0 46px; + border-top: 1px solid var(--border-strong); + color: var(--muted); + font-size: 14px; +} +.footer-brand { color: var(--text); font-size: 20px; font-weight: 700; } +.site-footer p { margin: 7px 0 0; } +.site-footer nav { display: flex; gap: 24px; } + +@media (max-width: 980px) { + .header-inner { grid-template-columns: 150px 1fr auto; gap: 16px; } + .main-nav { gap: 22px; } + .main-nav a { font-size: 14px; } + .page-heading h1 { font-size: 56px; } + .lead-story { grid-template-columns: 1.1fr .9fr; min-height: 420px; } + .lead-media { min-height: 420px; } + .lead-copy { padding-left: 30px; } + .lead-copy h2 { font-size: 33px; } + .feed-row { grid-template-columns: 290px minmax(0, 1fr); gap: 26px; } + .article-layout { grid-template-columns: minmax(0, 680px) 220px; gap: 44px; } +} + +@media (max-width: 760px) { + :root { --gutter: 16px; } + body { font-size: 17px; } + .header-inner { min-height: 64px; grid-template-columns: 1fr auto; } + .brand img { width: 38px; height: 38px; } + .main-nav { + position: absolute; + top: 64px; + right: 0; + left: 0; + display: none; + padding: 12px var(--gutter) 18px; + border-bottom: 1px solid var(--border); + background: var(--bg); + } + .main-nav.is-open { display: grid; } + .main-nav a { min-height: 44px; border-bottom: 1px solid var(--border); font-size: 16px; } + .main-nav a[aria-current="page"]::after { display: none; } + .menu-button { display: grid; } + .page-heading { padding: 38px 0 30px; } + .page-heading h1, .taxonomy-heading h1 { font-size: 44px; line-height: 1; } + .lead-story { display: block; min-height: 0; } + .lead-media { min-height: 0; aspect-ratio: 4 / 3; } + .lead-copy { padding: 25px 0 34px; } + .lead-copy h2 { margin: 20px 0 15px; font-size: 32px; } + .lead-copy > p { font-size: 18px; } + .feed-row { grid-template-columns: 132px minmax(0, 1fr); gap: 16px; padding: 18px 0; } + .feed-row-media { aspect-ratio: 4 / 3; } + .feed-row h2 { margin: 10px 0 0; font-size: 20px; } + .feed-row p { display: none; } + .article-meta { font-size: 12px; } + .article-meta time { display: none; } + .article-meta > * + *::before { height: 14px; margin: 0 10px; } + .article-meta .category-link::before { display: none; } + .article-header { margin-bottom: 30px; } + .article-header h1 { font-size: 42px; } + .article-lead { font-size: 20px; } + .article-layout { display: block; } + .article-body { font-size: 19px; } + .article-facts { margin-top: 42px; } + .related-section { margin-top: 64px; } +} + +@media (max-width: 460px) { + .feed-row { grid-template-columns: 112px minmax(0, 1fr); gap: 13px; } + .feed-row h2 { font-size: 18px; line-height: 1.18; } + .article-meta .producer-link::before { display: none; } + .article-meta .category-link { display: none; } + .article-header h1 { font-size: 36px; } + .article-byline { display: grid; gap: 7px; } + .site-footer { display: grid; gap: 24px; } +} + +@media (prefers-reduced-motion: reduce) { + html { scroll-behavior: auto; } + *, *::before, *::after { transition-duration: .01ms !important; } +} diff --git a/forma_n8_site/static/site.js b/forma_n8_site/static/site.js new file mode 100644 index 0000000..debf74e --- /dev/null +++ b/forma_n8_site/static/site.js @@ -0,0 +1,16 @@ +const menuButton = document.querySelector('[data-menu-toggle]'); +const nav = document.querySelector('#main-nav'); +const searchButton = document.querySelector('[data-search-toggle]'); +const searchForm = document.querySelector('[data-search-form]'); + +menuButton?.addEventListener('click', () => { + const open = menuButton.getAttribute('aria-expanded') === 'true'; + menuButton.setAttribute('aria-expanded', String(!open)); + nav?.classList.toggle('is-open', !open); +}); + +searchButton?.addEventListener('click', () => { + const opening = searchForm?.hasAttribute('hidden'); + searchForm?.toggleAttribute('hidden', !opening); + if (opening) searchForm?.querySelector('input')?.focus(); +}); diff --git a/forma_n8_site/templates/404.html b/forma_n8_site/templates/404.html new file mode 100644 index 0000000..459108c --- /dev/null +++ b/forma_n8_site/templates/404.html @@ -0,0 +1,4 @@ +{% extends "base.html" %} +{% block content %} +

404

Страница не найдена

Вернуться в ленту
+{% endblock %} diff --git a/forma_n8_site/templates/_article_rows.html b/forma_n8_site/templates/_article_rows.html new file mode 100644 index 0000000..a173c47 --- /dev/null +++ b/forma_n8_site/templates/_article_rows.html @@ -0,0 +1,29 @@ +{% macro media(article, eager=false) %} + {% set item = article.media[0] if article.media else none %} + {% if item and item.type == 'image' %} + {{ item.alt or article.title }} + {% elif item and item.type == 'video' %} + + {% else %} + + {% endif %} +{% endmacro %} + +{% macro meta(article) %} +
+ + {{ article.category_name }} + {{ article.producer_name }} +
+{% endmacro %} + +{% macro row(article) %} + +{% endmacro %} diff --git a/forma_n8_site/templates/_pagination.html b/forma_n8_site/templates/_pagination.html new file mode 100644 index 0000000..fb581f3 --- /dev/null +++ b/forma_n8_site/templates/_pagination.html @@ -0,0 +1,7 @@ +{% if pages > 1 %} + +{% endif %} diff --git a/forma_n8_site/templates/article.html b/forma_n8_site/templates/article.html new file mode 100644 index 0000000..45ad529 --- /dev/null +++ b/forma_n8_site/templates/article.html @@ -0,0 +1,38 @@ +{% extends "base.html" %} +{% from "_article_rows.html" import media, row %} +{% block og_type %}article{% endblock %} +{% block content %} +
+
+
Лента/{{ article.category_name }}
+

{{ article.title }}

+ {% if article.lead %}

{{ article.lead }}

{% endif %} +
+ + {{ article.producer_name }} +
+
+ {% if article.media %} +
+ {% for item in article.media %} + {% if item.type == 'image' %}
{{ item.alt or article.title }}
{% endif %} + {% if item.type == 'video' %}
{% endif %} + {% endfor %} +
+ {% endif %} +
+
{{ article.body|paragraphs }}
+ +
+
+ {% if related %} +
+

По теме

+
{% for item in related %}{{ row(item) }}{% endfor %}
+
+ {% endif %} +{% endblock %} diff --git a/forma_n8_site/templates/base.html b/forma_n8_site/templates/base.html new file mode 100644 index 0000000..1769bf8 --- /dev/null +++ b/forma_n8_site/templates/base.html @@ -0,0 +1,67 @@ + + + + + + {{ meta_title }} + + {% if noindex %}{% else %}{% endif %} + + + + + + + + + + {% if og_image %}{% endif %} + + {% if json_ld %}{% endif %} + + + +
+
+ + + FORMA
N8 + + +
+ + +
+
+
+ +
+ + +
+
+
+
{% block content %}{% endblock %}
+
+
+ FORMA N8 +

Новости снаряжения и экипировки.

+
+ +
+ + + diff --git a/forma_n8_site/templates/feed.html b/forma_n8_site/templates/feed.html new file mode 100644 index 0000000..c336afc --- /dev/null +++ b/forma_n8_site/templates/feed.html @@ -0,0 +1,34 @@ +{% extends "base.html" %} +{% from "_article_rows.html" import media, meta, row %} +{% block content %} +
+
+

Новости снаряжения

+ {% if q %}

Результаты поиска: «{{ q }}»

{% endif %} +
+ {% if articles %} + {% if page == 1 and not q %} + {% set lead = articles[0] %} + +
+ {% for article in articles[1:] %}{{ row(article) }}{% endfor %} +
+ {% else %} +
+ {% for article in articles %}{{ row(article) }}{% endfor %} +
+ {% endif %} + {% include "_pagination.html" %} + {% else %} +

Ничего не найдено

Попробуйте изменить запрос или вернуться к ленте.

Вернуться в ленту
+ {% endif %} +
+{% endblock %} diff --git a/forma_n8_site/templates/taxonomy.html b/forma_n8_site/templates/taxonomy.html new file mode 100644 index 0000000..bdd89f5 --- /dev/null +++ b/forma_n8_site/templates/taxonomy.html @@ -0,0 +1,18 @@ +{% extends "base.html" %} +{% from "_article_rows.html" import row %} +{% block content %} +
+
+

{{ taxonomy_label }}

+

{{ heading }}

+
+ {% if articles %} +
+ {% for article in articles %}{{ row(article) }}{% endfor %} +
+ {% include "_pagination.html" %} + {% else %} +

Публикаций пока нет

Новые материалы появятся здесь после публикации.

Вернуться в ленту
+ {% endif %} +
+{% endblock %} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7c83571 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +aiohttp==3.12.13 +aiogram==3.21.0 +asyncpg==0.30.0 +fastapi==0.115.14 +jinja2==3.1.6 +loguru==0.7.3 +litellm==1.76.0 +pydantic-settings==2.10.1 +Pillow==11.3.0 +python-multipart==0.0.20 +uvicorn[standard]==0.35.0 +yt-dlp==2026.6.9 diff --git a/scripts/apply_migrations.py b/scripts/apply_migrations.py new file mode 100644 index 0000000..1579e94 --- /dev/null +++ b/scripts/apply_migrations.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from vk_parser_app.db import apply_migrations, close_pool + + +async def main() -> None: + root = Path(__file__).resolve().parents[1] + await apply_migrations(root / "db" / "migrations") + await close_pool() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/backfill_max_publication_urls.py b/scripts/backfill_max_publication_urls.py new file mode 100644 index 0000000..265bceb --- /dev/null +++ b/scripts/backfill_max_publication_urls.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import asyncio + +from loguru import logger + +from vk_parser_app.constants import PUBLICATION_STATUS_PUBLISHED +from vk_parser_app.db import close_pool, fetch_int_setting, fetch_setting, get_pool +from vk_parser_app.workers.max_poster import MAXAPIClient, MAXAPIError, MAXPoster + + +async def main() -> None: + pool = await get_pool() + token = str(await fetch_setting("max_poster_bot_token", "") or "").strip() + base_url = str(await fetch_setting("max_poster_api_base_url", "https://platform-api2.max.ru") or "").strip() + if not token: + raise RuntimeError("max_poster_bot_token is empty") + + rows = await pool.fetch( + """ + SELECT id, raw_post_id, external_id + FROM post_publications + WHERE platform='max' + AND status=$1 + AND COALESCE(url, '')='' + AND COALESCE(external_id, '')<>'' + ORDER BY published_at DESC NULLS LAST, id DESC + """, + PUBLICATION_STATUS_PUBLISHED, + ) + + poster = MAXPoster() + updated = 0 + missing = 0 + async with MAXAPIClient( + token, + base_url, + timeout_sec=max(30, await fetch_int_setting("max_poster_timeout_sec", 120)), + max_attempts=max(1, await fetch_int_setting("max_poster_max_attempts", 3)), + retry_backoff_max_sec=max(1, await fetch_int_setting("max_poster_retry_backoff_max_sec", 30)), + ) as client: + for row in rows: + try: + data = await client.get_message(str(row["external_id"])) + except MAXAPIError as exc: + missing += 1 + logger.warning( + "MAX URL backfill skipped raw_post={} message={} error={}", + row["raw_post_id"], + row["external_id"], + exc, + ) + continue + url = poster.message_url_from_response(data) + if not url: + missing += 1 + continue + await pool.execute( + """ + UPDATE post_publications + SET url=$2, + updated_at=NOW() + WHERE id=$1 + """, + int(row["id"]), + url, + ) + updated += 1 + logger.info("MAX URL backfilled raw_post={} url={}", row["raw_post_id"], url) + + logger.info("MAX URL backfill done: scanned={} updated={} missing={}", len(rows), updated, missing) + await close_pool() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/backfill_pending_videos.py b/scripts/backfill_pending_videos.py new file mode 100644 index 0000000..5b0813d --- /dev/null +++ b/scripts/backfill_pending_videos.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import asyncio +import html +import json +import os +from pathlib import Path +from typing import Any + +from aiogram.types import FSInputFile + +from vk_parser_app.constants import MEDIA_STATUS_FAILED, MEDIA_STATUS_LINK_ONLY +from vk_parser_app.db import get_pool +from vk_parser_app.workers.vk_storage_uploader import TMP_DIR, TMP_PREFIX, TelegramStorageUploader + + +def jlog(event: str, **payload: Any) -> None: + print(json.dumps({"event": event, **payload}, ensure_ascii=False, default=str), flush=True) + + +async def load_pending_videos(worker: TelegramStorageUploader, limit: int) -> list[dict[str, Any]]: + rows = await worker.pool.fetch( + """ + SELECT m.*, rp.original_url AS post_url + FROM raw_post_media m + JOIN raw_posts rp ON rp.id=m.raw_post_id + WHERE m.media_type='video' + AND m.status='pending' + AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = '' + AND COALESCE(m.error, '') ILIKE '%timeout%' + AND COALESCE(m.duration_sec, 0) <= $1 + ORDER BY COALESCE(rp.publication_status, 'pending')='published', m.id DESC + LIMIT $2 + """, + worker.video_max_duration_sec, + limit, + ) + return [dict(row) for row in rows] + + +async def send_video_to_storage(worker: TelegramStorageUploader, media: dict[str, Any], path: str) -> None: + caption = ( + f"Backfill video media #{int(media['id'])} for raw_post #{int(media['raw_post_id'])}\n" + f'source post' + ) + message = await worker.tg_retry( + lambda: worker.bot.send_video( + video=FSInputFile(path, filename=f"video_{int(media['id'])}.mp4"), + caption=caption[: worker.caption_limit], + parse_mode="HTML", + **worker.chat_kwargs(), + ) + ) + if not message.video: + raise RuntimeError("Telegram accepted message without video payload") + await worker.mark_media_uploaded(int(media["id"]), message.video.file_id, message.video.file_unique_id) + + +async def main() -> None: + limit = int(os.getenv("BACKFILL_LIMIT", "200")) + worker = TelegramStorageUploader() + await worker.init() + videos = await load_pending_videos(worker, limit) + jlog("start", count=len(videos), limit=limit) + ok = 0 + failed = 0 + link_only = 0 + try: + for media in videos: + media_id = int(media["id"]) + raw_post_id = int(media["raw_post_id"]) + path = str(TMP_DIR / f"{TMP_PREFIX}backfill_{raw_post_id}_{media_id}.mp4") + jlog( + "video_start", + media_id=media_id, + raw_post_id=raw_post_id, + url=media.get("original_url"), + duration_sec=media.get("duration_sec"), + ) + try: + info = await worker.download_video(str(media.get("original_url") or ""), path) + if not info: + await worker.mark_media_failed_attempt(media_id, "video download failed") + failed += 1 + jlog("video_failed", media_id=media_id, raw_post_id=raw_post_id, error="video download failed") + continue + if info.get("error"): + if info.get("permanent"): + await worker.mark_media_link_only(media_id, str(info["error"])) + link_only += 1 + else: + await worker.mark_media_failed_attempt(media_id, str(info["error"])) + failed += 1 + jlog("video_failed", media_id=media_id, raw_post_id=raw_post_id, error=info.get("error")) + continue + await send_video_to_storage(worker, media, path) + ok += 1 + jlog("video_uploaded", media_id=media_id, raw_post_id=raw_post_id, size_bytes=info.get("size_bytes")) + await asyncio.sleep(worker.media_upload_delay_sec) + except Exception as exc: + await worker.mark_media_failed_attempt(media_id, str(exc)) + failed += 1 + jlog("video_failed", media_id=media_id, raw_post_id=raw_post_id, error=str(exc)) + finally: + Path(path).unlink(missing_ok=True) + finally: + await worker.close() + if worker.pool: + await worker.pool.close() + jlog("done", ok=ok, failed=failed, link_only=link_only, total=len(videos)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/publish_site_batch.py b/scripts/publish_site_batch.py new file mode 100644 index 0000000..88c8a46 --- /dev/null +++ b/scripts/publish_site_batch.py @@ -0,0 +1,529 @@ +from __future__ import annotations + +import argparse +import asyncio +import base64 +import hashlib +import hmac +import html +import io +import json +import os +import re +import subprocess +import tempfile +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +try: + import fcntl +except ImportError: # pragma: no cover - production runs on Linux + fcntl = None + +import aiohttp +from aiogram import Bot +from aiogram.client.session.aiohttp import AiohttpSession +from aiogram.client.telegram import TelegramAPIServer +from PIL import Image, ImageOps + +from vk_parser_app.config import settings +from vk_parser_app.db import close_pool, fetch_setting, get_pool + + +SEPARATOR_RE = re.compile(r"^━+$") +HASHTAG_LINE_RE = re.compile(r"^(?:#[\wа-яё]+\s*)+$", re.IGNORECASE) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Publish selected accepted posts to Ghost") + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--ids", help="Comma-separated raw_post ids") + group.add_argument("--all-ready", action="store_true", help="Publish every photo-ready post not yet published to site") + group.add_argument("--all-ready-videos", action="store_true", help="Publish video-only ready posts") + parser.add_argument("--ghost-url", required=True) + parser.add_argument("--key-file", default="") + parser.add_argument("--fallback-title", default="Новость") + parser.add_argument("--limit", type=int, default=0, help="Maximum posts to publish in this run; 0 means unlimited") + parser.add_argument("--retry-failed", action="store_true", help="Include previous site publish_failed rows") + return parser.parse_args() + + +def ghost_token(key: str) -> str: + key_id, secret = key.strip().split(":", 1) + now = int(time.time()) + + def encode(value: dict[str, Any] | bytes) -> str: + raw = value if isinstance(value, bytes) else json.dumps(value, separators=(",", ":")).encode() + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode() + + header = encode({"alg": "HS256", "typ": "JWT", "kid": key_id}) + payload = encode({"iat": now, "exp": now + 300, "aud": "/admin/"}) + signature = encode(hmac.new(bytes.fromhex(secret), f"{header}.{payload}".encode(), hashlib.sha256).digest()) + return f"{header}.{payload}.{signature}" + + +def split_post_text(value: str, fallback_title: str) -> tuple[str, str]: + lines = [line.strip() for line in str(value or "").replace("\r", "").split("\n")] + while lines and not lines[0]: + lines.pop(0) + title = lines.pop(0) if lines else fallback_title + while lines and (not lines[0] or SEPARATOR_RE.fullmatch(lines[0])): + lines.pop(0) + while lines and (not lines[-1] or HASHTAG_LINE_RE.fullmatch(lines[-1])): + lines.pop() + body = "\n".join(lines).strip() + return title[:255], body + + +def paragraphs(value: str) -> str: + chunks = [chunk.strip() for chunk in re.split(r"\n\s*\n", value) if chunk.strip()] + return "".join(f"

{html.escape(chunk).replace(chr(10), '
')}

" for chunk in chunks) + + +def excerpt(value: str, limit: int = 260) -> str: + clean = re.sub(r"\s+", " ", value).strip() + if len(clean) <= limit: + return clean + return clean[: limit - 1].rsplit(" ", 1)[0] + "…" + + +def compress_photo(data: bytes) -> tuple[bytes, int, int]: + with Image.open(io.BytesIO(data)) as source: + image = ImageOps.exif_transpose(source) + if image.mode not in {"RGB", "RGBA"}: + image = image.convert("RGB") + image.thumbnail((2000, 2000), Image.Resampling.LANCZOS) + output = io.BytesIO() + image.save(output, "WEBP", quality=82, method=6) + return output.getvalue(), image.width, image.height + + +def video_thumbnail(data: bytes) -> tuple[bytes, int, int]: + video_path = "" + image_path = "" + try: + with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as video_file: + video_file.write(data) + video_path = video_file.name + with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as image_file: + image_path = image_file.name + command = [ + "ffmpeg", "-y", "-ss", "1", "-i", video_path, + "-frames:v", "1", "-vf", "scale='min(2000,iw)':-2", "-q:v", "3", image_path, + ] + result = subprocess.run(command, capture_output=True, timeout=90) + if result.returncode != 0 or not Path(image_path).exists() or Path(image_path).stat().st_size == 0: + command[3] = "0" + result = subprocess.run(command, capture_output=True, timeout=90) + if result.returncode != 0: + raise RuntimeError("ffmpeg thumbnail extraction failed") + return compress_photo(Path(image_path).read_bytes()) + finally: + for path in (video_path, image_path): + if path: + try: + os.unlink(path) + except FileNotFoundError: + pass + + +def normalize_video(data: bytes, duration: int, height: int | None) -> bytes: + max_bytes = 40 * 1024 * 1024 + if len(data) <= max_bytes and (not height or height <= 720): + return data + input_path = "" + output_path = "" + try: + with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as input_file: + input_file.write(data) + input_path = input_file.name + with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as output_file: + output_path = output_file.name + safe_duration = max(1, int(duration or 1)) + target_total_kbps = int((38 * 1024 * 8) / safe_duration) + video_kbps = max(280, min(2800, target_total_kbps - 96)) + command = [ + "ffmpeg", "-y", "-i", input_path, + "-map_metadata", "-1", "-c:v", "libx264", "-preset", "fast", + "-b:v", f"{video_kbps}k", "-maxrate", f"{video_kbps}k", "-bufsize", f"{video_kbps * 2}k", + ] + if height and int(height) > 720: + command.extend(["-vf", "scale=-2:720"]) + command.extend(["-c:a", "aac", "-b:a", "96k", "-movflags", "+faststart", output_path]) + result = subprocess.run(command, capture_output=True, timeout=max(180, safe_duration * 3)) + if result.returncode != 0 or not Path(output_path).exists(): + raise RuntimeError("ffmpeg video normalization failed") + normalized = Path(output_path).read_bytes() + if len(normalized) > 48 * 1024 * 1024: + raise RuntimeError(f"normalized video remains too large: {len(normalized)} bytes") + return normalized + finally: + for path in (input_path, output_path): + if path: + try: + os.unlink(path) + except FileNotFoundError: + pass + + +def gallery_html(images: list[dict[str, Any]], title: str) -> str: + if not images: + return "" + if len(images) == 1: + image = images[0] + return ( + '
' + f'' + "
" + ) + rows = [] + for offset in range(0, len(images), 3): + cells = [] + for image in images[offset : offset + 3]: + cells.append( + '
' + f'' + "
" + ) + rows.append('
' + "".join(cells) + "
") + return '
' + "".join(rows) + "
" + + +def videos_html(videos: list[dict[str, Any]], title: str, poster_url: str) -> str: + return "".join( + '
' + f'' + "
" + for video in videos + ) + + +def lexical_paragraph(text: str) -> dict[str, Any]: + return { + "children": [{"detail": 0, "format": 0, "mode": "normal", "style": "", "text": text, "type": "extended-text", "version": 1}], + "direction": None, + "format": "", + "indent": 0, + "type": "paragraph", + "version": 1, + } + + +def video_lexical(videos: list[dict[str, Any]], body: str, thumbnail: dict[str, Any]) -> str: + children: list[dict[str, Any]] = [] + for video in videos: + children.append( + { + "type": "video", + "version": 1, + "src": video["url"], + "caption": "", + "fileName": video["file_name"], + "mimeType": "video/mp4", + "width": video.get("width"), + "height": video.get("height"), + "duration": video.get("duration") or 0, + "thumbnailSrc": thumbnail["url"], + "customThumbnailSrc": "", + "thumbnailWidth": thumbnail["width"], + "thumbnailHeight": thumbnail["height"], + "cardWidth": "wide", + "loop": False, + } + ) + children.extend(lexical_paragraph(chunk.strip()) for chunk in re.split(r"\n\s*\n", body) if chunk.strip()) + return json.dumps( + {"root": {"children": children, "direction": None, "format": "", "indent": 0, "type": "root", "version": 1}}, + ensure_ascii=False, + separators=(",", ":"), + ) + + +class GhostClient: + def __init__(self, session: aiohttp.ClientSession, base_url: str, key: str) -> None: + self.session = session + self.base_url = base_url.rstrip("/") + self.key = key + + def headers(self) -> dict[str, str]: + return {"Authorization": f"Ghost {ghost_token(self.key)}", "Accept-Version": "v6.0"} + + async def upload_image(self, data: bytes, name: str, ref: str) -> str: + form = aiohttp.FormData() + form.add_field("file", data, filename=name, content_type="image/webp") + form.add_field("purpose", "image") + form.add_field("ref", ref) + async with self.session.post( + f"{self.base_url}/ghost/api/admin/images/upload/", data=form, headers=self.headers() + ) as response: + payload = await response.json(content_type=None) + if response.status >= 300: + raise RuntimeError(f"Ghost image upload {response.status}: {payload}") + return str(payload["images"][0]["url"]) + + async def upload_media(self, data: bytes, name: str, content_type: str) -> str: + form = aiohttp.FormData() + form.add_field("file", data, filename=name, content_type=content_type) + async with self.session.post( + f"{self.base_url}/ghost/api/admin/media/upload/", data=form, headers=self.headers() + ) as response: + payload = await response.json(content_type=None) + if response.status >= 300: + raise RuntimeError(f"Ghost media upload {response.status}: {payload}") + return str(payload["media"][0]["url"]) + + async def create_post(self, post: dict[str, Any]) -> dict[str, Any]: + suffix = "/" if "lexical" in post else "/?source=html" + async with self.session.post( + f"{self.base_url}/ghost/api/admin/posts{suffix}", + json={"posts": [post]}, + headers=self.headers(), + ) as response: + payload = await response.json(content_type=None) + if response.status >= 300: + raise RuntimeError(f"Ghost post create {response.status}: {payload}") + return dict(payload["posts"][0]) + + +async def main() -> None: + args = parse_args() + key = os.environ.get("SITE_POSTER_GHOST_ADMIN_KEY", "").strip() + if not key: + if not args.key_file: + raise RuntimeError("SITE_POSTER_GHOST_ADMIN_KEY or --key-file is required") + key = Path(args.key_file).read_text(encoding="utf-8").strip() + pool = await get_pool() + if args.all_ready or args.all_ready_videos: + media_condition = ( + """EXISTS ( + SELECT 1 FROM raw_post_media rpm + WHERE rpm.raw_post_id=rp.id AND rpm.media_type='video' + AND COALESCE(rpm.editor_hidden,FALSE)=FALSE + AND COALESCE(rpm.tg_file_id,rpm.storage_attachment_id,'')<>'' + ) AND NOT EXISTS ( + SELECT 1 FROM raw_post_media rpm + WHERE rpm.raw_post_id=rp.id AND rpm.media_type='photo' + AND COALESCE(rpm.editor_hidden,FALSE)=FALSE + AND COALESCE(rpm.tg_file_id,rpm.storage_attachment_id,'')<>'' + )""" + if args.all_ready_videos + else """EXISTS ( + SELECT 1 FROM raw_post_media rpm + WHERE rpm.raw_post_id=rp.id AND rpm.media_type='photo' + AND COALESCE(rpm.editor_hidden,FALSE)=FALSE + AND COALESCE(rpm.tg_file_id,rpm.storage_attachment_id,'')<>'' + )""" + ) + candidates = [ + dict(row) + for row in await pool.fetch( + f""" + SELECT rp.id, + COALESCE(rp.final_category_tag,rp.rewrite_category_tag,'') AS category, + COALESCE(rp.final_source_tag,rp.rewrite_source_tag,s.tag,'') AS source + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + JOIN content_categories cc ON cc.sort_order=COALESCE(rp.final_category_id,rp.rewrite_category_id) + LEFT JOIN post_publications pp ON pp.raw_post_id=rp.id AND pp.platform='site' + WHERE rp.status='storage_ready' + AND rp.rewrite_status='ready' + AND COALESCE(rp.editorial_status,'review') IN ('accepted','published','publish_failed') + AND cc.site_enabled=TRUE + AND COALESCE(pp.status,'pending') {"<> 'published'" if args.retry_failed else "= 'pending'"} + AND {media_condition} + ORDER BY COALESCE(rp.reviewed_at,rp.edited_at,rp.rewritten_at,rp.created_at),rp.id + """ + ) + ] + recent = [ + dict(row) + for row in await pool.fetch( + """ + SELECT COALESCE(rp.final_category_tag,rp.rewrite_category_tag,'') AS category, + COALESCE(rp.final_source_tag,rp.rewrite_source_tag,s.tag,'') AS source + FROM post_publications pp + JOIN raw_posts rp ON rp.id=pp.raw_post_id + JOIN sources s ON s.id=rp.source_id + WHERE pp.platform='site' AND pp.status='published' + ORDER BY pp.published_at DESC NULLS LAST,pp.id DESC LIMIT 20 + """ + ) + ] + selected: list[dict[str, Any]] = [] + while candidates: + previous = selected[-1] if selected else (recent[0] if recent else {}) + recent_window = recent + selected[-20:] + best = min( + range(len(candidates)), + key=lambda index: ( + int(bool(candidates[index]["category"]) and candidates[index]["category"] == previous.get("category")) + + int(bool(candidates[index]["source"]) and candidates[index]["source"] == previous.get("source")), + sum(3 for row in recent_window if row.get("category") == candidates[index]["category"]) + + sum(4 for row in recent_window if row.get("source") == candidates[index]["source"]), + index, + ), + ) + selected.append(candidates.pop(best)) + ids = [int(row["id"]) for row in selected] + if args.limit > 0: + ids = ids[: args.limit] + else: + ids = [int(item) for item in str(args.ids or "").split(",") if item.strip()] + if not ids: + print("NO_READY_POSTS") + await close_pool() + return + print(f"QUEUE {len(ids)}") + rows = await pool.fetch( + """ + SELECT rp.*, s.name AS source_name, s.tag AS source_tag, + cc.site_name, cc.site_slug, cc.site_enabled, + pp.status AS site_publication_status + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + JOIN content_categories cc ON cc.sort_order=COALESCE(rp.final_category_id, rp.rewrite_category_id) + LEFT JOIN post_publications pp ON pp.raw_post_id=rp.id AND pp.platform='site' + WHERE rp.id=ANY($1::bigint[]) + """, + ids, + ) + by_id = {int(row["id"]): dict(row) for row in rows} + local_api = str(await fetch_setting("local_bot_api_url", settings.local_bot_api_url) or "").strip() + bot_session = AiohttpSession(api=TelegramAPIServer.from_base(local_api.rstrip("/"), is_local=True)) if local_api else AiohttpSession() + bot = Bot(settings.tg_bot_token, session=bot_session) + timeout = aiohttp.ClientTimeout(total=120) + published_base = datetime.now(timezone.utc) - timedelta(minutes=len(ids)) + try: + async with aiohttp.ClientSession(timeout=timeout) as http_session: + ghost = GhostClient(http_session, args.ghost_url, key) + for position, raw_post_id in enumerate(ids): + post = by_id.get(raw_post_id) + if not post: + print(f"SKIP {raw_post_id}: missing or unmapped category") + continue + if not post.get("site_enabled"): + print(f"SKIP {raw_post_id}: category disabled for site") + continue + if post.get("site_publication_status") == "published": + print(f"SKIP {raw_post_id}: already published") + continue + try: + media_rows = await pool.fetch( + """ + SELECT * FROM raw_post_media + WHERE raw_post_id=$1 AND media_type IN ('photo','video') + AND COALESCE(editor_hidden,FALSE)=FALSE + AND COALESCE(tg_file_id,storage_attachment_id,'')<>'' + ORDER BY sort_order,id + """, + raw_post_id, + ) + uploaded_photos = [] + uploaded_videos = [] + thumbnail = None + for index, media in enumerate(media_rows): + file_id = str(media["tg_file_id"] or media["storage_attachment_id"]) + telegram_file = await bot.get_file(file_id) + buffer = io.BytesIO() + await bot.download_file(telegram_file.file_path, destination=buffer) + data = buffer.getvalue() + if media["media_type"] == "photo": + compressed, width, height = compress_photo(data) + url = await ghost.upload_image(compressed, f"post-{raw_post_id}-{index + 1}.webp", f"raw-post-{raw_post_id}-{index + 1}") + uploaded_photos.append({"type": "photo", "url": url, "width": width, "height": height, "bytes": len(compressed)}) + else: + data = normalize_video(data, int(media.get("duration_sec") or 0), media.get("height")) + url = await ghost.upload_media(data, f"post-{raw_post_id}-{index + 1}.mp4", "video/mp4") + uploaded_videos.append({ + "type": "video", "url": url, "bytes": len(data), + "file_name": f"post-{raw_post_id}-{index + 1}.mp4", + "width": media.get("width"), "height": media.get("height"), + "duration": media.get("duration_sec") or 0, + }) + if thumbnail is None: + compressed, width, height = video_thumbnail(data) + thumb_url = await ghost.upload_image(compressed, f"post-{raw_post_id}-video-cover.webp", f"raw-post-{raw_post_id}-video-cover") + thumbnail = {"type": "thumbnail", "url": thumb_url, "width": width, "height": height, "bytes": len(compressed)} + feature = uploaded_photos[0] if uploaded_photos else thumbnail + if not feature: + raise RuntimeError("no downloadable site media") + title, body = split_post_text(post.get("final_text") or post.get("rewritten_text") or "", args.fallback_title) + producer = str(post.get("source_name") or "").strip() + content = gallery_html(uploaded_photos[1:], title) + videos_html(uploaded_videos, title, feature["url"]) + paragraphs(body) + attachments = uploaded_photos + uploaded_videos + ([thumbnail] if thumbnail else []) + payload = { + "title": title, + "status": "published", + "published_at": (published_base + timedelta(minutes=position)).isoformat(), + "custom_excerpt": excerpt(body), + "meta_description": excerpt(body), + "feature_image": feature["url"], + "feature_image_alt": title, + "tags": [ + {"name": str(post["site_name"]), "slug": str(post["site_slug"])}, + {"name": f"#producer:{producer}"}, + {"name": f"#raw-post:{raw_post_id}"}, + ], + } + if uploaded_videos and not uploaded_photos: + payload["lexical"] = video_lexical(uploaded_videos, body, thumbnail) + else: + payload["html"] = content + created = await ghost.create_post(payload) + await pool.execute( + """ + INSERT INTO post_publications(raw_post_id,platform,status,target_id,external_id,url,error,attachments_json,published_at) + VALUES($1,'site','published',$2,$3,$4,NULL,$5::jsonb,NOW()) + ON CONFLICT(raw_post_id,platform) DO UPDATE SET + status='published',target_id=EXCLUDED.target_id,external_id=EXCLUDED.external_id, + url=EXCLUDED.url,error=NULL,attachments_json=EXCLUDED.attachments_json, + published_at=NOW(),updated_at=NOW() + """, + raw_post_id, + args.ghost_url, + str(created["id"]), + str(created["url"]), + json.dumps(attachments, ensure_ascii=False), + ) + print(f"PUBLISHED {raw_post_id} {created['url']} media={len(attachments)} bytes={sum(x['bytes'] for x in attachments)}") + except Exception as exc: + await pool.execute( + """ + INSERT INTO post_publications(raw_post_id,platform,status,target_id,error,updated_at) + VALUES($1,'site','publish_failed',$2,$3,NOW()) + ON CONFLICT(raw_post_id,platform) DO UPDATE SET + status='publish_failed',target_id=EXCLUDED.target_id,error=EXCLUDED.error,updated_at=NOW() + """, + raw_post_id, + args.ghost_url, + str(exc)[:2000], + ) + print(f"FAILED {raw_post_id}: {exc}") + finally: + await bot.session.close() + await close_pool() + + +def run_locked() -> None: + if fcntl is None: + asyncio.run(main()) + return + with Path("/tmp/site-poster.lock").open("w") as lock_file: + try: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + print("ALREADY_RUNNING") + return + asyncio.run(main()) + + +if __name__ == "__main__": + run_locked() diff --git a/scripts/update_ghost_about.mjs b/scripts/update_ghost_about.mjs new file mode 100644 index 0000000..75c02aa --- /dev/null +++ b/scripts/update_ghost_about.mjs @@ -0,0 +1,39 @@ +import {createHmac} from "node:crypto"; +import {readFileSync} from "node:fs"; + +const baseUrl = (process.env.GHOST_URL || "").replace(/\/$/, ""); +const keyFile = process.env.GHOST_ADMIN_KEY_FILE || ""; +if (!baseUrl || !keyFile) { + throw new Error("GHOST_URL and GHOST_ADMIN_KEY_FILE are required"); +} +const [keyId, secret] = readFileSync(keyFile, "utf8").trim().split(":"); +const encode = (value) => Buffer.from(JSON.stringify(value)).toString("base64url"); +const now = Math.floor(Date.now() / 1000); +const unsigned = `${encode({alg: "HS256", typ: "JWT", kid: keyId})}.${encode({iat: now, exp: now + 300, aud: "/admin/"})}`; +const token = `${unsigned}.${createHmac("sha256", Buffer.from(secret, "hex")).update(unsigned).digest("base64url")}`; +const headers = {Authorization: `Ghost ${token}`, "Content-Type": "application/json"}; + +const oldText = "Мы собираем материалы производителей и профильных магазинов, проверяем их релевантность и приводим к единому удобному формату."; +const newText = "Ежедневно мы собираем новости снаряжения с сотен источников, отбираем лучшее и публикуем в одном месте!"; + +const response = await fetch(`${baseUrl}/ghost/api/admin/pages/slug/about/?formats=html`, {headers}); +if (!response.ok) throw new Error(`Ghost page read failed: ${response.status} ${await response.text()}`); +const page = (await response.json()).pages[0]; +const html = String(page.html || ""); +if (!html.includes(oldText) && !String(page.custom_excerpt || "").includes(oldText)) { + throw new Error("Expected about text was not found"); +} +const update = { + id: page.id, + updated_at: page.updated_at, + html: html.replaceAll(oldText, newText), + custom_excerpt: String(page.custom_excerpt || "").replaceAll(oldText, newText), +}; +const saved = await fetch(`${baseUrl}/ghost/api/admin/pages/${page.id}/?source=html`, { + method: "PUT", + headers, + body: JSON.stringify({pages: [update]}), +}); +if (!saved.ok) throw new Error(`Ghost page update failed: ${saved.status} ${await saved.text()}`); +const result = (await saved.json()).pages[0]; +console.log(JSON.stringify({id: result.id, slug: result.slug, updated_at: result.updated_at})); diff --git a/scripts/vk_storage_spike.py b/scripts/vk_storage_spike.py new file mode 100644 index 0000000..a7ea48a --- /dev/null +++ b/scripts/vk_storage_spike.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from vk_parser_app.config import settings +from vk_parser_app.vk_api import VKAPIClient, post_vk_url + + +async def main() -> None: + if not settings.vk_access_token: + raise RuntimeError("VK_ACCESS_TOKEN is empty") + if not settings.vk_storage_group_id: + raise RuntimeError("VK_STORAGE_GROUP_ID is empty") + + async with VKAPIClient() as client: + attachments: list[str] = [] + sample = Path("sample_photo.jpg") + if sample.exists(): + saved = await client.upload_wall_photo_bytes( + abs(settings.vk_storage_group_id), + sample.read_bytes(), + filename="sample_photo.jpg", + ) + attachments.append(f"photo{saved['owner_id']}_{saved['id']}") + + post_id = await client.create_wall_post( + owner_id=settings.vk_storage_owner_id, + message="VK storage spike: test raw copy post", + attachments=attachments, + from_group=True, + ) + print(post_vk_url(settings.vk_storage_owner_id, post_id)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/vk_parser_app/__init__.py b/src/vk_parser_app/__init__.py new file mode 100644 index 0000000..a05eb9a --- /dev/null +++ b/src/vk_parser_app/__init__.py @@ -0,0 +1,3 @@ +__all__ = ["__version__"] + +__version__ = "0.1.0" diff --git a/src/vk_parser_app/admin.py b/src/vk_parser_app/admin.py new file mode 100644 index 0000000..a4e34d5 --- /dev/null +++ b/src/vk_parser_app/admin.py @@ -0,0 +1,4011 @@ +from __future__ import annotations + +import asyncio +import json +import base64 +import hashlib +import mimetypes +import re +import secrets +import time +from datetime import date, datetime, timedelta, timezone +from pathlib import Path +from typing import Any +from urllib.parse import urlencode +from zoneinfo import ZoneInfo + +import aiohttp +from fastapi import FastAPI, File, Form, Request, UploadFile, status +from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, RedirectResponse +from fastapi.staticfiles import StaticFiles +from fastapi.templating import Jinja2Templates +from loguru import logger + +from .config import settings +from .constants import PLATFORM_VK +from .db import fetch_int_setting, fetch_setting, get_pool +from .security import hash_password, new_token, token_hash, verify_password +from .text_utils import build_publication_text, normalize_hash_tag, parse_categories +from .vk_api import VKAPIClient, normalize_vk_source +from .workers.ai_qualifier import AIQualifierWorker, normalize_model, response_usage +from .workers.ai_writer import AIWriterWorker +from .workers.daily_report import DailyReportWorker +from .workers.max_poster import MAXPoster +from .workers.parser import VKParserWorker +from .workers.site_poster import SitePoster +from .workers.tg_poster import TelegramPoster +from .workers.tg_reactor import TelegramReactor +from .workers.vk_poster import VKPoster +from .workers.vk_storage_uploader import TelegramStorageUploader + +COOKIE_NAME = "vk_parser_admin" +VK_OAUTH_VERIFIER_COOKIE = "vk_oauth_verifier" +VK_OAUTH_STATE_COOKIE = "vk_oauth_state" +BASE_DIR = Path(__file__).resolve().parent +PROJECT_ROOT = BASE_DIR.parents[1] +FAVICON_PATH = PROJECT_ROOT / "favi.png" +templates = Jinja2Templates(directory=str(BASE_DIR / "templates")) +app = FastAPI(title=settings.display_site_title) +LOCAL_TZ = ZoneInfo("Asia/Yekaterinburg") +MODEL_CACHE: dict[str, Any] = {"key": "", "at": 0.0, "models": []} +UPLOAD_ROOT = Path("uploads").resolve() +EDITOR_MEDIA_DIR = UPLOAD_ROOT / "editor_media" +EDITOR_MEDIA_DIR.mkdir(parents=True, exist_ok=True) +app.mount("/uploads", StaticFiles(directory=str(UPLOAD_ROOT)), name="uploads") + + +@app.get("/favi.png") +async def favicon_png(): + return FileResponse(FAVICON_PATH) + + +@app.get("/favicon.ico") +async def favicon_ico(): + return FileResponse(FAVICON_PATH, media_type="image/png") + +PROVIDER_OPTIONS = [ + {"value": "openrouter", "label": "OpenRouter"}, + {"value": "openai", "label": "OpenAI"}, + {"value": "anthropic", "label": "Anthropic"}, + {"value": "gemini", "label": "Google Gemini"}, + {"value": "openai_compatible", "label": "OpenAI-compatible API"}, +] + +SITE_POSTER_PROVIDER_OPTIONS = [ + {"value": "", "label": "Не выбран"}, + {"value": "ghost", "label": "Ghost"}, +] + +PROMPT_HINTS = { + "ai_qualifier_prompt": { + "title": "Свободная инструкция квалификатора", + "body": ( + "Здесь можно спокойно менять смысл: роль, тон, критерии пригодности, примеры хороших и плохих постов, " + "логику оценки по шкале 1-10." + ), + "safe": ( + "Можно менять: критерии, примеры, тон причины, пороговые объяснения. " + "Не добавляй сюда JSON-схему: для неё есть отдельный технический контракт ниже." + ), + "input": "На вход уходит JSON-массив постов: id, source, original_url, media_count, media_types, text.", + }, + "ai_qualifier_contract": { + "title": "Технический контракт квалификатора", + "body": ( + "Эта часть защищает парсер от сломанного ответа. Меняй её только если осознанно меняешь формат ответа " + "и одновременно готов править валидатор в коде." + ), + "safe": "Обычно не трогаем. Здесь живут JSON-схема, допустимые decision/reject_tag и правило вернуть результат на каждый id.", + "input": "Контракт склеивается после свободной инструкции и отправляется как system prompt.", + "contract": '{"results":[{"id":123,"score":8,"decision":"accepted","reason":"до 10 слов на русском","reject_tag":null}]}', + }, + "ai_writer_prompt": { + "title": "Свободная инструкция райтера", + "body": ( + "Здесь можно менять редакторскую часть: стиль, длину, голос канала, запреты на выдумки, примеры хорошего текста." + ), + "safe": ( + "Можно менять: tone of voice, правила фактов, длину, примеры фраз. " + "Хэштеги руками писать не надо: модель выбирает category, а код сам соберёт #category и #producer_tag." + ), + "input": "На вход уходит JSON-объект: categories и posts. В каждом post есть id, producer_name, producer_tag, original_url, qualification_score, media_count, media_types, text.", + }, + "ai_writer_contract": { + "title": "Технический контракт райтера", + "body": ( + "Эта часть фиксирует формат JSON, запрет хэштегов в тексте и выбор категории. " + "Менять её стоит только вместе с валидатором райтера." + ), + "safe": "Обычно не трогаем. Если промпт райтера переписывается, свободную часть меняем выше, контракт оставляем стабильным.", + "input": "Контракт склеивается после свободной инструкции и отправляется как system prompt.", + "contract": '{"rewrites":[{"id":123,"category_id":2,"text":"готовый текст до 900 символов без хэштегов; явно упоминает producer_name","notes":"короткая заметка для редактора"}]}', + }, +} + +MODEL_FALLBACKS = { + "openrouter": [ + "openrouter/openai/gpt-4.1-mini", + "openrouter/anthropic/claude-3.5-sonnet", + "openrouter/google/gemini-2.5-flash", + ], + "openai": ["gpt-4.1-mini", "gpt-4o-mini", "o4-mini"], + "anthropic": ["anthropic/claude-haiku-4-5-20251001", "anthropic/claude-sonnet-4-20250514"], + "gemini": ["gemini/gemini-2.5-flash", "gemini/gemini-2.5-pro"], + "openai_compatible": [], +} + +PREFERRED_MODELS = { + "openrouter": [ + "openrouter/anthropic/claude-sonnet-4", + "openrouter/anthropic/claude-3.5-sonnet", + "openrouter/openai/gpt-4.1-mini", + "openrouter/google/gemini-2.5-flash", + ], + "anthropic": [ + "anthropic/claude-haiku-4-5-20251001", + "anthropic/claude-4-sonnet-20250514", + "anthropic/claude-sonnet-4-20250514", + "anthropic/claude-3-7-sonnet-20250219", + "anthropic/claude-3-5-sonnet-20241022", + "anthropic/claude-3-5-haiku-20241022", + ], + "openai": ["gpt-4.1-mini", "gpt-4o-mini", "o4-mini"], + "gemini": ["gemini/gemini-2.5-flash", "gemini/gemini-2.5-pro"], +} + +CATEGORY_TITLES = { + "AI Qualifier": "AI-квалификатор", + "AI Writer": "AI-райтер", + "TG Poster": "TG-постер", + "TG Reactor": "TG-реактор", + "VK Poster": "VK-постер", + "MAX Poster": "MAX-постер", + "Daily Report": "Ежедневный отчет", + "Parser": "Парсер", + "Uploader": "Аплоадер", + "VK": "VK API", + "General": "Общие", +} + +CATEGORY_ORDER = { + "AI Qualifier": 10, + "AI Writer": 20, + "TG Poster": 30, + "TG Reactor": 40, + "VK Poster": 50, + "MAX Poster": 51, + "Site Poster": 52, + "Daily Report": 55, + "Parser": 60, + "VK": 70, + "Uploader": 80, + "General": 100, +} + +AI_WORKER_ENABLED_SETTINGS = { + "ai-qualifier": "ai_qualifier_enabled", + "ai-writer": "ai_writer_enabled", + "tg-poster": "tg_poster_enabled", + "tg-reactor": "tg_reactor_enabled", + "vk-poster": "vk_poster_enabled", + "max-poster": "max_poster_enabled", + "site-poster": "site_poster_enabled", + "daily-report": "daily_report_enabled", +} + +SETTING_ORDER = { + "AI Qualifier": [ + "ai_qualifier_provider", + "ai_qualifier_model", + "ai_qualifier_api_key", + "ai_qualifier_api_base", + "ai_qualifier_prompt", + "ai_qualifier_contract", + "ai_qualifier_batch_size", + "ai_qualifier_min_score", + "ai_qualifier_max_text_chars", + "ai_qualifier_temperature", + "ai_qualifier_timeout_sec", + "ai_qualifier_interval_sec", + ], + "AI Writer": [ + "ai_writer_provider", + "ai_writer_model", + "ai_writer_api_key", + "ai_writer_api_base", + "ai_writer_prompt", + "ai_writer_contract", + "ai_writer_categories", + "ai_writer_batch_size", + "ai_writer_max_text_chars", + "ai_writer_temperature", + "ai_writer_timeout_sec", + "ai_writer_interval_sec", + ], + "TG Poster": [ + "tg_poster_bot_token", + "tg_poster_chat_id", + "tg_poster_interval_sec", + "tg_poster_media_group_max_items", + "tg_poster_caption_limit", + "tg_poster_message_limit", + "tg_poster_max_attempts", + "tg_poster_retry_backoff_max_sec", + "tg_poster_send_delay_sec", + "tg_poster_text_overflow_caption", + "tg_poster_recent_window", + "tg_poster_category_repeat_penalty", + "tg_poster_source_repeat_penalty", + ], + "TG Reactor": [ + "tg_reactor_bot_tokens", + "tg_reactor_reactions_json", + "tg_reactor_since", + "tg_reactor_delay_sec", + "tg_reactor_interval_sec", + "tg_reactor_reaction_pause_sec", + "tg_reactor_react_all_messages", + "tg_reactor_max_attempts", + "tg_reactor_retry_backoff_max_sec", + ], + "VK Poster": [ + "vk_poster_app_id", + "vk_poster_client_secret", + "vk_poster_access_token", + "vk_poster_refresh_token", + "vk_poster_token_device_id", + "vk_poster_token_expires_at", + "vk_poster_owner_id", + "vk_poster_from_group", + "vk_poster_interval_sec", + "vk_poster_timeout_sec", + "vk_poster_max_attempts", + "vk_poster_retry_backoff_max_sec", + "vk_poster_send_delay_sec", + "vk_poster_recent_window", + "vk_poster_category_repeat_penalty", + "vk_poster_source_repeat_penalty", + "vk_poster_dry_run", + ], + "MAX Poster": [ + "max_poster_bot_token", + "max_poster_chat_id", + "max_poster_api_base_url", + "max_poster_interval_sec", + "max_poster_message_limit", + "max_poster_media_max_items", + "max_poster_media_process_delay_sec", + "max_poster_video_ready_attempts", + "max_poster_video_ready_delay_sec", + "max_poster_max_attempts", + "max_poster_retry_backoff_max_sec", + "max_poster_send_delay_sec", + "max_poster_recent_window", + "max_poster_category_repeat_penalty", + "max_poster_source_repeat_penalty", + "max_poster_auto_reaction_enabled", + "max_poster_auto_reaction", + "max_poster_reaction_path_template", + "max_poster_dry_run", + ], + "Site Poster": [ + "site_poster_provider", + "site_poster_base_url", + "site_poster_secret_file", + "site_poster_fallback_title", + "site_poster_interval_sec", + "site_poster_photo_batch_size", + "site_poster_video_batch_size", + "site_poster_ghost_url", + "site_poster_key_file", + ], + "Daily Report": [ + "daily_report_bot_token", + "daily_report_recipient_ids", + "daily_report_time", + "daily_report_interval_sec", + "daily_report_last_sent_date", + ], + "Parser": [ + "parser_interval_sec", + "parser_new_source_lookback_days", + "parser_reparse_overlap_minutes", + "parser_min_text_length", + "parser_skip_empty_text", + "parser_skip_no_media", + "parser_skip_text_too_short", + "parser_skip_reposts", + "parser_store_skipped_posts", + "parser_dedupe_content_hash", + "parser_source_pause_sec", + ], + "VK": [ + "vk_requests_per_second", + "vk_wall_page_size", + "vk_api_timeout_total_sec", + "vk_api_timeout_connect_sec", + "vk_rate_limit_sleep_sec", + "vk_api_retry_attempts", + "vk_api_retry_min_delay_sec", + "vk_api_retry_max_delay_sec", + ], + "Uploader": [ + "tg_media_channel_id", + "local_bot_api_url", + "uploader_interval_sec", + "uploader_download_timeout_sec", + "media_group_max_items", + "media_upload_delay_sec", + "media_post_job_pause_sec", + "max_media_attempts", + "tg_retry_attempts", + "tg_retry_backoff_max_sec", + "video_max_size_mb", + "video_max_height", + "video_max_duration_sec", + "uploader_yt_dlp_timeout_sec", + ], +} + + +def now_utc() -> datetime: + return datetime.now(timezone.utc) + + +def redirect(path: str) -> RedirectResponse: + return RedirectResponse(path, status_code=status.HTTP_303_SEE_OTHER) + + +@app.middleware("http") +async def add_security_headers(request: Request, call_next): + response = await call_next(request) + response.headers.setdefault("X-Frame-Options", "DENY") + response.headers.setdefault("X-Content-Type-Options", "nosniff") + response.headers.setdefault("Referrer-Policy", "same-origin") + response.headers.setdefault("Permissions-Policy", "geolocation=(), microphone=(), camera=()") + return response + + +def should_audit_request(request: Request) -> bool: + path = request.url.path + if path in {"/health", "/favi.png", "/favicon.ico"}: + return False + if path.startswith("/uploads/"): + return False + if path == "/login" and request.method == "GET": + return False + return request.method in {"GET", "POST"} + + +@app.middleware("http") +async def add_request_audit(request: Request, call_next): + user = await get_current_user(request) if should_audit_request(request) else None + response = await call_next(request) + if user: + try: + await audit( + user["id"], + f"request.{request.method.lower()}", + "http_request", + None, + {"status_code": response.status_code}, + request, + ) + except Exception as exc: + logger.warning("Could not write request audit log: {}", exc) + return response + + +def pkce_challenge(verifier: str) -> str: + digest = hashlib.sha256(verifier.encode("ascii")).digest() + return base64.urlsafe_b64encode(digest).decode("ascii").rstrip("=") + + +async def bootstrap_admin() -> None: + """Create or sync the bootstrap admin user from env secrets on every startup.""" + if not settings.admin_bootstrap_login or not settings.admin_bootstrap_password: + return + pool = await get_pool() + login = settings.admin_bootstrap_login.strip().lower() + exists = await pool.fetchval("SELECT 1 FROM admin_users WHERE login=$1", login) + if exists: + # Always sync password from env so rotating the secret takes effect immediately. + await pool.execute( + "UPDATE admin_users SET password_hash=$1, is_active=TRUE, updated_at=NOW() WHERE login=$2", + hash_password(settings.admin_bootstrap_password), + login, + ) + logger.info("Bootstrap admin password synced from env: {}", login) + return + await pool.execute( + """ + INSERT INTO admin_users(login, password_hash, role, is_active) + VALUES($1, $2, 'admin', TRUE) + """, + login, + hash_password(settings.admin_bootstrap_password), + ) + logger.warning("Bootstrap admin user created: {}", login) + + +async def get_current_user(request: Request) -> dict | None: + token = request.cookies.get(COOKIE_NAME) + if not token: + return None + pool = await get_pool() + row = await pool.fetchrow( + """ + SELECT s.csrf_token, u.id, u.login, u.role, u.is_active + FROM admin_sessions s + JOIN admin_users u ON u.id=s.user_id + WHERE s.token_hash=$1 + AND s.expires_at > NOW() + AND u.is_active=TRUE + """, + token_hash(token, settings.app_secret_key), + ) + return dict(row) if row else None + + +def require_csrf(user: dict, csrf_token: str) -> None: + if not user or csrf_token != user["csrf_token"]: + raise PermissionError("bad csrf") + + +def base_context(request: Request, user: dict | None, **extra: Any) -> dict[str, Any]: + ctx = { + "request": request, + "user": user, + "app_env": settings.app_env, + "app_title": settings.display_admin_title, + "site_title": settings.display_site_title, + "site_poster_provider_options": SITE_POSTER_PROVIDER_OPTIONS, + } + ctx.update(extra) + return ctx + + +EDITOR_ALLOWED_PATHS = ( + "/", + "/raw", + "/editor", + "/sources", + "/uploads", + "/logout", + "/login", + "/favi.png", + "/favicon.ico", + "/health", +) + + +def is_editor_allowed_path(path: str) -> bool: + return any(path == allowed or path.startswith(f"{allowed}/") for allowed in EDITOR_ALLOWED_PATHS if allowed != "/") or path == "/" + + +@app.middleware("http") +async def restrict_editor_access(request: Request, call_next): + user = await get_current_user(request) + if user and user.get("role") == "editor" and not is_editor_allowed_path(request.url.path): + if request.method == "GET" and not request.headers.get("hx-request"): + return redirect("/raw") + return JSONResponse({"detail": "Forbidden"}, status_code=status.HTTP_403_FORBIDDEN) + return await call_next(request) + + +def client_ip(request: Request) -> str: + forwarded = request.headers.get("x-forwarded-for", "") + if forwarded: + return forwarded.split(",", 1)[0].strip()[:80] + real_ip = request.headers.get("x-real-ip", "") + if real_ip: + return real_ip.strip()[:80] + return (request.client.host if request.client else "unknown")[:80] + + +def preserved_query(request: Request, exclude: set[str] | None = None) -> list[dict[str, str]]: + exclude = exclude or {"page"} + items = [] + for key, value in request.query_params.multi_items(): + if key not in exclude: + items.append({"key": key, "value": value}) + return items + + +def query_path(request: Request, **replace: str) -> str: + values: list[tuple[str, str]] = [] + excluded = set(replace) | {"page"} + for key, value in request.query_params.multi_items(): + if key not in excluded: + values.append((key, value)) + for key, value in replace.items(): + if value != "": + values.append((key, value)) + encoded = urlencode(values) + return f"{request.url.path}?{encoded}" if encoded else request.url.path + + +def raw_sort_headers(request: Request, current_sort: str) -> dict[str, dict[str, str | bool]]: + columns = { + "id": ("id_asc", "id_desc"), + "source": ("source_asc", "source_desc"), + "stage": ("stage_asc", "stage_desc"), + "post": ("text_asc", "text_desc"), + "ai": ("score_asc", "score_desc"), + } + headers: dict[str, dict[str, str | bool]] = {} + for key, (asc, desc) in columns.items(): + next_sort = desc if current_sort == asc else asc + headers[key] = { + "url": query_path(request, sort=next_sort), + "active": current_sort in {asc, desc}, + "direction": "asc" if current_sort == asc else "desc" if current_sort == desc else "", + } + return headers + + +def parse_date_filter(value: str) -> date | None: + value = (value or "").strip() + if not value: + return None + try: + return datetime.strptime(value, "%Y-%m-%d").date() + except ValueError: + return None + + +def format_dt(value: datetime | None) -> str: + if not value: + return "" + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(LOCAL_TZ).strftime("%d.%m.%y %H:%M") + + +def clamp_int(value: int, min_value: int, max_value: int) -> int: + return max(min_value, min(max_value, int(value))) + + +def pagination(page: int, per_page: int, total: int) -> dict[str, int | bool]: + page = clamp_int(page, 1, 1_000_000) + per_page = clamp_int(per_page, 10, 200) + pages = max(1, (int(total) + per_page - 1) // per_page) + page = min(page, pages) + return { + "page": page, + "per_page": per_page, + "total": int(total), + "pages": pages, + "offset": (page - 1) * per_page, + "has_prev": page > 1, + "has_next": page < pages, + } + + +FILTER_OPS = { + "text": [ + {"value": "contains", "label": "содержит"}, + {"value": "eq", "label": "равно"}, + {"value": "neq", "label": "не равно"}, + ], + "enum": [ + {"value": "eq", "label": "равно"}, + {"value": "neq", "label": "не равно"}, + ], + "number": [ + {"value": "eq", "label": "="}, + {"value": "gte", "label": ">="}, + {"value": "lte", "label": "<="}, + ], + "date": [ + {"value": "gte", "label": "от"}, + {"value": "lte", "label": "до"}, + {"value": "eq", "label": "день"}, + ], +} +FILTER_OP_OPTIONS = [ + {"value": "contains", "label": "содержит"}, + {"value": "eq", "label": "равно / день"}, + {"value": "neq", "label": "не равно"}, + {"value": "gte", "label": ">= / от"}, + {"value": "lte", "label": "<= / до"}, +] + +SOURCE_FIELD_SPECS = { + "id": {"label": "ID", "expr": "s.id", "type": "number"}, + "platform": {"label": "Площадка", "expr": "s.platform", "type": "enum"}, + "name": {"label": "Название", "expr": "s.name", "type": "text"}, + "tag": {"label": "Тэг", "expr": "COALESCE(s.tag,'')", "type": "text"}, + "url": {"label": "Ссылка", "expr": "s.url", "type": "text"}, + "external_id": {"label": "VK id", "expr": "COALESCE(s.external_id,'')", "type": "text"}, + "active": {"label": "Включён", "expr": "s.active", "type": "enum"}, + "status": {"label": "Статус", "expr": "s.status", "type": "enum"}, + "posts_count": { + "label": "Постов", + "expr": "(SELECT COUNT(*) FROM raw_posts rp2 WHERE rp2.source_id=s.id)", + "type": "number", + }, + "posts_24h": { + "label": "Постов за 24ч", + "expr": "(SELECT COUNT(*) FROM raw_posts rp3 WHERE rp3.source_id=s.id AND rp3.created_at > NOW() - INTERVAL '24 hours')", + "type": "number", + }, + "last_parsed_at": {"label": "Последний парсинг", "expr": "timezone('Asia/Yekaterinburg', s.last_parsed_at)::date", "sort_expr": "s.last_parsed_at", "type": "date"}, + "created_at": {"label": "Создан", "expr": "timezone('Asia/Yekaterinburg', s.created_at)::date", "sort_expr": "s.created_at", "type": "date"}, +} + +RAW_FIELD_SPECS = { + "id": {"label": "ID", "expr": "rp.id", "type": "number"}, + "source_name": {"label": "Источник", "expr": "s.name", "type": "text"}, + "source_tag": {"label": "Тэг источника", "expr": "COALESCE(s.tag,'')", "type": "text"}, + "platform": {"label": "Площадка", "expr": "rp.platform", "type": "enum"}, + "status": {"label": "Raw статус", "expr": "rp.status", "type": "enum"}, + "original_url": {"label": "Оригинал", "expr": "rp.original_url", "type": "text"}, + "text": {"label": "Текст", "expr": "rp.raw_text", "type": "text"}, + "media_count": { + "label": "Медиа", + "expr": "(SELECT COUNT(*) FROM raw_post_media rmf WHERE rmf.raw_post_id=rp.id)", + "type": "number", + }, + "posted_at": {"label": "Дата VK", "expr": "timezone('Asia/Yekaterinburg', rp.posted_at)::date", "sort_expr": "rp.posted_at", "type": "date"}, + "created_at": {"label": "Дата загрузки", "expr": "timezone('Asia/Yekaterinburg', rp.created_at)::date", "sort_expr": "rp.created_at", "type": "date"}, + "qualification_status": {"label": "Итог AI", "expr": "COALESCE(rp.qualification_status,'pending')", "type": "enum"}, + "qualification_model_decision": {"label": "Решение модели", "expr": "COALESCE(rp.qualification_model_decision, rp.qualification_decision, '')", "type": "enum"}, + "qualification_score": {"label": "Оценка", "expr": "rp.qualification_score", "type": "number"}, + "qualification_reject_tag": {"label": "Reject tag", "expr": "COALESCE(rp.qualification_reject_tag,'')", "type": "enum"}, + "rewrite_status": {"label": "Статус рерайта", "expr": "COALESCE(rp.rewrite_status,'pending')", "type": "enum"}, + "rewrite_category": {"label": "Категория", "expr": "COALESCE(rp.rewrite_category,'')", "type": "enum"}, + "rewrite_source_tag": {"label": "Хэштег источника", "expr": "COALESCE(rp.rewrite_source_tag,'')", "type": "text"}, +} + +EDITOR_STATUS_OPTIONS = [ + {"value": "review", "label": "На проверке"}, + {"value": "edited", "label": "Отредактировано"}, + {"value": "accepted", "label": "Принято"}, + {"value": "published", "label": "Опубликовано"}, + {"value": "publish_failed", "label": "Ошибка публикации"}, + {"value": "rejected", "label": "Отклонено"}, +] + + +def field_options(specs: dict[str, dict[str, str]]) -> list[dict[str, str]]: + return [{"value": key, "label": spec["label"], "type": spec["type"]} for key, spec in specs.items()] + + +def parse_table_filters(request: Request, specs: dict[str, dict[str, str]], slots: int = 4) -> list[dict[str, str]]: + qp = request.query_params + fields = qp.getlist("f_field") + ops = qp.getlist("f_op") + values = qp.getlist("f_value") + rows: list[dict[str, str]] = [] + for i in range(max(slots, len(fields))): + field = fields[i] if i < len(fields) else "" + op = ops[i] if i < len(ops) else "" + value = values[i] if i < len(values) else "" + if field not in specs: + field = "" + field_type = specs[field]["type"] if field else "text" + allowed_ops = {item["value"] for item in FILTER_OPS[field_type]} + if op not in allowed_ops: + op = "contains" if field_type == "text" else "eq" + rows.append({"field": field, "op": op, "value": value}) + return rows + + +def parse_table_sorts(request: Request, specs: dict[str, dict[str, str]], slots: int = 3) -> list[dict[str, str]]: + qp = request.query_params + fields = qp.getlist("sort_field") + dirs = qp.getlist("sort_dir") + rows: list[dict[str, str]] = [] + for i in range(max(slots, len(fields))): + field = fields[i] if i < len(fields) else "" + direction = dirs[i] if i < len(dirs) else "desc" + if field not in specs: + field = "" + if direction not in {"asc", "desc"}: + direction = "desc" + rows.append({"field": field, "dir": direction}) + return rows + + +def build_filter_sql(filters: list[dict[str, str]], specs: dict[str, dict[str, str]], start_index: int = 1) -> tuple[list[str], list[Any]]: + clauses: list[str] = [] + args: list[Any] = [] + index = start_index + for item in filters: + field = item.get("field") or "" + value = str(item.get("value") or "").strip() + if not field or field not in specs or value == "": + continue + spec = specs[field] + expr = spec["expr"] + field_type = spec["type"] + op = item.get("op") or ("contains" if field_type == "text" else "eq") + if field_type == "text": + if op == "eq": + clauses.append(f"{expr} = ${index}") + args.append(value) + elif op == "neq": + clauses.append(f"{expr} <> ${index}") + args.append(value) + else: + clauses.append(f"{expr} ILIKE '%' || ${index} || '%'") + args.append(value) + elif field_type == "enum": + if op == "neq": + clauses.append(f"{expr} <> ${index}") + else: + clauses.append(f"{expr} = ${index}") + if value.lower() in {"true", "false"}: + args.append(value.lower() == "true") + else: + args.append(value) + elif field_type == "number": + try: + number_value = int(value) + except ValueError: + continue + sign = {"gte": ">=", "lte": "<=", "neq": "<>", "eq": "="}.get(op, "=") + clauses.append(f"{expr} {sign} ${index}") + args.append(number_value) + elif field_type == "date": + sign = {"gte": ">=", "lte": "<=", "eq": "="}.get(op, "=") + clauses.append(f"{expr} {sign} ${index}::date") + args.append(value) + index += 1 + return clauses, args + + +def build_sort_sql(sorts: list[dict[str, str]], specs: dict[str, dict[str, str]], default_sql: str) -> str: + parts: list[str] = [] + seen: set[str] = set() + for item in sorts: + field = item.get("field") or "" + if not field or field not in specs or field in seen: + continue + spec = specs[field] + expr = spec.get("sort_expr") or spec["expr"] + direction = "ASC" if item.get("dir") == "asc" else "DESC" + nulls = " NULLS LAST" if direction == "DESC" else " NULLS FIRST" + parts.append(f"{expr} {direction}{nulls}") + seen.add(field) + if parts: + if "id" not in seen and "id" in specs: + parts.append(f"{specs['id'].get('sort_expr') or specs['id']['expr']} DESC") + return ", ".join(part for part in parts if part) + return default_sql + + +def selected_values(request: Request, name: str) -> list[str]: + return [str(value).strip() for value in request.query_params.getlist(name) if str(value).strip()] + + +def selected_int_values(request: Request, name: str) -> list[int]: + values: list[int] = [] + for value in selected_values(request, name): + try: + values.append(int(value)) + except ValueError: + continue + return values + + +def add_where(clauses: list[str], args: list[Any], sql_template: str, value: Any) -> None: + args.append(value) + clauses.append(sql_template.format(i=len(args))) + + +async def raw_filter_facets(pool) -> dict[str, Any]: + source_rows = await pool.fetch( + """ + SELECT s.id, s.name, COALESCE(s.tag, '') AS tag, COUNT(rp.id) AS count + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + GROUP BY s.id, s.name, s.tag + ORDER BY COUNT(rp.id) DESC, s.name ASC + """ + ) + category_rows = await pool.fetch( + """ + SELECT COALESCE(final_category, rewrite_category, '') AS value, COUNT(*) AS count + FROM raw_posts + WHERE COALESCE(final_category, rewrite_category, '') <> '' + GROUP BY COALESCE(final_category, rewrite_category, '') + ORDER BY COUNT(*) DESC, value ASC + """ + ) + status_rows = await pool.fetch("SELECT status AS value, COUNT(*) AS count FROM raw_posts GROUP BY status ORDER BY count DESC, value") + qualification_rows = await pool.fetch( + """ + SELECT COALESCE(qualification_status, 'pending') AS value, COUNT(*) AS count + FROM raw_posts + GROUP BY COALESCE(qualification_status, 'pending') + ORDER BY count DESC, value + """ + ) + rewrite_rows = await pool.fetch( + """ + SELECT COALESCE(rewrite_status, 'pending') AS value, COUNT(*) AS count + FROM raw_posts + GROUP BY COALESCE(rewrite_status, 'pending') + ORDER BY count DESC, value + """ + ) + return { + "sources": [dict(row) for row in source_rows], + "categories": [dict(row) for row in category_rows], + "statuses": [dict(row) for row in status_rows], + "qualification_statuses": [dict(row) for row in qualification_rows], + "rewrite_statuses": [dict(row) for row in rewrite_rows], + } + + +async def editor_filter_facets(pool) -> dict[str, Any]: + source_rows = await pool.fetch( + """ + SELECT s.id, s.name, COALESCE(s.tag, '') AS tag, COUNT(rp.id) AS count + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + WHERE rp.rewrite_status='ready' + GROUP BY s.id, s.name, s.tag + ORDER BY COUNT(rp.id) DESC, s.name ASC + """ + ) + category_rows = await pool.fetch( + """ + SELECT COALESCE(final_category, rewrite_category, '') AS value, COUNT(*) AS count + FROM raw_posts + WHERE rewrite_status='ready' + AND COALESCE(final_category, rewrite_category, '') <> '' + GROUP BY COALESCE(final_category, rewrite_category, '') + ORDER BY COUNT(*) DESC, value ASC + """ + ) + status_rows = await pool.fetch( + """ + SELECT COALESCE(editorial_status, 'review') AS value, COUNT(*) AS count + FROM raw_posts + WHERE rewrite_status='ready' + GROUP BY COALESCE(editorial_status, 'review') + ORDER BY count DESC, value + """ + ) + return { + "sources": [dict(row) for row in source_rows], + "categories": [dict(row) for row in category_rows], + "statuses": [dict(row) for row in status_rows], + } + + +def parse_source_line(line: str) -> dict[str, str]: + raw = (line or "").strip() + if not raw: + return {"name": "", "tag": "", "url": "", "error": "Пустая строка"} + parts = re.split(r"\s+", raw) + url_index = next((i for i, part in enumerate(parts) if "vk.com/" in part or part.startswith("club")), -1) + if url_index < 0: + if len(parts) == 1: + value = parts[0].strip() + url = value if value.startswith("http") else f"https://vk.com/{value}" + return {"name": "", "tag": "", "url": url, "error": ""} + return {"name": "", "tag": "", "url": "", "error": "Не нашёл ссылку VK"} + url = parts[url_index].strip() + before_url = parts[:url_index] + if len(before_url) >= 2: + tag = before_url[-1].strip() + name = " ".join(before_url[:-1]).strip() + elif len(before_url) == 1: + name = before_url[0].strip() + tag = before_url[0].strip() + else: + name = "" + tag = "" + if url.startswith("vk.com/"): + url = f"https://{url}" + return {"name": name, "tag": normalize_hash_tag(tag, ""), "url": url, "error": ""} + + +async def build_sources_preview(lines_text: str, default_active: bool = True) -> list[dict[str, Any]]: + lines = [line for line in (lines_text or "").splitlines() if line.strip()] + parsed = [parse_source_line(line) for line in lines] + normalized_values = [normalize_vk_source(item["url"]) for item in parsed if item.get("url")] + pool = await get_pool() + existing_rows = await pool.fetch( + """ + SELECT lower(external_id) AS external_id, lower(url) AS url, lower(COALESCE(tag,'')) AS tag + FROM sources + WHERE archived_at IS NULL + AND (lower(external_id)=ANY($1::text[]) OR lower(url)=ANY($2::text[]) OR lower(COALESCE(tag,''))=ANY($3::text[])) + """, + [v.lower() for v in normalized_values if v], + [str(item.get("url") or "").lower() for item in parsed], + [normalize_hash_tag(str(item.get("tag") or ""), "").lower() for item in parsed if item.get("tag")], + ) + existing_ids = {str(row["external_id"] or "").lower() for row in existing_rows} + existing_urls = {str(row["url"] or "").lower() for row in existing_rows} + existing_tags = {str(row["tag"] or "").lower() for row in existing_rows} + + seen: set[str] = set() + seen_tags: set[str] = set() + async with VKAPIClient(rps=2, timeout_total_sec=12, timeout_connect_sec=5, retry_attempts=2) as client: + preview = [] + for idx, item in enumerate(parsed, start=1): + url = item.get("url") or "" + external_id = normalize_vk_source(url) if url else "" + row = { + "line_no": idx, + "platform": PLATFORM_VK, + "name": item.get("name") or "", + "tag": normalize_hash_tag(item.get("tag") or external_id, external_id or "source"), + "url": url, + "external_id": external_id, + "external_owner_id": None, + "active": default_active, + "ok": False, + "error": item.get("error") or "", + } + key = external_id.lower() + if not row["error"] and key in seen: + row["error"] = "Дубль в этом списке" + if not row["error"] and row["tag"].lower() in seen_tags: + row["error"] = "Дубль тэга в этом списке" + if not row["error"] and (key in existing_ids or url.lower() in existing_urls): + row["error"] = "Уже есть в источниках" + if not row["error"] and row["tag"].lower() in existing_tags: + row["error"] = "Такой тэг уже есть" + if not row["error"] and not external_id: + row["error"] = "Не удалось разобрать VK-ссылку" + if not row["error"]: + try: + screen_name, owner_id, resolved_name = await client.resolve_group(url) + row["external_id"] = screen_name + row["external_owner_id"] = owner_id + row["name"] = row["name"] or resolved_name + row["tag"] = normalize_hash_tag(row["tag"] or screen_name, screen_name) + row["url"] = f"https://vk.com/{screen_name}" + row["ok"] = True + seen.add(key) + seen_tags.add(row["tag"].lower()) + except Exception as exc: + row["error"] = str(exc) + preview.append(row) + return preview + + +def pipeline_stage(post: dict[str, Any]) -> dict[str, str]: + status = str(post.get("status") or "") + qualification_status = str(post.get("qualification_status") or "").strip().lower() + if status == "failed": + return {"key": "failed", "label": "Ошибка", "class": "bad"} + if status == "skipped": + return {"key": "skipped", "label": "Пропущен", "class": "muted"} + if status in {"raw_saved", "storage_pending"}: + return {"key": status, "label": "Медиа", "class": ""} + if status != "storage_ready": + return {"key": status, "label": status or "Raw", "class": ""} + rewrite_status = str(post.get("rewrite_status") or "").strip().lower() + if rewrite_status == "ready": + return {"key": "rewrite_ready", "label": "Рерайт готов", "class": "ok"} + if rewrite_status == "processing": + return {"key": "rewrite_processing", "label": "AI пишет", "class": "warn"} + if rewrite_status == "failed": + return {"key": "rewrite_failed", "label": "Рерайт ошибка", "class": "bad"} + if qualification_status in {"accepted"}: + return {"key": "qualified_accepted", "label": "AI принят", "class": "ok"} + if qualification_status in {"rejected"}: + return {"key": "qualified_rejected", "label": "AI отклонён", "class": "bad"} + if qualification_status == "processing": + return {"key": "qualification_processing", "label": "AI проверяет", "class": "warn"} + if qualification_status == "failed": + return {"key": "qualification_failed", "label": "AI ошибка", "class": "bad"} + return {"key": "qualification_pending", "label": "Ждёт AI", "class": ""} + + +def ai_badge(post: dict[str, Any]) -> dict[str, str]: + status = str(post.get("qualification_status") or "pending").strip().lower() + score = post.get("qualification_score") + decision = str(post.get("qualification_decision") or status or "pending") + reason = str(post.get("qualification_reason") or "") + if isinstance(score, int): + if score >= 8: + css = "ok" + elif score >= 5: + css = "warn" + else: + css = "bad" + label = f"{score}/10" + sublabel = decision + elif status == "processing": + css = "warn" + label = "..." + sublabel = "processing" + elif status == "failed": + css = "bad" + label = "!" + sublabel = "failed" + else: + css = "" + label = "AI" + sublabel = "pending" + return {"class": css, "label": label, "sublabel": sublabel, "reason": reason} + + +def prepare_raw_post(row: Any) -> dict[str, Any]: + post = dict(row) + media_items = post.get("media_items") or [] + if isinstance(media_items, str): + try: + media_items = json.loads(media_items) + except json.JSONDecodeError: + media_items = [] + post["media_items"] = media_items + post["posted_at_fmt"] = format_dt(post.get("posted_at")) + post["created_at_fmt"] = format_dt(post.get("created_at")) + post["qualified_at_fmt"] = format_dt(post.get("qualified_at")) + post["stage"] = pipeline_stage(post) + post["ai_badge"] = ai_badge(post) + return post + + +def prepare_editor_post(row: Any) -> dict[str, Any]: + post = prepare_raw_post(row) + category_name = post.get("final_category") or post.get("rewrite_category") or "" + category_tag = post.get("resolved_category_tag") or post.get("final_category_tag") or post.get("rewrite_category_tag") or category_name + source_tag = post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "" + post["review_text"] = post.get("final_text") or post.get("rewritten_text") or "" + post["publication_preview_text"] = build_publication_text(post["review_text"], category_tag, source_tag) + post["review_category"] = category_name + post["review_category_tag"] = category_tag + post["review_source_tag"] = source_tag + publication_status = post.get("publication_status") or "pending" + if publication_status == "published": + post["publication_status_label"] = "опубликовано" + post["publication_status_class"] = "ok" + elif publication_status == "publish_failed": + post["publication_status_label"] = "ошибка публикации" + post["publication_status_class"] = "bad" + else: + post["publication_status_label"] = "не опубликовано" + post["publication_status_class"] = "" + platform_rows = post.get("platform_publications") or [] + if isinstance(platform_rows, str): + try: + platform_rows = json.loads(platform_rows) + except json.JSONDecodeError: + platform_rows = [] + platforms: dict[str, dict[str, Any]] = {} + for item in platform_rows or []: + if not isinstance(item, dict) or not item.get("platform"): + continue + platforms[str(item["platform"])] = item + if "tg" not in platforms and post.get("publication_status") and post.get("publication_status") != "pending": + platforms["tg"] = { + "platform": "tg", + "status": post.get("publication_status"), + "url": post.get("tg_publication_url"), + "error": post.get("publication_error"), + } + labels = {"tg": "TG", "vk": "VK", "max": "MAX", "site": "Сайт"} + status_labels = { + "published": "опубликовано", + "publish_failed": "ошибка", + "pending": "ожидает", + } + status_classes = {"published": "ok", "publish_failed": "bad", "pending": ""} + post["publication_platforms"] = [ + { + **platforms.get(platform, {"platform": platform, "status": "pending"}), + "label": label, + "status_label": status_labels.get(str(platforms.get(platform, {}).get("status") or "pending"), str(platforms.get(platform, {}).get("status") or "pending")), + "class": status_classes.get(str(platforms.get(platform, {}).get("status") or "pending"), ""), + } + for platform, label in labels.items() + ] + post["editorial_status_label"] = next( + (item["label"] for item in EDITOR_STATUS_OPTIONS if item["value"] == (post.get("editorial_status") or "review")), + post.get("editorial_status") or "review", + ) + post["reviewed_at_fmt"] = format_dt(post.get("reviewed_at")) + post["edited_at_fmt"] = format_dt(post.get("edited_at")) + return post + + +async def resolve_category_for_editor(name: str, category_id: int | None = None) -> dict[str, Any]: + pool = await get_pool() + row = None + if category_id is not None: + row = await pool.fetchrow( + """ + SELECT sort_order AS id, name, tag + FROM content_categories + WHERE sort_order=$1 + LIMIT 1 + """, + category_id, + ) + if row is None: + raw_name = (name or "").strip().strip("#") + row = await pool.fetchrow( + """ + SELECT sort_order AS id, name, tag + FROM content_categories + WHERE lower(name)=lower($1) OR lower(tag)=lower($1) + ORDER BY is_active DESC, sort_order, name + LIMIT 1 + """, + raw_name, + ) + if row: + return {"id": int(row["id"]), "name": str(row["name"]), "tag": normalize_hash_tag(str(row["tag"]), "category")} + fallback_name = (name or "").strip().strip("#") + return {"id": category_id, "name": fallback_name, "tag": normalize_hash_tag(fallback_name, "category")} + + +def uploaded_editor_media_path(url: str) -> Path | None: + raw = str(url or "").strip() + prefix = "/uploads/editor_media/" + if not raw.startswith(prefix): + return None + name = Path(raw.removeprefix(prefix)).name + if not name: + return None + path = (EDITOR_MEDIA_DIR / name).resolve() + try: + path.relative_to(EDITOR_MEDIA_DIR.resolve()) + except ValueError: + return None + return path + + +async def writer_categories() -> list[str]: + pool = await get_pool() + rows = await pool.fetch( + """ + SELECT name + FROM content_categories + WHERE is_active=TRUE + ORDER BY sort_order, name + """ + ) + categories = [str(row["name"]) for row in rows] + if categories: + return categories + legacy_categories = parse_categories(await fetch_setting("ai_writer_categories", [])) + return legacy_categories or [ + "защита", + "одежда", + "разгрузка", + "рюкзаки", + "airsoft", + "патчи", + "электроника", + "аксессуары", + "производство", + ] + + +async def writer_category_options() -> list[dict[str, str]]: + pool = await get_pool() + rows = await pool.fetch( + """ + SELECT name, tag + FROM content_categories + WHERE is_active=TRUE + ORDER BY sort_order, name + """ + ) + options = [ + {"name": str(row["name"]), "tag": normalize_hash_tag(str(row["tag"]), "category")} + for row in rows + ] + if options: + return options + return [{"name": name, "tag": normalize_hash_tag(name, "category")} for name in await writer_categories()] + + +def normalize_tg_schedule(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list): + value = [] + rows: list[dict[str, Any]] = [] + for idx, item in enumerate(value): + if not isinstance(item, dict): + continue + row_id = str(item.get("id") or f"slot-{idx + 1}").strip()[:80] + time_value = str(item.get("time") or "").strip() + count = item.get("count") or 1 + try: + count_int = clamp_int(int(count), 1, 20) + except Exception: + count_int = 1 + if not re.fullmatch(r"(?:[01]\d|2[0-3]):[0-5]\d", time_value): + time_value = "10:00" + rows.append({"id": row_id, "time": time_value, "count": count_int, "enabled": bool(item.get("enabled", True))}) + return sorted(rows, key=lambda row: row["time"]) + + +async def tg_poster_schedule_rows() -> list[dict[str, Any]]: + value = await fetch_setting("tg_poster_schedule_json", []) + rows = normalize_tg_schedule(value) + if rows: + return rows + return [ + {"id": "morning", "time": "10:00", "count": 1, "enabled": True}, + {"id": "evening", "time": "18:00", "count": 1, "enabled": True}, + ] + + +async def vk_poster_schedule_rows() -> list[dict[str, Any]]: + value = await fetch_setting("vk_poster_schedule_json", []) + rows = normalize_tg_schedule(value) + if rows: + return rows + return [ + {"id": "vk-morning", "time": "10:05", "count": 1, "enabled": True}, + {"id": "vk-evening", "time": "18:05", "count": 1, "enabled": True}, + ] + + +async def save_tg_poster_schedule(rows: list[dict[str, Any]]) -> None: + pool = await get_pool() + await pool.execute( + """ + INSERT INTO app_settings(key, value_json, value_type, title, description, category) + VALUES( + 'tg_poster_schedule_json', + $1::jsonb, + 'json', + 'TG publication schedule', + 'EKB time schedule rows: id, time HH:MM, count, enabled.', + 'TG Poster' + ) + ON CONFLICT (key) DO UPDATE + SET value_json=EXCLUDED.value_json, + updated_at=NOW() + """, + json.dumps(normalize_tg_schedule(rows), ensure_ascii=False), + ) + + +async def save_vk_poster_schedule(rows: list[dict[str, Any]]) -> None: + pool = await get_pool() + await pool.execute( + """ + INSERT INTO app_settings(key, value_json, value_type, title, description, category) + VALUES( + 'vk_poster_schedule_json', + $1::jsonb, + 'json', + 'VK publication schedule', + 'EKB time schedule rows: id, time HH:MM, count, enabled.', + 'VK Poster' + ) + ON CONFLICT (key) DO UPDATE + SET value_json=EXCLUDED.value_json, + updated_at=NOW() + """, + json.dumps(normalize_tg_schedule(rows), ensure_ascii=False), + ) + + +async def writer_category_payload() -> list[dict[str, Any]]: + pool = await get_pool() + rows = await pool.fetch( + """ + SELECT sort_order AS id, name, tag + FROM content_categories + WHERE is_active=TRUE + ORDER BY sort_order, name + """ + ) + categories = [{"id": int(row["id"]), "name": str(row["name"]), "tag": str(row["tag"])} for row in rows] + if categories: + return categories + return [ + {"id": idx + 1, "name": name, "tag": normalize_hash_tag(name, "category")} + for idx, name in enumerate(await writer_categories()) + ] + + +async def category_rows() -> list[dict[str, Any]]: + pool = await get_pool() + rows = await pool.fetch( + """ + SELECT id, name, tag, site_name, site_slug, site_enabled, + is_active, sort_order, created_at, updated_at + FROM content_categories + ORDER BY is_active DESC, sort_order, name + """ + ) + result = [] + for row in rows: + item = dict(row) + item["created_at_fmt"] = format_dt(item.get("created_at")) + item["updated_at_fmt"] = format_dt(item.get("updated_at")) + result.append(item) + return result + + +PROMPT_TEST_STATUS_OPTIONS = [ + {"value": "all", "label": "Все"}, + {"value": "qualified", "label": "Оцененные"}, + {"value": "accepted", "label": "Принятые"}, + {"value": "maybe", "label": "Спорные"}, + {"value": "rejected", "label": "Отклоненные"}, + {"value": "pending", "label": "Ждут AI"}, +] + + +def normalize_prompt_test_status(value: str | None) -> str: + value = str(value or "all").strip().lower() + allowed = {item["value"] for item in PROMPT_TEST_STATUS_OPTIONS} + return value if value in allowed else "all" + + +def prompt_test_status_where(status_filter: str) -> str: + if status_filter == "qualified": + return "AND rp.qualification_status IN ('accepted', 'maybe', 'rejected')" + if status_filter in {"accepted", "maybe", "rejected", "pending"}: + return f"AND rp.qualification_status = '{status_filter}'" + return "" + + +async def prompt_test_posts( + selected_post_id: int | None = None, + status_filter: str = "all", +) -> tuple[list[dict[str, Any]], dict[str, Any] | None]: + pool = await get_pool() + status_filter = normalize_prompt_test_status(status_filter) + where_sql = prompt_test_status_where(status_filter) + rows = await pool.fetch( + f""" + SELECT rp.id, rp.raw_text, rp.original_url, rp.posted_at, rp.created_at, rp.qualification_status, + rp.qualification_score, s.name AS source_name, s.tag AS source_tag, + rp.qualification_reason, + (SELECT COUNT(*) FROM raw_post_media m WHERE m.raw_post_id=rp.id) AS media_count, + (SELECT ARRAY_AGG(DISTINCT m.media_type ORDER BY m.media_type) + FROM raw_post_media m WHERE m.raw_post_id=rp.id) AS media_types + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + WHERE TRUE + {where_sql} + ORDER BY rp.created_at DESC, rp.id DESC + LIMIT 150 + """ + ) + posts = [] + for row in rows: + item = dict(row) + text = str(item.get("raw_text") or "").strip() + item["created_at_fmt"] = format_dt(item.get("created_at")) + item["posted_at_fmt"] = format_dt(item.get("posted_at")) + item["label"] = f"#{item['id']} · {item.get('source_name') or 'source'} · {item['created_at_fmt']}" + item["snippet"] = text[:180] + ("..." if len(text) > 180 else "") + posts.append(item) + selected = next((item for item in posts if int(item["id"]) == int(selected_post_id or 0)), None) + if not selected and selected_post_id: + row = await pool.fetchrow( + """ + SELECT rp.id, rp.raw_text, rp.original_url, rp.posted_at, rp.created_at, rp.qualification_status, + rp.qualification_score, s.name AS source_name, s.tag AS source_tag, + rp.qualification_reason, + (SELECT COUNT(*) FROM raw_post_media m WHERE m.raw_post_id=rp.id) AS media_count, + (SELECT ARRAY_AGG(DISTINCT m.media_type ORDER BY m.media_type) + FROM raw_post_media m WHERE m.raw_post_id=rp.id) AS media_types + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + WHERE rp.id=$1 + """, + selected_post_id, + ) + if row: + selected = dict(row) + text = str(selected.get("raw_text") or "").strip() + selected["created_at_fmt"] = format_dt(selected.get("created_at")) + selected["posted_at_fmt"] = format_dt(selected.get("posted_at")) + selected["label"] = f"#{selected['id']} · {selected.get('source_name') or 'source'} · {selected['created_at_fmt']}" + selected["snippet"] = text[:180] + ("..." if len(text) > 180 else "") + posts.insert(0, selected) + if not selected and posts: + selected = posts[0] + return posts, selected + + +def prompt_test_payload(post: dict[str, Any], max_text_chars: int) -> dict[str, Any]: + return { + "task": "rewrite_accepted_posts", + "categories": [], + "posts": [ + { + "id": int(post["id"]), + "producer_name": post.get("source_name") or "", + "producer_tag": normalize_hash_tag(post.get("source_tag") or post.get("source_name") or "", "source"), + "original_url": post.get("original_url") or "", + "qualification_score": post.get("qualification_score"), + "qualification_reason": post.get("qualification_reason") or "", + "media_count": int(post.get("media_count") or 0), + "media_types": list(post.get("media_types") or []), + "text": str(post.get("raw_text") or "")[:max_text_chars], + } + ], + } + + +def prompt_test_first_rewrite(parsed: Any) -> dict[str, Any] | None: + if not isinstance(parsed, dict): + return None + rewrites = parsed.get("rewrites") + if not isinstance(rewrites, list) and isinstance(parsed.get("data"), dict): + rewrites = parsed["data"].get("rewrites") + if not isinstance(rewrites, list) or not rewrites: + return None + first = rewrites[0] + return first if isinstance(first, dict) else None + + +def prompt_test_preview_text(parsed: Any) -> str: + rewrite = prompt_test_first_rewrite(parsed) + return str((rewrite or {}).get("text") or "").strip() + + +def prompt_test_preview_category(parsed: Any) -> str: + rewrite = prompt_test_first_rewrite(parsed) + if not rewrite: + return "" + category_id = rewrite.get("category_id") + if category_id not in {None, ""}: + return f"category_id: {category_id}" + return str(rewrite.get("category") or "").strip() + + +def prompt_test_preview_notes(parsed: Any) -> str: + rewrite = prompt_test_first_rewrite(parsed) + notes = (rewrite or {}).get("notes") + return "" if notes in {None, ""} else str(notes).strip() + + +def setting_value(settings_rows: list[dict], key: str, default: Any = None) -> Any: + for row in settings_rows: + if row.get("key") == key: + return row.get("value_json", default) + return default + + +def setting_sort_key(item: dict) -> tuple[int, int, str]: + category = str(item.get("category") or "General") + key = str(item.get("key") or "") + keys = SETTING_ORDER.get(category, []) + pos = keys.index(key) if key in keys else 1000 + return (CATEGORY_ORDER.get(category, 999), pos, key) + + +def model_label(model_id: str, name: str | None = None, context: int | None = None) -> str: + label = name or model_id + if context: + label = f"{label} · ctx {context}" + return label + + +async def fetch_openrouter_models() -> list[dict[str, str]]: + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=8)) as session: + async with session.get("https://openrouter.ai/api/v1/models") as resp: + resp.raise_for_status() + data = await resp.json() + models = [] + for item in data.get("data", []): + model_id = str(item.get("id") or "").strip() + if not model_id: + continue + context = item.get("context_length") + models.append( + { + "value": f"openrouter/{model_id}", + "label": model_label(model_id, item.get("name"), context if isinstance(context, int) else None), + } + ) + return models + + +async def fetch_anthropic_models(api_key: str) -> list[dict[str, str]]: + if not api_key: + return [] + headers = { + "x-api-key": api_key, + "anthropic-version": "2023-06-01", + } + models: list[dict[str, str]] = [] + url = "https://api.anthropic.com/v1/models" + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=10)) as session: + for _ in range(5): + async with session.get(url, headers=headers) as resp: + resp.raise_for_status() + data = await resp.json() + for item in data.get("data", []): + model_id = str(item.get("id") or "").strip() + if not model_id: + continue + display_name = item.get("display_name") or model_id + models.append({"value": f"anthropic/{model_id}", "label": f"{display_name} · live Anthropic"}) + if not data.get("has_more"): + break + last_id = data.get("last_id") + if not last_id: + break + url = f"https://api.anthropic.com/v1/models?after_id={last_id}" + return models + + +async def fetch_openai_models(api_key: str) -> list[dict[str, str]]: + if not api_key: + return [] + headers = {"Authorization": f"Bearer {api_key}"} + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=10)) as session: + async with session.get("https://api.openai.com/v1/models", headers=headers) as resp: + resp.raise_for_status() + data = await resp.json() + models = [] + for item in data.get("data", []): + model_id = str(item.get("id") or "").strip() + if not model_id: + continue + if model_id.startswith(("gpt-", "o1", "o3", "o4")): + models.append({"value": model_id, "label": f"{model_id} · live OpenAI"}) + return sorted(models, key=lambda item: item["value"]) + + +def litellm_catalog_models(provider: str) -> list[dict[str, str]]: + try: + import litellm + except Exception: + return [] + + catalog = getattr(litellm, "model_cost", {}) or {} + models: list[dict[str, str]] = [] + for model_id, meta in catalog.items(): + if not isinstance(model_id, str): + continue + normalized = model_id + if provider == "openai": + if "/" in model_id or not model_id.startswith(("gpt-", "o1", "o3", "o4")): + continue + elif provider == "anthropic": + if model_id.startswith("anthropic/"): + normalized = model_id + elif model_id.startswith("claude"): + normalized = f"anthropic/{model_id}" + else: + continue + elif provider == "gemini": + if model_id.startswith("gemini/"): + normalized = model_id + elif model_id.startswith("gemini"): + normalized = f"gemini/{model_id}" + else: + continue + else: + continue + context = meta.get("max_input_tokens") if isinstance(meta, dict) else None + models.append( + { + "value": normalized, + "label": f"{model_label(normalized, context=context if isinstance(context, int) else None)} · LiteLLM catalog", + } + ) + unique = {m["value"]: m for m in models} + preferred = PREFERRED_MODELS.get(provider, []) + + def sort_key(item: dict[str, str]) -> tuple[int, str]: + value = item["value"] + if value in preferred: + return (preferred.index(value), "") + return (1000, item["label"].lower()) + + return sorted(unique.values(), key=sort_key) + + +def sort_model_options(provider: str, models: list[dict[str, str]]) -> list[dict[str, str]]: + preferred = PREFERRED_MODELS.get(provider, []) + + def sort_key(item: dict[str, str]) -> tuple[int, str]: + value = item["value"] + if value in preferred: + return (preferred.index(value), "") + return (1000, item["label"].lower()) + + unique = {m["value"]: m for m in models} + return sorted(unique.values(), key=sort_key) + + +async def model_options_for_provider(provider: str, api_key: str = "") -> list[dict[str, str]]: + provider = (provider or "openrouter").strip().lower() + now = time.time() + cache_key = f"{provider}:{hashlib.sha256((api_key or '').encode('utf-8')).hexdigest()[:10]}" + if MODEL_CACHE["key"] == cache_key and now - float(MODEL_CACHE["at"]) < 3600: + return list(MODEL_CACHE["models"]) + + models: list[dict[str, str]] = [] + if provider == "openrouter": + try: + models = await fetch_openrouter_models() + except Exception as exc: + logger.warning("OpenRouter model list failed: {}", exc) + elif provider == "anthropic": + try: + models = await fetch_anthropic_models(api_key) + except Exception as exc: + logger.warning("Anthropic model list failed: {}", exc) + elif provider == "openai": + try: + models = await fetch_openai_models(api_key) + except Exception as exc: + logger.warning("OpenAI model list failed: {}", exc) + if not models: + models = litellm_catalog_models(provider) + if not models: + models = [{"value": m, "label": f"{m} · fallback"} for m in MODEL_FALLBACKS.get(provider, [])] + + models = sort_model_options(provider, models) + MODEL_CACHE.update({"key": cache_key, "at": now, "models": models}) + return list(models) + + +async def prune_audit_log() -> None: + pool = await get_pool() + await pool.execute("DELETE FROM audit_log WHERE created_at < NOW() - INTERVAL '30 days'") + + +def request_audit_meta(request: Request | None, extra: dict[str, Any] | None = None) -> dict[str, Any]: + meta = dict(extra or {}) + if request: + meta.update( + { + "ip": client_ip(request), + "method": request.method, + "path": request.url.path, + "query": str(request.url.query), + "user_agent": (request.headers.get("user-agent") or "")[:300], + } + ) + return meta + + +async def audit( + actor_id: int | None, + action: str, + entity_type: str, + entity_id: int | None, + after: Any = None, + request: Request | None = None, +) -> None: + pool = await get_pool() + payload = request_audit_meta(request, after if isinstance(after, dict) else {"value": after} if after is not None else None) + await pool.execute( + """ + INSERT INTO audit_log(actor_id, action, entity_type, entity_id, after_json) + VALUES($1, $2, $3, $4, $5::jsonb) + """, + actor_id, + action, + entity_type, + entity_id, + json.dumps(payload, ensure_ascii=False) if payload else None, + ) + + +def prepare_audit_row(row: Any) -> dict[str, Any]: + item = dict(row) + payload = item.get("after_json") + if isinstance(payload, str): + try: + payload = json.loads(payload) + except json.JSONDecodeError: + pass + item["created_at_fmt"] = format_dt(item.get("created_at")) + item["actor_label"] = item.get("actor_login") or (f"user #{item['actor_id']}" if item.get("actor_id") else "system") + item["details_pretty"] = json.dumps(payload, ensure_ascii=False, indent=2, default=str) if payload is not None else "" + return item + + +async def start_worker_task(worker: Any, name: str) -> None: + try: + logger.info("Starting background worker task: {}", name) + await worker.run_loop() + except Exception as exc: + logger.exception("Background worker task {} stopped unexpectedly: {}", name, exc) + + +@app.on_event("startup") +async def startup() -> None: + await bootstrap_admin() + await prune_audit_log() + workers = [ + ("ai-qualifier", AIQualifierWorker()), + ("ai-writer", AIWriterWorker()), + ("daily-report", DailyReportWorker()), + ("max-poster", MAXPoster()), + ("vk-parser", VKParserWorker()), + ("site-poster", SitePoster()), + ("tg-poster", TelegramPoster()), + ("tg-reactor", TelegramReactor()), + ("vk-poster", VKPoster()), + ("vk-storage-uploader", TelegramStorageUploader()), + ] + for name, worker in workers: + asyncio.create_task(start_worker_task(worker, name)) + + +@app.get("/health") +async def health() -> dict[str, str]: + return {"ok": "true"} + + +@app.get("/vk/oauth/callback", response_class=HTMLResponse) +async def vk_oauth_callback(request: Request, code: str = "", error: str = "", error_description: str = ""): + return templates.TemplateResponse( + "vk_oauth_callback.html", + base_context( + request, + None, + code=code, + code_verifier=request.cookies.get(VK_OAUTH_VERIFIER_COOKIE, ""), + device_id=request.query_params.get("device_id", ""), + state=request.query_params.get("state", ""), + expected_state=request.cookies.get(VK_OAUTH_STATE_COOKIE, ""), + error=error, + error_description=error_description, + ), + ) + + +@app.get("/vk/oauth/start") +async def vk_oauth_start() -> RedirectResponse: + verifier = secrets.token_urlsafe(64) + state = secrets.token_urlsafe(24) + redirect_uri = "https://sw.exostring.xyz/vk/oauth/callback" + params = { + "client_id": "54635120", + "redirect_uri": redirect_uri, + "response_type": "code", + "scope": "wall photos video groups offline", + "state": state, + "code_challenge": pkce_challenge(verifier), + "code_challenge_method": "s256", + "origin": "https://sw.exostring.xyz", + "v": "5.199", + } + response = RedirectResponse(f"https://id.vk.ru/authorize?{urlencode(params)}") + response.set_cookie(VK_OAUTH_VERIFIER_COOKIE, verifier, httponly=True, samesite="lax", max_age=15 * 60) + response.set_cookie(VK_OAUTH_STATE_COOKIE, state, httponly=True, samesite="lax", max_age=15 * 60) + return response + + +@app.get("/vk/group-oauth/start") +async def vk_group_oauth_start() -> RedirectResponse: + group_id = abs(int(settings.vk_storage_group_id)) + params = { + "client_id": "54635120", + "group_ids": str(group_id), + "display": "page", + "redirect_uri": "https://sw.exostring.xyz/vk/oauth/callback", + "scope": "manage,photos,docs", + "response_type": "token", + "state": secrets.token_urlsafe(24), + "v": "5.199", + } + return RedirectResponse(f"https://oauth.vk.com/authorize?{urlencode(params)}") + + +@app.get("/vk-oauth/start") +async def vk_poster_oauth_start(request: Request) -> RedirectResponse: + user = await get_current_user(request) + if not user: + return redirect("/login") + client_id = str(await fetch_setting("vk_poster_app_id", "") or "").strip() + if not client_id: + return redirect("/workers") + verifier = secrets.token_urlsafe(64) + state = secrets.token_urlsafe(24) + redirect_uri = "https://sw.exostring.xyz/vk-oauth/callback" + params = { + "client_id": client_id, + "redirect_uri": redirect_uri, + "response_type": "code", + "scope": "wall photos", + "state": state, + "code_challenge": pkce_challenge(verifier), + "code_challenge_method": "s256", + "origin": "https://sw.exostring.xyz", + "v": "5.199", + } + response = RedirectResponse(f"https://id.vk.ru/authorize?{urlencode(params)}") + response.set_cookie(VK_OAUTH_VERIFIER_COOKIE, verifier, httponly=True, secure=(request.url.scheme == "https"), samesite="lax", max_age=15 * 60) + response.set_cookie(VK_OAUTH_STATE_COOKIE, state, httponly=True, secure=(request.url.scheme == "https"), samesite="lax", max_age=15 * 60) + return response + + +@app.get("/vk-oauth/callback", response_class=HTMLResponse) +async def vk_poster_oauth_callback( + request: Request, + code: str = "", + state: str = "", + error: str = "", + error_description: str = "", +): + user = await get_current_user(request) + result: dict[str, Any] = { + "ok": False, + "error": error, + "error_description": error_description, + "state_ok": bool(state and state == request.cookies.get(VK_OAUTH_STATE_COOKIE, "")), + } + if not user: + result.update({"error": "auth_required", "error_description": "Сначала войдите в админку, затем повторите VK OAuth."}) + elif error: + pass + elif not code: + result.update({"error": "missing_code", "error_description": "VK callback пришёл без code."}) + elif not result["state_ok"]: + result.update({"error": "state_mismatch", "error_description": "OAuth state не совпал. Повторите авторизацию из админки."}) + else: + pool = await get_pool() + client_id = str(await fetch_setting("vk_poster_app_id", "") or "").strip() + client_secret = str(await fetch_setting("vk_poster_client_secret", "") or "").strip() + owner_id = int(await fetch_int_setting("vk_poster_owner_id", 0)) + redirect_uri = "https://sw.exostring.xyz/vk-oauth/callback" + code_verifier = request.cookies.get(VK_OAUTH_VERIFIER_COOKIE, "") + device_id = request.query_params.get("device_id", "") + try: + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=30)) as session: + async with session.post( + "https://id.vk.ru/oauth2/auth", + data={ + "grant_type": "authorization_code", + "client_id": client_id, + "client_secret": client_secret, + "redirect_uri": redirect_uri, + "code": code, + "code_verifier": code_verifier, + "device_id": device_id, + "state": state, + }, + ) as resp: + token_payload = await resp.json(content_type=None) + if token_payload.get("error"): + raise RuntimeError(f"{token_payload.get('error')}: {token_payload.get('error_description')}") + access_token = str(token_payload.get("access_token") or "").strip() + if not access_token: + raise RuntimeError(f"VK did not return access_token: {token_payload}") + refresh_token = str(token_payload.get("refresh_token") or "").strip() + expires_in = token_payload.get("expires_in") + expires_at = None + try: + expires_at = (now_utc() + timedelta(seconds=int(expires_in))).isoformat() if expires_in is not None else None + except Exception: + expires_at = None + updates = { + "vk_poster_access_token": access_token, + "vk_poster_refresh_token": refresh_token, + "vk_poster_token_device_id": str(token_payload.get("device_id") or device_id or ""), + "vk_poster_token_expires_at": expires_at or "", + } + for key, value in updates.items(): + await pool.execute( + """ + UPDATE app_settings + SET value_json=$2::jsonb, + updated_by=$3, + updated_at=NOW() + WHERE key=$1 + """, + key, + json.dumps(value), + user["id"], + ) + check: dict[str, Any] = {} + async with VKAPIClient(token=access_token, rps=2, timeout_total_sec=30, retry_attempts=1) as client: + try: + upload_url = await client.get_wall_upload_server(abs(owner_id)) + check["photo_upload_server_ok"] = bool(upload_url) + except Exception as exc: + check["photo_upload_server_ok"] = False + check["photo_upload_server_error"] = str(exc) + try: + response = await client.call("wall.get", owner_id=owner_id, count=1, filter="owner") + check["wall_get_ok"] = isinstance(response, dict) + except Exception as exc: + check["wall_get_ok"] = False + check["wall_get_error"] = str(exc) + await audit( + user["id"], + "vk_oauth.token_saved", + "vk_poster", + None, + { + "owner_id": owner_id, + "user_id": token_payload.get("user_id"), + "expires_in": token_payload.get("expires_in"), + "has_refresh_token": bool(refresh_token), + "expires_at": expires_at, + **check, + }, + request, + ) + result.update( + { + "ok": True, + "error": "", + "error_description": "", + "owner_id": owner_id, + "user_id": token_payload.get("user_id"), + "expires_in": token_payload.get("expires_in"), + "has_refresh_token": bool(refresh_token), + "expires_at": expires_at, + **check, + } + ) + except Exception as exc: + result.update({"error": "token_exchange_failed", "error_description": str(exc)}) + response = templates.TemplateResponse("vk_oauth_callback.html", base_context(request, user, result=result)) + response.delete_cookie(VK_OAUTH_STATE_COOKIE) + response.delete_cookie(VK_OAUTH_VERIFIER_COOKIE) + return response + + +@app.get("/", response_class=HTMLResponse) +async def index(request: Request): + user = await get_current_user(request) + if not user: + return redirect("/login") + return redirect("/sources") + + +@app.get("/login", response_class=HTMLResponse) +async def login_get(request: Request): + return templates.TemplateResponse("login.html", base_context(request, None, error="")) + + +@app.post("/login", response_class=HTMLResponse) +async def login_post(request: Request, login: str = Form(...), password: str = Form(...)): + pool = await get_pool() + login_value = login.strip().lower() + ip = client_ip(request) + await pool.execute("DELETE FROM admin_login_attempts WHERE created_at < NOW() - INTERVAL '7 days'") + ip_failures = int( + await pool.fetchval( + """ + SELECT COUNT(*) + FROM admin_login_attempts + WHERE ip=$1 AND success=FALSE AND created_at > NOW() - INTERVAL '15 minutes' + """, + ip, + ) + or 0 + ) + login_failures = int( + await pool.fetchval( + """ + SELECT COUNT(*) + FROM admin_login_attempts + WHERE login=$1 AND success=FALSE AND created_at > NOW() - INTERVAL '15 minutes' + """, + login_value, + ) + or 0 + ) + # Stricter limits: 5 bad attempts per IP or 3 per login in 15 minutes → block. + if ip_failures >= 5 or login_failures >= 3: + await pool.execute( + "INSERT INTO admin_login_attempts(ip, login, success) VALUES($1, $2, FALSE)", + ip, + login_value, + ) + await audit(None, "auth.login_blocked", "admin_login", None, {"login": login_value}, request) + return templates.TemplateResponse( + "login.html", + base_context(request, None, error="Слишком много попыток. Подожди 15 минут и попробуй снова."), + status_code=status.HTTP_429_TOO_MANY_REQUESTS, + ) + row = await pool.fetchrow( + "SELECT id, login, password_hash, role, is_active FROM admin_users WHERE login=$1", + login_value, + ) + if not row or not row["is_active"] or not verify_password(password, row["password_hash"]): + await pool.execute( + "INSERT INTO admin_login_attempts(ip, login, success) VALUES($1, $2, FALSE)", + ip, + login_value, + ) + await audit(None, "auth.login_failed", "admin_login", None, {"login": login_value}, request) + # Constant-time delay to slow brute-force attempts. + await asyncio.sleep(1.0) + return templates.TemplateResponse("login.html", base_context(request, None, error="Неверный логин или пароль")) + + await pool.execute("INSERT INTO admin_login_attempts(ip, login, success) VALUES($1, $2, TRUE)", ip, login_value) + token = new_token() + csrf = new_token() + await pool.execute( + """ + INSERT INTO admin_sessions(token_hash, user_id, csrf_token, expires_at) + VALUES($1, $2, $3, $4) + """, + token_hash(token, settings.app_secret_key), + row["id"], + csrf, + now_utc() + timedelta(days=14), + ) + await audit(row["id"], "auth.login_success", "admin_user", int(row["id"]), {"login": login_value}, request) + response = redirect("/sources") + response.set_cookie(COOKIE_NAME, token, httponly=True, secure=(request.url.scheme == "https"), samesite="lax", max_age=14 * 24 * 3600) + return response + + +@app.post("/logout") +async def logout(request: Request): + token = request.cookies.get(COOKIE_NAME) + user = await get_current_user(request) if token else None + if token: + pool = await get_pool() + await pool.execute("DELETE FROM admin_sessions WHERE token_hash=$1", token_hash(token, settings.app_secret_key)) + if user: + await audit(user["id"], "auth.logout", "admin_user", int(user["id"]), None, request) + response = redirect("/login") + response.delete_cookie(COOKIE_NAME) + return response + + +@app.get("/sources", response_class=HTMLResponse) +async def sources_list(request: Request, q: str = "", status_filter: str = "", active_filter: str = "", page: int = 1, per_page: int = 50): + user = await get_current_user(request) + if not user: + return redirect("/login") + pool = await get_pool() + table_filters = parse_table_filters(request, SOURCE_FIELD_SPECS) + table_sorts = parse_table_sorts(request, SOURCE_FIELD_SPECS) + filter_clauses, filter_args = build_filter_sql(table_filters, SOURCE_FIELD_SPECS, 4) + where_parts = [ + "s.archived_at IS NULL", + "($1='' OR s.name ILIKE '%' || $1 || '%' OR COALESCE(s.tag,'') ILIKE '%' || $1 || '%' OR s.url ILIKE '%' || $1 || '%' OR COALESCE(s.external_id,'') ILIKE '%' || $1 || '%')", + "($2='' OR s.status=$2)", + "($3='' OR ($3='on' AND s.active=TRUE) OR ($3='off' AND s.active=FALSE))", + *filter_clauses, + ] + where_sql = " AND ".join(where_parts) + base_args = [q.strip(), status_filter.strip(), active_filter.strip(), *filter_args] + total = int( + await pool.fetchval( + f""" + SELECT COUNT(*) + FROM sources s + WHERE {where_sql} + """, + *base_args, + ) + or 0 + ) + pager = pagination(page, per_page, total) + limit_index = len(base_args) + 1 + offset_index = len(base_args) + 2 + order_sql = build_sort_sql(table_sorts, SOURCE_FIELD_SPECS, "s.active DESC, s.priority ASC, s.id DESC") + rows = await pool.fetch( + f""" + SELECT s.*, + COUNT(rp.id) AS posts_count, + COUNT(rp.id) FILTER (WHERE rp.created_at > NOW() - INTERVAL '24 hours') AS posts_24h + FROM sources s + LEFT JOIN raw_posts rp ON rp.source_id=s.id + WHERE {where_sql} + GROUP BY s.id + ORDER BY {order_sql} + LIMIT ${limit_index} OFFSET ${offset_index} + """, + *base_args, + pager["per_page"], + pager["offset"], + ) + sources = [] + for row in rows: + item = dict(row) + item["last_parsed_at_fmt"] = format_dt(item.get("last_parsed_at")) + item["last_checked_at_fmt"] = format_dt(item.get("last_checked_at")) + item["created_at_fmt"] = format_dt(item.get("created_at")) + sources.append(item) + template_name = "sources_content.html" if request.headers.get("hx-request") else "sources.html" + return templates.TemplateResponse( + template_name, + base_context( + request, + user, + sources=sources, + q=q, + status_filter=status_filter, + active_filter=active_filter, + pagination=pager, + table_filters=table_filters, + table_sorts=table_sorts, + field_options=field_options(SOURCE_FIELD_SPECS), + filter_ops=FILTER_OPS, + filter_op_options=FILTER_OP_OPTIONS, + source_preview=[], + bulk_text="", + bulk_active=True, + ), + ) + + +@app.post("/sources/preview", response_class=HTMLResponse) +async def sources_preview( + request: Request, + csrf_token: str = Form(...), + bulk_text: str = Form(""), + bulk_active: str = Form("off"), + q: str = Form(""), + status_filter: str = Form(""), + per_page: int = Form(50), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + preview = await build_sources_preview(bulk_text, default_active=bulk_active == "on") + pool = await get_pool() + table_filters = parse_table_filters(request, SOURCE_FIELD_SPECS) + table_sorts = parse_table_sorts(request, SOURCE_FIELD_SPECS) + filter_clauses, filter_args = build_filter_sql(table_filters, SOURCE_FIELD_SPECS, 3) + where_parts = [ + "s.archived_at IS NULL", + "($1='' OR s.name ILIKE '%' || $1 || '%' OR COALESCE(s.tag,'') ILIKE '%' || $1 || '%' OR s.url ILIKE '%' || $1 || '%' OR COALESCE(s.external_id,'') ILIKE '%' || $1 || '%')", + "($2='' OR s.status=$2)", + *filter_clauses, + ] + where_sql = " AND ".join(where_parts) + base_args = [q.strip(), status_filter.strip(), *filter_args] + total = int(await pool.fetchval(f"SELECT COUNT(*) FROM sources s WHERE {where_sql}", *base_args) or 0) + pager = pagination(1, per_page, total) + limit_index = len(base_args) + 1 + offset_index = len(base_args) + 2 + order_sql = build_sort_sql(table_sorts, SOURCE_FIELD_SPECS, "s.active DESC, s.priority ASC, s.id DESC") + rows = await pool.fetch( + f""" + SELECT s.*, + COUNT(rp.id) AS posts_count, + COUNT(rp.id) FILTER (WHERE rp.created_at > NOW() - INTERVAL '24 hours') AS posts_24h + FROM sources s + LEFT JOIN raw_posts rp ON rp.source_id=s.id + WHERE {where_sql} + GROUP BY s.id + ORDER BY {order_sql} + LIMIT ${limit_index} OFFSET ${offset_index} + """, + *base_args, + pager["per_page"], + pager["offset"], + ) + sources = [] + for row in rows: + item = dict(row) + item["last_parsed_at_fmt"] = format_dt(item.get("last_parsed_at")) + item["last_checked_at_fmt"] = format_dt(item.get("last_checked_at")) + item["created_at_fmt"] = format_dt(item.get("created_at")) + sources.append(item) + return templates.TemplateResponse( + "sources.html", + base_context( + request, + user, + sources=sources, + q=q, + status_filter=status_filter, + pagination=pager, + table_filters=table_filters, + table_sorts=table_sorts, + field_options=field_options(SOURCE_FIELD_SPECS), + filter_ops=FILTER_OPS, + filter_op_options=FILTER_OP_OPTIONS, + source_preview=preview, + bulk_text=bulk_text, + bulk_active=bulk_active == "on", + ), + ) + + +@app.post("/sources/bulk") +async def sources_bulk_create( + request: Request, + csrf_token: str = Form(...), + bulk_payload: str = Form("[]"), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + try: + items = json.loads(bulk_payload) + except json.JSONDecodeError: + items = [] + pool = await get_pool() + created = 0 + for item in items: + if not isinstance(item, dict) or not item.get("ok"): + continue + try: + row = await pool.fetchrow( + """ + INSERT INTO sources(platform, name, tag, url, external_id, external_owner_id, active, priority, created_by) + VALUES($1, $2, $3, $4, $5, $6, $7, 100, $8) + ON CONFLICT DO NOTHING + RETURNING id + """, + PLATFORM_VK, + str(item.get("name") or item.get("external_id") or "").strip(), + normalize_hash_tag(str(item.get("tag") or item.get("external_id") or ""), str(item.get("external_id") or "source")), + str(item.get("url") or "").strip(), + str(item.get("external_id") or "").strip(), + item.get("external_owner_id"), + bool(item.get("active", True)), + user["id"], + ) + if row: + created += 1 + await audit(user["id"], "source.create", "source", int(row["id"]), {"url": item.get("url"), "bulk": True}) + except Exception: + logger.exception("Bulk source insert failed: {}", item) + return redirect(f"/sources?q=&status_filter=&created={created}") + + +@app.get("/sources/new", response_class=HTMLResponse) +async def source_new(request: Request): + user = await get_current_user(request) + if not user: + return redirect("/login") + return templates.TemplateResponse( + "source_form.html", + base_context(request, user, source=None, action="/sources/new", title="Новый источник"), + ) + + +@app.post("/sources/new") +async def source_create( + request: Request, + csrf_token: str = Form(...), + platform: str = Form(PLATFORM_VK), + name: str = Form(...), + tag: str = Form(""), + url: str = Form(...), + active: str = Form("off"), + priority: int = Form(100), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + platform = platform.strip().lower() or PLATFORM_VK + external_id = normalize_vk_source(url) if platform == PLATFORM_VK else "" + external_owner_id = None + resolved_name = name.strip() + resolved_url = url.strip() + status_value = "new" + status_msg = None + if platform == PLATFORM_VK and external_id: + try: + async with VKAPIClient(rps=2, timeout_total_sec=12, timeout_connect_sec=5, retry_attempts=2) as client: + screen_name, owner_id, vk_name = await client.resolve_group(url) + external_id = screen_name + external_owner_id = owner_id + resolved_name = resolved_name or vk_name + resolved_url = f"https://vk.com/{screen_name}" + except Exception as exc: + resolved_name = resolved_name or external_id + status_value = "error" + status_msg = str(exc)[:500] + resolved_tag = normalize_hash_tag(tag or external_id or resolved_name, external_id or "source") + pool = await get_pool() + row = await pool.fetchrow( + """ + INSERT INTO sources(platform, name, tag, url, external_id, external_owner_id, active, priority, status, status_msg, created_by) + VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + ON CONFLICT DO NOTHING + RETURNING id + """, + platform, + resolved_name, + resolved_tag, + resolved_url, + external_id, + external_owner_id, + active == "on", + priority, + status_value, + status_msg, + user["id"], + ) + if row: + await audit(user["id"], "source.create", "source", int(row["id"]), {"url": url, "platform": platform}) + return redirect("/sources") + + +@app.get("/sources/{source_id}/edit", response_class=HTMLResponse) +async def source_edit(request: Request, source_id: int): + user = await get_current_user(request) + if not user: + return redirect("/login") + pool = await get_pool() + source = await pool.fetchrow("SELECT * FROM sources WHERE id=$1 AND archived_at IS NULL", source_id) + if not source: + return redirect("/sources") + return templates.TemplateResponse( + "source_form.html", + base_context( + request, + user, + source=dict(source), + action=f"/sources/{source_id}/edit", + title=f"Источник #{source_id}", + ), + ) + + +@app.post("/sources/{source_id}/edit") +async def source_update( + request: Request, + source_id: int, + csrf_token: str = Form(...), + platform: str = Form(PLATFORM_VK), + name: str = Form(...), + tag: str = Form(""), + url: str = Form(...), + active: str = Form("off"), + priority: int = Form(100), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + platform = platform.strip().lower() or PLATFORM_VK + external_id = normalize_vk_source(url) if platform == PLATFORM_VK else "" + pool = await get_pool() + await pool.execute( + """ + UPDATE sources + SET platform=$2, + name=$3, + tag=$4, + url=$5, + external_id=$6, + active=$7, + priority=$8, + updated_at=NOW() + WHERE id=$1 + """, + source_id, + platform, + name.strip(), + normalize_hash_tag(tag or external_id or name, external_id or "source"), + url.strip(), + external_id, + active == "on", + priority, + ) + await audit(user["id"], "source.update", "source", source_id, {"url": url, "platform": platform}) + return redirect("/sources") + + +@app.post("/sources/{source_id}/toggle") +async def source_toggle(request: Request, source_id: int, csrf_token: str = Form(...)): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + pool = await get_pool() + row = await pool.fetchrow( + """ + UPDATE sources + SET active=NOT active, + status=CASE WHEN active THEN 'paused' ELSE 'new' END, + updated_at=NOW() + WHERE id=$1 + RETURNING active + """, + source_id, + ) + await audit(user["id"], "source.toggle", "source", source_id, {"active": bool(row["active"]) if row else None}) + if request.headers.get("hx-request"): + # Fetch the updated source + source_row = await pool.fetchrow("SELECT * FROM sources WHERE id=$1", source_id) + if source_row: + s = dict(source_row) + # Fetch stats just for this source to render correctly + stats = await pool.fetchrow( + """ + SELECT COUNT(*) AS total_posts, + COUNT(*) FILTER (WHERE created_at >= NOW() - INTERVAL '24 hours') AS recent_posts + FROM raw_posts + WHERE source_id=$1 + """, + source_id + ) + s["posts_count"] = stats["total_posts"] + s["posts_24h"] = stats["recent_posts"] + s["last_parsed_at_fmt"] = s["last_parsed_at"].strftime("%d.%m.%Y %H:%M") if s.get("last_parsed_at") else None + return templates.TemplateResponse("source_row.html", base_context(request, user, s=s)) + return redirect("/sources") + + +@app.post("/sources/{source_id}/delete") +async def source_delete(request: Request, source_id: int, csrf_token: str = Form(...)): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + pool = await get_pool() + await pool.execute("UPDATE sources SET archived_at=NOW(), active=FALSE, updated_at=NOW() WHERE id=$1", source_id) + await audit(user["id"], "source.archive", "source", source_id) + if request.headers.get("hx-request"): + return HTMLResponse("") + return redirect("/sources") + + +@app.get("/raw", response_class=HTMLResponse) +async def raw_posts( + request: Request, + status_filter: str = "", + date_from: str = "", + date_to: str = "", + score_min: str = "", + score_max: str = "", + sort: str = "created_desc", + page: int = 1, + per_page: int = 50, +): + user = await get_current_user(request) + if not user: + return redirect("/login") + pool = await get_pool() + order_map = { + "id_desc": "rp.id DESC", + "id_asc": "rp.id ASC", + "created_desc": "rp.created_at DESC", + "created_asc": "rp.created_at ASC", + "posted_desc": "rp.posted_at DESC NULLS LAST, rp.created_at DESC", + "posted_asc": "rp.posted_at ASC NULLS LAST, rp.created_at ASC", + "source_asc": "lower(s.name) ASC, rp.created_at DESC", + "source_desc": "lower(s.name) DESC, rp.created_at DESC", + "stage_asc": "rp.status ASC, COALESCE(rp.qualification_status, 'pending') ASC, COALESCE(rp.rewrite_status, 'pending') ASC, rp.created_at DESC", + "stage_desc": "rp.status DESC, COALESCE(rp.qualification_status, 'pending') DESC, COALESCE(rp.rewrite_status, 'pending') DESC, rp.created_at DESC", + "text_asc": "lower(rp.raw_text) ASC, rp.created_at DESC", + "text_desc": "lower(rp.raw_text) DESC, rp.created_at DESC", + "score_desc": "rp.qualification_score DESC NULLS LAST, rp.created_at DESC", + "score_asc": "rp.qualification_score ASC NULLS LAST, rp.created_at DESC", + } + raw_statuses = selected_values(request, "raw_status") + if status_filter.strip() and not raw_statuses: + raw_statuses = [status_filter.strip()] + source_ids = selected_int_values(request, "source_id") + categories_selected = selected_values(request, "category") + qualification_statuses = selected_values(request, "qualification_status") + rewrite_statuses = selected_values(request, "rewrite_status") + where_parts: list[str] = [] + base_args: list[Any] = [] + if raw_statuses: + add_where(where_parts, base_args, "rp.status = ANY(${i}::text[])", raw_statuses) + if source_ids: + add_where(where_parts, base_args, "rp.source_id = ANY(${i}::bigint[])", source_ids) + if categories_selected: + add_where(where_parts, base_args, "COALESCE(rp.final_category, rp.rewrite_category, '') = ANY(${i}::text[])", categories_selected) + if qualification_statuses: + add_where(where_parts, base_args, "COALESCE(rp.qualification_status, 'pending') = ANY(${i}::text[])", qualification_statuses) + if rewrite_statuses: + add_where(where_parts, base_args, "COALESCE(rp.rewrite_status, 'pending') = ANY(${i}::text[])", rewrite_statuses) + parsed_date_from = parse_date_filter(date_from) + parsed_date_to = parse_date_filter(date_to) + if parsed_date_from: + add_where(where_parts, base_args, "timezone('Asia/Yekaterinburg', COALESCE(rp.posted_at, rp.created_at))::date >= ${i}::date", parsed_date_from) + if parsed_date_to: + add_where(where_parts, base_args, "timezone('Asia/Yekaterinburg', COALESCE(rp.posted_at, rp.created_at))::date <= ${i}::date", parsed_date_to) + if score_min.strip().isdigit(): + add_where(where_parts, base_args, "rp.qualification_score >= ${i}::int", int(score_min.strip())) + if score_max.strip().isdigit(): + add_where(where_parts, base_args, "rp.qualification_score <= ${i}::int", int(score_max.strip())) + where_sql = " AND ".join(where_parts) if where_parts else "TRUE" + order_sql = order_map.get(sort, order_map["created_desc"]) + total = int( + await pool.fetchval( + f""" + SELECT COUNT(*) + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + WHERE {where_sql} + """, + *base_args, + ) + or 0 + ) + pager = pagination(page, per_page, total) + limit_index = len(base_args) + 1 + offset_index = len(base_args) + 2 + rows = await pool.fetch( + f""" + SELECT rp.*, s.name AS source_name, s.tag AS source_tag, s.platform AS source_platform, + COUNT(rpm.id) AS media_count, + COALESCE( + jsonb_agg( + jsonb_build_object( + 'id', rpm.id, + 'type', rpm.media_type, + 'url', rpm.original_url, + 'status', rpm.status, + 'error', rpm.error, + 'duration_sec', rpm.duration_sec + ) + ORDER BY rpm.sort_order ASC, rpm.id ASC + ) FILTER (WHERE rpm.id IS NOT NULL), + '[]'::jsonb + ) AS media_items, + COALESCE( + ( + SELECT jsonb_agg( + jsonb_build_object( + 'platform', pp.platform, + 'status', pp.status, + 'target_id', pp.target_id, + 'external_id', pp.external_id, + 'url', pp.url, + 'error', pp.error, + 'published_at', pp.published_at + ) + ORDER BY pp.platform + ) + FROM post_publications pp + WHERE pp.raw_post_id=rp.id + ), + '[]'::jsonb + ) AS platform_publications + , + COALESCE( + ( + SELECT jsonb_agg( + jsonb_build_object( + 'platform', pp.platform, + 'status', pp.status, + 'target_id', pp.target_id, + 'external_id', pp.external_id, + 'url', pp.url, + 'error', pp.error, + 'published_at', pp.published_at + ) + ORDER BY pp.platform + ) + FROM post_publications pp + WHERE pp.raw_post_id=rp.id + ), + '[]'::jsonb + ) AS platform_publications + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + LEFT JOIN raw_post_media rpm ON rpm.raw_post_id=rp.id AND COALESCE(rpm.editor_added, FALSE)=FALSE + WHERE {where_sql} + GROUP BY rp.id, s.name, s.tag, s.platform + ORDER BY {order_sql} + LIMIT ${limit_index} OFFSET ${offset_index} + """, + *base_args, + pager["per_page"], + pager["offset"], + ) + posts = [prepare_raw_post(row) for row in rows] + + template_name = "raw_posts_content.html" if request.headers.get("hx-request") else "raw_posts.html" + + return templates.TemplateResponse( + template_name, + base_context( + request, + user, + posts=posts, + raw_statuses=raw_statuses, + source_ids=source_ids, + categories_selected=categories_selected, + qualification_statuses=qualification_statuses, + rewrite_statuses=rewrite_statuses, + date_from=date_from, + date_to=date_to, + score_min=score_min, + score_max=score_max, + sort=sort, + facets=await raw_filter_facets(pool), + preserved_query=preserved_query(request, {"page", "per_page"}), + sort_headers=raw_sort_headers(request, sort), + pagination=pager, + ), + ) + + +@app.post("/raw/send-to-rewrite") +async def raw_send_to_rewrite( + request: Request, + csrf_token: str = Form(...), + raw_post_id: list[int] | None = Form(default=None), + next_url: str = Form("/raw"), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + post_ids = sorted({int(post_id) for post_id in (raw_post_id or []) if int(post_id) > 0}) + safe_next = next_url if next_url.startswith("/raw") and not next_url.startswith("//") else "/raw" + if not post_ids: + return redirect(safe_next) + pool = await get_pool() + updated_rows = await pool.fetch( + """ + WITH selected AS ( + SELECT id + FROM raw_posts + WHERE id=ANY($1::bigint[]) + AND status='storage_ready' + AND COALESCE(rewrite_status, 'pending') IN ('pending', 'failed') + FOR UPDATE + ) + UPDATE raw_posts rp + SET qualification_status='accepted', + qualification_decision='accepted', + qualification_model_decision='manual', + qualification_reason='Ручная отправка в рерайт из Raw', + qualification_reject_tag=NULL, + qualified_at=NOW(), + rewrite_status='pending', + rewrite_notes=NULL, + rewrite_batch_id=NULL, + editorial_status=NULL, + updated_at=NOW() + FROM selected + WHERE rp.id=selected.id + RETURNING rp.id + """, + post_ids, + ) + updated_ids = [int(row["id"]) for row in updated_rows] + await audit( + user["id"], + "raw.manual_send_to_rewrite", + "raw_post", + None, + {"selected_ids": post_ids, "updated_ids": updated_ids, "ignored_count": len(post_ids) - len(updated_ids)}, + ) + return redirect(safe_next) + + +async def fetch_single_editor_row(pool, post_id: int): + return await pool.fetchrow( + """ + SELECT rp.*, s.name AS source_name, s.tag AS source_tag, s.platform AS source_platform, s.url AS source_url, + u.login AS reviewed_by_login, + COALESCE(c_id.tag, c_name.tag) AS resolved_category_tag, + COUNT(rpm.id) AS media_count, + COALESCE( + jsonb_agg( + jsonb_build_object( + 'id', rpm.id, + 'type', rpm.media_type, + 'url', rpm.original_url, + 'status', rpm.status, + 'error', rpm.error, + 'duration_sec', rpm.duration_sec + ) + ORDER BY rpm.sort_order ASC, rpm.id ASC + ) FILTER (WHERE rpm.id IS NOT NULL), + '[]'::jsonb + ) AS media_items, + COALESCE( + ( + SELECT jsonb_agg( + jsonb_build_object( + 'platform', pp.platform, + 'status', pp.status, + 'target_id', pp.target_id, + 'external_id', pp.external_id, + 'url', pp.url, + 'error', pp.error, + 'published_at', pp.published_at + ) + ORDER BY pp.platform + ) + FROM post_publications pp + WHERE pp.raw_post_id=rp.id + ), + '[]'::jsonb + ) AS platform_publications + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + LEFT JOIN admin_users u ON u.id=rp.reviewed_by + LEFT JOIN content_categories c_id ON c_id.sort_order=COALESCE(rp.final_category_id, rp.rewrite_category_id) + LEFT JOIN content_categories c_name ON lower(c_name.name)=lower(COALESCE(rp.final_category, rp.rewrite_category, '')) + LEFT JOIN raw_post_media rpm ON rpm.raw_post_id=rp.id AND COALESCE(rpm.editor_hidden, FALSE)=FALSE + WHERE rp.id = $1 + GROUP BY rp.id, s.name, s.tag, s.platform, s.url, u.login, c_id.tag, c_name.tag + """, + post_id + ) + + +async def editor_post_update_response( + request: Request, + user: dict, + pool, + post_id: int, + action_message: str = "", + action_tone: str = "info", +): + row = await fetch_single_editor_row(pool, post_id) + if not row: + return HTMLResponse("", status_code=204) + return templates.TemplateResponse( + "editor_post_update.html", + base_context( + request, + user, + p=prepare_editor_post(row), + action_message=action_message, + action_tone=action_tone, + ), + ) + + +@app.get("/editor", response_class=HTMLResponse) +async def editor_feed( + request: Request, + status_filter: str = "review", + category_filter: str = "", + date_from: str = "", + date_to: str = "", + score_min: str = "", + score_max: str = "", + sort: str = "rewritten_desc", + page: int = 1, + per_page: int = 25, +): + user = await get_current_user(request) + if not user: + return redirect("/login") + pool = await get_pool() + editorial_statuses = selected_values(request, "editorial_status") + if status_filter.strip() and not editorial_statuses: + editorial_statuses = [status_filter.strip()] + categories_selected = selected_values(request, "category") + if category_filter.strip() and not categories_selected: + categories_selected = [category_filter.strip()] + source_ids = selected_int_values(request, "source_id") + sort_map = { + "rewritten_desc": "rp.rewritten_at DESC NULLS LAST, rp.id DESC", + "rewritten_asc": "rp.rewritten_at ASC NULLS LAST, rp.id ASC", + "score_desc": "rp.qualification_score DESC NULLS LAST, rp.rewritten_at DESC NULLS LAST", + "score_asc": "rp.qualification_score ASC NULLS LAST, rp.rewritten_at DESC NULLS LAST", + "posted_desc": "rp.posted_at DESC NULLS LAST, rp.rewritten_at DESC NULLS LAST", + "posted_asc": "rp.posted_at ASC NULLS LAST, rp.rewritten_at DESC NULLS LAST", + } + where_parts = ["rp.rewrite_status='ready'"] + base_args: list[Any] = [] + if editorial_statuses: + add_where(where_parts, base_args, "COALESCE(rp.editorial_status, 'review') = ANY(${i}::text[])", editorial_statuses) + if categories_selected: + add_where(where_parts, base_args, "COALESCE(rp.final_category, rp.rewrite_category, '') = ANY(${i}::text[])", categories_selected) + if source_ids: + add_where(where_parts, base_args, "rp.source_id = ANY(${i}::bigint[])", source_ids) + parsed_date_from = parse_date_filter(date_from) + parsed_date_to = parse_date_filter(date_to) + if parsed_date_from: + add_where(where_parts, base_args, "timezone('Asia/Yekaterinburg', COALESCE(rp.posted_at, rp.created_at))::date >= ${i}::date", parsed_date_from) + if parsed_date_to: + add_where(where_parts, base_args, "timezone('Asia/Yekaterinburg', COALESCE(rp.posted_at, rp.created_at))::date <= ${i}::date", parsed_date_to) + if score_min.strip().isdigit(): + add_where(where_parts, base_args, "rp.qualification_score >= ${i}::int", int(score_min.strip())) + if score_max.strip().isdigit(): + add_where(where_parts, base_args, "rp.qualification_score <= ${i}::int", int(score_max.strip())) + where_sql = " AND ".join(where_parts) + total = int( + await pool.fetchval( + f""" + SELECT COUNT(*) + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + WHERE {where_sql} + """, + *base_args, + ) + or 0 + ) + pager = pagination(page, per_page, total) + limit_index = len(base_args) + 1 + offset_index = len(base_args) + 2 + rows = await pool.fetch( + f""" + SELECT rp.*, s.name AS source_name, s.tag AS source_tag, s.platform AS source_platform, s.url AS source_url, + u.login AS reviewed_by_login, + COALESCE(c_id.tag, c_name.tag) AS resolved_category_tag, + COUNT(rpm.id) AS media_count, + COALESCE( + jsonb_agg( + jsonb_build_object( + 'id', rpm.id, + 'type', rpm.media_type, + 'url', rpm.original_url, + 'status', rpm.status, + 'error', rpm.error, + 'duration_sec', rpm.duration_sec + ) + ORDER BY rpm.sort_order ASC, rpm.id ASC + ) FILTER (WHERE rpm.id IS NOT NULL), + '[]'::jsonb + ) AS media_items, + COALESCE( + ( + SELECT jsonb_agg( + jsonb_build_object( + 'platform', pp.platform, + 'status', pp.status, + 'target_id', pp.target_id, + 'external_id', pp.external_id, + 'url', pp.url, + 'error', pp.error, + 'published_at', pp.published_at + ) + ORDER BY pp.platform + ) + FROM post_publications pp + WHERE pp.raw_post_id=rp.id + ), + '[]'::jsonb + ) AS platform_publications + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + LEFT JOIN admin_users u ON u.id=rp.reviewed_by + LEFT JOIN content_categories c_id ON c_id.sort_order=COALESCE(rp.final_category_id, rp.rewrite_category_id) + LEFT JOIN content_categories c_name ON lower(c_name.name)=lower(COALESCE(rp.final_category, rp.rewrite_category, '')) + LEFT JOIN raw_post_media rpm ON rpm.raw_post_id=rp.id AND COALESCE(rpm.editor_hidden, FALSE)=FALSE + WHERE {where_sql} + GROUP BY rp.id, s.name, s.tag, s.platform, s.url, u.login, c_id.tag, c_name.tag + ORDER BY + CASE COALESCE(rp.editorial_status, 'review') + WHEN 'review' THEN 0 + WHEN 'edited' THEN 1 + WHEN 'regenerating' THEN 2 + WHEN 'rejected' THEN 3 + WHEN 'accepted' THEN 4 + WHEN 'publish_failed' THEN 5 + WHEN 'published' THEN 6 + ELSE 7 + END, + {sort_map.get(sort, sort_map["rewritten_desc"])} + LIMIT ${limit_index} OFFSET ${offset_index} + """, + *base_args, + pager["per_page"], + pager["offset"], + ) + posts = [prepare_editor_post(row) for row in rows] + categories = await writer_category_options() + counts_rows = await pool.fetch( + """ + SELECT COALESCE(editorial_status, 'review') AS status, COUNT(*) AS count + FROM raw_posts + WHERE rewrite_status='ready' + GROUP BY COALESCE(editorial_status, 'review') + """ + ) + counts = {str(row["status"]): int(row["count"]) for row in counts_rows} + template_name = "editor_content.html" if request.headers.get("hx-request") else "editor.html" + return templates.TemplateResponse( + template_name, + base_context( + request, + user, + posts=posts, + categories=categories, + status_options=EDITOR_STATUS_OPTIONS, + status_filter=editorial_statuses[0] if len(editorial_statuses) == 1 else "", + editorial_statuses=editorial_statuses, + categories_selected=categories_selected, + source_ids=source_ids, + date_from=date_from, + date_to=date_to, + score_min=score_min, + score_max=score_max, + sort=sort, + facets=await editor_filter_facets(pool), + counts=counts, + preserved_query=preserved_query(request, {"page", "per_page"}), + pagination=pager, + ), + ) + + +@app.post("/editor/{post_id}/accept") +async def editor_accept(request: Request, post_id: int, csrf_token: str = Form(...)): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + pool = await get_pool() + row = await pool.fetchrow( + """ + SELECT rp.final_text, + rp.rewritten_text, + COALESCE(rp.final_category, rp.rewrite_category, '') AS category_name, + COALESCE(rp.final_category_id, rp.rewrite_category_id) AS category_id, + COALESCE(rp.final_source_tag, rp.rewrite_source_tag, s.tag, '') AS source_tag + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + WHERE rp.id=$1 AND rp.rewrite_status='ready' + """, + post_id, + ) + if not row: + return redirect("/editor") + category = await resolve_category_for_editor(row["category_name"], row["category_id"]) + source_tag = normalize_hash_tag(row["source_tag"] or "", "source") + final_text = row["final_text"] or row["rewritten_text"] or "" + await pool.execute( + """ + UPDATE raw_posts + SET editorial_status='accepted', + final_text=$2, + final_category=$3, + final_category_tag=$4, + final_source_tag=$5, + final_category_id=$6, + reviewed_by=$7, + reviewed_at=NOW(), + updated_at=NOW() + WHERE id=$1 AND rewrite_status='ready' + """, + post_id, + final_text, + category["name"], + category["tag"], + source_tag, + category["id"], + user["id"], + ) + await audit(user["id"], "editor.accept", "raw_post", post_id) + if request.headers.get("hx-request"): + return await editor_post_update_response(request, user, pool, post_id, "Принято к публикации", "success") + return redirect("/editor") + + +@app.post("/editor/{post_id}/reject") +async def editor_reject(request: Request, post_id: int, csrf_token: str = Form(...), editor_notes: str = Form("")): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + pool = await get_pool() + await pool.execute( + """ + UPDATE raw_posts + SET editorial_status='rejected', + editor_notes=$2, + reviewed_by=$3, + reviewed_at=NOW(), + updated_at=NOW() + WHERE id=$1 AND rewrite_status='ready' + """, + post_id, + editor_notes.strip()[:1000], + user["id"], + ) + await audit(user["id"], "editor.reject", "raw_post", post_id) + if request.headers.get("hx-request"): + return await editor_post_update_response(request, user, pool, post_id, "Отклонено", "error") + return redirect("/editor") + + +@app.post("/editor/{post_id}/return") +async def editor_return_to_review(request: Request, post_id: int, csrf_token: str = Form(...)): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + pool = await get_pool() + await pool.execute( + """ + UPDATE raw_posts + SET editorial_status=NULL, + reviewed_by=NULL, + reviewed_at=NULL, + updated_at=NOW() + WHERE id=$1 AND rewrite_status='ready' + """, + post_id, + ) + await audit(user["id"], "editor.return_to_review", "raw_post", post_id) + if request.headers.get("hx-request"): + return await editor_post_update_response(request, user, pool, post_id, "Возвращено на проверку", "warning") + return redirect("/editor") + + +@app.post("/editor/{post_id}/save") +async def editor_save( + request: Request, + post_id: int, + csrf_token: str = Form(...), + final_text: str = Form(...), + final_category: str = Form(...), + final_source_tag: str = Form(""), + editor_notes: str = Form(""), + action: str = Form("save"), + delete_media_ids: list[int] | None = Form(default=None), + media_files: list[UploadFile] | None = File(default=None), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + final_category = final_category.strip() + category = await resolve_category_for_editor(final_category) + final_category = category["name"] + final_category_tag = category["tag"] + final_source_tag = normalize_hash_tag(final_source_tag, "source") + status_value = "accepted" if action == "accept" else "edited" + pool = await get_pool() + await pool.execute( + """ + UPDATE raw_posts + SET editorial_status=$2, + final_text=$3, + final_category=$4, + final_category_tag=$5, + final_source_tag=$6, + editor_notes=$7, + final_category_id=$8, + reviewed_by=CASE WHEN $2='accepted' THEN $9 ELSE reviewed_by END, + reviewed_at=CASE WHEN $2='accepted' THEN NOW() ELSE reviewed_at END, + edited_at=NOW(), + updated_at=NOW() + WHERE id=$1 AND rewrite_status='ready' + """, + post_id, + status_value, + final_text.strip(), + final_category, + final_category_tag, + final_source_tag, + editor_notes.strip()[:1000], + category["id"], + user["id"], + ) + delete_media_ids = delete_media_ids or [] + media_files = media_files or [] + if delete_media_ids: + local_media_paths = [ + uploaded_editor_media_path(str(row["original_url"] or "")) + for row in await pool.fetch( + """ + SELECT original_url + FROM raw_post_media + WHERE raw_post_id=$1 + AND id=ANY($2::bigint[]) + AND editor_added=TRUE + """, + post_id, + delete_media_ids, + ) + ] + await pool.execute( + """ + UPDATE raw_post_media + SET editor_hidden=TRUE, + updated_at=NOW() + WHERE raw_post_id=$1 AND id=ANY($2::bigint[]) + """, + post_id, + delete_media_ids, + ) + for path in local_media_paths: + if path and path.is_file(): + try: + path.unlink(missing_ok=True) + except OSError: + logger.warning("Could not delete editor media file {}", path) + for media_file in media_files: + if not media_file.filename: + continue + content = await media_file.read() + if not content or len(content) > 50 * 1024 * 1024: + continue + original_name = Path(media_file.filename or "media").name + suffix = Path(original_name).suffix.lower() + if not suffix: + guessed = mimetypes.guess_extension(media_file.content_type or "") + suffix = guessed or ".bin" + safe_name = f"{post_id}_{int(time.time())}_{secrets.token_hex(6)}{suffix}" + target = EDITOR_MEDIA_DIR / safe_name + target.write_bytes(content) + public_url = f"/uploads/editor_media/{safe_name}" + content_type = media_file.content_type or mimetypes.guess_type(original_name)[0] or "" + if content_type.startswith("image/"): + media_type = "photo" + elif content_type.startswith("video/"): + media_type = "video" + else: + media_type = "doc" + sort_order = int( + await pool.fetchval("SELECT COALESCE(MAX(sort_order), -1) + 1 FROM raw_post_media WHERE raw_post_id=$1", post_id) + or 0 + ) + await pool.execute( + """ + INSERT INTO raw_post_media( + raw_post_id, platform, media_type, original_url, preview_url, sort_order, status, + editor_added, editor_uploaded_by, editor_uploaded_at + ) + VALUES($1, 'manual', $2, $3, $3, $4, 'ready', TRUE, $5, NOW()) + """, + post_id, + media_type, + public_url, + sort_order, + user["id"], + ) + await audit(user["id"], "editor.save" if status_value == "edited" else "editor.save_accept", "raw_post", post_id) + if request.headers.get("hx-request"): + message = "Принято к публикации" if status_value == "accepted" else "Сохранено как черновик" + return await editor_post_update_response(request, user, pool, post_id, message, "success") + return redirect("/editor") + + +@app.get("/raw/{post_id}", response_class=HTMLResponse) +async def raw_post_detail(request: Request, post_id: int): + user = await get_current_user(request) + if not user: + return redirect("/login") + pool = await get_pool() + row = await pool.fetchrow( + """ + SELECT rp.*, s.name AS source_name, s.tag AS source_tag, s.platform AS source_platform, s.url AS source_url, + COUNT(rpm.id) AS media_count, + COALESCE( + jsonb_agg( + jsonb_build_object( + 'id', rpm.id, + 'type', rpm.media_type, + 'url', rpm.original_url, + 'status', rpm.status, + 'error', rpm.error, + 'duration_sec', rpm.duration_sec + ) + ORDER BY rpm.sort_order ASC, rpm.id ASC + ) FILTER (WHERE rpm.id IS NOT NULL), + '[]'::jsonb + ) AS media_items, + b.provider AS qualification_provider, + b.model AS batch_model, + b.status AS batch_status, + b.posts_count AS batch_posts_count, + b.accepted_count AS batch_accepted_count, + b.rejected_count AS batch_rejected_count, + b.maybe_count AS batch_maybe_count, + b.error AS batch_error, + b.prompt_text AS batch_prompt_text, + b.created_at AS batch_created_at, + b.completed_at AS batch_completed_at, + b.response_json AS batch_response_json, + b.prompt_tokens AS batch_prompt_tokens, + b.completion_tokens AS batch_completion_tokens, + b.total_tokens AS batch_total_tokens, + b.estimated_cost_usd AS batch_estimated_cost_usd, + wb.provider AS rewrite_provider, + wb.model AS writer_batch_model, + wb.status AS writer_batch_status, + wb.posts_count AS writer_batch_posts_count, + wb.ready_count AS writer_batch_ready_count, + wb.failed_count AS writer_batch_failed_count, + wb.error AS writer_batch_error, + wb.prompt_text AS writer_batch_prompt_text, + wb.created_at AS writer_batch_created_at, + wb.completed_at AS writer_batch_completed_at, + wb.response_json AS writer_batch_response_json, + wb.prompt_tokens AS writer_prompt_tokens, + wb.completion_tokens AS writer_completion_tokens, + wb.total_tokens AS writer_total_tokens, + wb.estimated_cost_usd AS writer_estimated_cost_usd + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + LEFT JOIN raw_post_media rpm ON rpm.raw_post_id=rp.id + LEFT JOIN ai_qualification_batches b ON b.id=rp.qualification_batch_id + LEFT JOIN ai_writer_batches wb ON wb.id=rp.rewrite_batch_id + WHERE rp.id=$1 + GROUP BY rp.id, s.name, s.tag, s.platform, s.url, b.id, wb.id + """, + post_id, + ) + if not row: + return redirect("/raw") + post = prepare_raw_post(row) + post["batch_created_at_fmt"] = format_dt(post.get("batch_created_at")) + post["batch_completed_at_fmt"] = format_dt(post.get("batch_completed_at")) + response_json = post.get("batch_response_json") or {} + if isinstance(response_json, str): + try: + response_json = json.loads(response_json) + except json.JSONDecodeError: + pass + post["batch_response_pretty"] = json.dumps(response_json, ensure_ascii=False, indent=2) + if not post.get("batch_prompt_text") and post.get("qualification_batch_id"): + post["batch_prompt_text"] = ( + "Точный prompt для этого старого batch ещё не сохранялся.\n" + f"Prompt hash: {post.get('qualification_prompt_hash') or '—'}" + ) + else: + post["batch_prompt_text"] = post.get("batch_prompt_text") or "" + writer_response_json = post.get("writer_batch_response_json") or {} + if isinstance(writer_response_json, str): + try: + writer_response_json = json.loads(writer_response_json) + except json.JSONDecodeError: + pass + post["writer_batch_created_at_fmt"] = format_dt(post.get("writer_batch_created_at")) + post["writer_batch_completed_at_fmt"] = format_dt(post.get("writer_batch_completed_at")) + post["rewritten_at_fmt"] = format_dt(post.get("rewritten_at")) + post["writer_batch_response_pretty"] = json.dumps(writer_response_json, ensure_ascii=False, indent=2) + if not post.get("writer_batch_prompt_text") and post.get("rewrite_batch_id"): + post["writer_batch_prompt_text"] = ( + "Точный prompt для этого старого writer batch ещё не сохранялся.\n" + f"Prompt hash: {post.get('rewrite_prompt_hash') or '—'}" + ) + else: + post["writer_batch_prompt_text"] = post.get("writer_batch_prompt_text") or "" + return templates.TemplateResponse("raw_post_detail.html", base_context(request, user, post=post)) + + +@app.get("/prompt-test", response_class=HTMLResponse) +async def prompt_test(request: Request, post_id: int | None = None, q_status: str = "all"): + user = await get_current_user(request) + if not user: + return redirect("/login") + q_status = normalize_prompt_test_status(q_status) + posts, selected = await prompt_test_posts(post_id, q_status) + category_payload = await writer_category_payload() + max_text_chars = max(200, int(await fetch_setting("ai_writer_max_text_chars", 3500) or 3500)) + prompt = str(await fetch_setting("ai_writer_prompt", "") or "") + provider = str(await fetch_setting("ai_writer_provider", "anthropic") or "anthropic") + model = str(await fetch_setting("ai_writer_model", "") or "") + payload = prompt_test_payload(selected, max_text_chars) if selected else [] + if payload: + payload["categories"] = category_payload + return templates.TemplateResponse( + "prompt_test.html", + base_context( + request, + user, + posts=posts, + selected=selected, + prompt=prompt, + provider=provider, + model=model, + q_status=q_status, + status_options=PROMPT_TEST_STATUS_OPTIONS, + max_text_chars=max_text_chars, + payload_json=json.dumps(payload, ensure_ascii=False, indent=2), + result=None, + error=None, + ), + ) + + +@app.post("/prompt-test", response_class=HTMLResponse) +async def prompt_test_run( + request: Request, + csrf_token: str = Form(...), + post_id: int = Form(...), + prompt: str = Form(...), + q_status: str = Form("all"), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + q_status = normalize_prompt_test_status(q_status) + posts, selected = await prompt_test_posts(post_id, q_status) + category_payload = await writer_category_payload() + max_text_chars = max(200, int(await fetch_setting("ai_writer_max_text_chars", 3500) or 3500)) + provider = str(await fetch_setting("ai_writer_provider", "anthropic") or "anthropic") + model = str(await fetch_setting("ai_writer_model", "") or "").strip() + api_key = str(await fetch_setting("ai_writer_api_key", "") or "").strip() + api_base = str(await fetch_setting("ai_writer_api_base", "") or "").strip() + temperature = max(0.0, float(await fetch_setting("ai_writer_temperature", 0.4) or 0.4)) + timeout = max(10, int(await fetch_setting("ai_writer_timeout_sec", 180) or 180)) + normalized_prompt = prompt.replace("\\r\\n", "\n").replace("\\n", "\n").strip() + payload = prompt_test_payload(selected, max_text_chars) if selected else [] + if payload: + payload["categories"] = category_payload + result = None + error = None + if not selected: + error = "Пост не найден." + elif not model or not api_key: + error = "Не заполнены модель или API-ключ райтера в настройках." + elif not normalized_prompt: + error = "Промпт пустой." + else: + try: + import litellm + + kwargs: dict[str, Any] = { + "model": normalize_model(provider, model), + "messages": [ + {"role": "system", "content": normalized_prompt}, + {"role": "user", "content": json.dumps(payload, ensure_ascii=False)}, + ], + "temperature": temperature, + "timeout": timeout, + } + if api_key: + kwargs["api_key"] = api_key + if api_base: + kwargs["api_base"] = api_base + response = await asyncio.to_thread(litellm.completion, **kwargs) + content = response.choices[0].message.content + parsed = None + raw_json = str(content or "").strip() + try: + if raw_json.startswith("```"): + raw_json = raw_json.strip("`") + if raw_json.lower().startswith("json"): + raw_json = raw_json[4:].strip() + parsed = json.loads(raw_json) + except Exception: + parsed = None + result = { + "content": content, + "parsed_json": json.dumps(parsed, ensure_ascii=False, indent=2) if parsed is not None else "", + "preview_text": prompt_test_preview_text(parsed), + "preview_category": prompt_test_preview_category(parsed), + "preview_notes": prompt_test_preview_notes(parsed), + "usage": response_usage(response), + "model": kwargs["model"], + } + await audit(user["id"], "writer_prompt_test.run", "raw_post", int(selected["id"]), {"provider": provider, "model": model}) + except Exception as exc: + logger.exception("Prompt test failed") + error = str(exc) + return templates.TemplateResponse( + "prompt_test.html", + base_context( + request, + user, + posts=posts, + selected=selected, + prompt=normalized_prompt, + provider=provider, + model=model, + q_status=q_status, + status_options=PROMPT_TEST_STATUS_OPTIONS, + max_text_chars=max_text_chars, + payload_json=json.dumps(payload, ensure_ascii=False, indent=2), + result=result, + error=error, + ), + ) + + +@app.get("/workers", response_class=HTMLResponse) +async def workers(request: Request): + user = await get_current_user(request) + if not user: + return redirect("/login") + pool = await get_pool() + rows = await pool.fetch( + """ + SELECT wc.name, wc.enabled, wc.settings_json, wh.heartbeat_at, wh.status, wh.current_job_id, wh.meta_json + FROM worker_controls wc + LEFT JOIN worker_heartbeats wh ON wh.name=wc.name + ORDER BY wc.name + """ + ) + settings_rows = await pool.fetch("SELECT * FROM app_settings ORDER BY category, key") + settings = [] + for row in settings_rows: + item = dict(row) + if isinstance(item.get("value_json"), str): + try: + item["value_json"] = json.loads(item["value_json"]) + except json.JSONDecodeError: + pass + if item.get("key") in {"ai_writer_categories", "tg_poster_schedule_json", "vk_poster_schedule_json", *AI_WORKER_ENABLED_SETTINGS.values()}: + continue + settings.append(item) + settings.sort(key=setting_sort_key) + ai_provider = str(setting_value(settings, "ai_qualifier_provider", "openrouter") or "openrouter") + ai_model = str(setting_value(settings, "ai_qualifier_model", "") or "") + ai_api_key = str(setting_value(settings, "ai_qualifier_api_key", "") or "") + ai_model_options = await model_options_for_provider(ai_provider, ai_api_key) + writer_provider = str(setting_value(settings, "ai_writer_provider", "anthropic") or "anthropic") + writer_model = str(setting_value(settings, "ai_writer_model", "") or "") + writer_api_key = str(setting_value(settings, "ai_writer_api_key", "") or "") + writer_model_options = await model_options_for_provider(writer_provider, writer_api_key) + return templates.TemplateResponse( + "workers.html", + base_context( + request, + user, + workers=[ + { + **dict(r), + "meta_json": json.loads(r["meta_json"]) if isinstance(r["meta_json"], str) else (r["meta_json"] or {}), + } + for r in rows + ], + settings=settings, + provider_options=PROVIDER_OPTIONS, + ai_provider=ai_provider, + ai_model=ai_model, + ai_model_options=ai_model_options, + writer_provider=writer_provider, + writer_model=writer_model, + writer_model_options=writer_model_options, + categories=await category_rows(), + tg_schedule=await tg_poster_schedule_rows(), + vk_schedule=await vk_poster_schedule_rows(), + category_titles=CATEGORY_TITLES, + prompt_hints=PROMPT_HINTS, + ), + ) + + +@app.post("/tg-poster-schedule/create") +async def tg_poster_schedule_create( + request: Request, + csrf_token: str = Form(...), + time_value: str = Form(..., alias="time"), + count: int = Form(1), + enabled: str | None = Form(default=None), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + rows = await tg_poster_schedule_rows() + rows.append( + { + "id": f"slot-{secrets.token_hex(4)}", + "time": time_value, + "count": count, + "enabled": enabled == "on", + } + ) + rows = normalize_tg_schedule(rows) + await save_tg_poster_schedule(rows) + await audit(user["id"], "tg_schedule.create", "tg_poster_schedule", None, {"rows": rows}) + return redirect("/workers") + + +@app.post("/tg-poster-schedule/{schedule_id}/update") +async def tg_poster_schedule_update( + request: Request, + schedule_id: str, + csrf_token: str = Form(...), + time_value: str = Form(..., alias="time"), + count: int = Form(1), + enabled: str | None = Form(default=None), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + rows = await tg_poster_schedule_rows() + for row in rows: + if row["id"] == schedule_id: + row.update({"time": time_value, "count": count, "enabled": enabled == "on"}) + break + rows = normalize_tg_schedule(rows) + await save_tg_poster_schedule(rows) + await audit(user["id"], "tg_schedule.update", "tg_poster_schedule", None, {"id": schedule_id, "rows": rows}) + return redirect("/workers") + + +@app.post("/tg-poster-schedule/{schedule_id}/delete") +async def tg_poster_schedule_delete(request: Request, schedule_id: str, csrf_token: str = Form(...)): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + rows = [row for row in await tg_poster_schedule_rows() if row["id"] != schedule_id] + await save_tg_poster_schedule(rows) + await audit(user["id"], "tg_schedule.delete", "tg_poster_schedule", None, {"id": schedule_id, "rows": rows}) + return redirect("/workers") + + +@app.post("/vk-poster-schedule/create") +async def vk_poster_schedule_create( + request: Request, + csrf_token: str = Form(...), + time_value: str = Form(..., alias="time"), + count: int = Form(1), + enabled: str | None = Form(default=None), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + rows = await vk_poster_schedule_rows() + rows.append({"id": f"vk-slot-{secrets.token_hex(4)}", "time": time_value, "count": count, "enabled": enabled == "on"}) + rows = normalize_tg_schedule(rows) + await save_vk_poster_schedule(rows) + await audit(user["id"], "vk_schedule.create", "vk_poster_schedule", None, {"rows": rows}) + return redirect("/workers") + + +@app.post("/vk-poster-schedule/{schedule_id}/update") +async def vk_poster_schedule_update( + request: Request, + schedule_id: str, + csrf_token: str = Form(...), + time_value: str = Form(..., alias="time"), + count: int = Form(1), + enabled: str | None = Form(default=None), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + rows = await vk_poster_schedule_rows() + for row in rows: + if row["id"] == schedule_id: + row.update({"time": time_value, "count": count, "enabled": enabled == "on"}) + break + rows = normalize_tg_schedule(rows) + await save_vk_poster_schedule(rows) + await audit(user["id"], "vk_schedule.update", "vk_poster_schedule", None, {"id": schedule_id, "rows": rows}) + return redirect("/workers") + + +@app.post("/vk-poster-schedule/{schedule_id}/delete") +async def vk_poster_schedule_delete(request: Request, schedule_id: str, csrf_token: str = Form(...)): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + rows = [row for row in await vk_poster_schedule_rows() if row["id"] != schedule_id] + await save_vk_poster_schedule(rows) + await audit(user["id"], "vk_schedule.delete", "vk_poster_schedule", None, {"id": schedule_id, "rows": rows}) + return redirect("/workers") + + +@app.post("/categories/create") +async def category_create( + request: Request, + csrf_token: str = Form(...), + name: str = Form(...), + tag: str = Form(""), + site_name: str = Form(...), + site_slug: str = Form(...), + site_enabled: str | None = Form(None), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + name = name.strip().strip("#") + if not name: + return redirect("/workers") + tag = normalize_hash_tag(tag or name, "category") + site_name = site_name.strip() + site_slug = re.sub(r"[^a-z0-9-]+", "-", site_slug.strip().lower().replace("_", "-")).strip("-") + if not site_name or not site_slug: + return redirect("/workers") + pool = await get_pool() + sort_order = int(await pool.fetchval("SELECT COALESCE(MAX(sort_order), 0) + 1 FROM content_categories") or 1) + await pool.execute( + """ + INSERT INTO content_categories(name, tag, sort_order, site_name, site_slug, site_enabled) + VALUES($1, $2, $3, $4, $5, $6) + ON CONFLICT (name) DO UPDATE + SET tag=$2, + site_name=$4, + site_slug=$5, + site_enabled=$6, + is_active=TRUE, + updated_at=NOW() + """, + name, + tag, + sort_order, + site_name, + site_slug, + site_enabled is not None, + ) + await audit(user["id"], "category.create", "content_category", None, {"name": name, "tag": tag, "site_name": site_name, "site_slug": site_slug, "site_enabled": site_enabled is not None}) + return redirect("/workers") + + +@app.post("/categories/{category_id}/update") +async def category_update( + request: Request, + category_id: int, + csrf_token: str = Form(...), + name: str = Form(...), + tag: str = Form(""), + site_name: str = Form(...), + site_slug: str = Form(...), + site_enabled: str | None = Form(None), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + name = name.strip().strip("#") + if not name: + return redirect("/workers") + tag = normalize_hash_tag(tag or name, "category") + site_name = site_name.strip() + site_slug = re.sub(r"[^a-z0-9-]+", "-", site_slug.strip().lower().replace("_", "-")).strip("-") + if not site_name or not site_slug: + return redirect("/workers") + pool = await get_pool() + await pool.execute( + """ + UPDATE content_categories + SET name=$2, + tag=$3, + site_name=$4, + site_slug=$5, + site_enabled=$6, + updated_at=NOW() + WHERE id=$1 + """, + category_id, + name, + tag, + site_name, + site_slug, + site_enabled is not None, + ) + await audit(user["id"], "category.update", "content_category", category_id, {"name": name, "tag": tag, "site_name": site_name, "site_slug": site_slug, "site_enabled": site_enabled is not None}) + return redirect("/workers") + + +@app.post("/categories/{category_id}/toggle") +async def category_toggle(request: Request, category_id: int, csrf_token: str = Form(...)): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + pool = await get_pool() + row = await pool.fetchrow( + """ + UPDATE content_categories + SET is_active=NOT is_active, + updated_at=NOW() + WHERE id=$1 + RETURNING name, is_active + """, + category_id, + ) + await audit( + user["id"], + "category.toggle", + "content_category", + category_id, + {"name": row["name"] if row else None, "is_active": bool(row["is_active"]) if row else None}, + ) + return redirect("/workers") + + +@app.post("/categories/{category_id}/delete") +async def category_delete(request: Request, category_id: int, csrf_token: str = Form(...)): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + pool = await get_pool() + row = await pool.fetchrow( + """ + UPDATE content_categories + SET is_active=FALSE, + updated_at=NOW() + WHERE id=$1 + RETURNING name + """, + category_id, + ) + await audit(user["id"], "category.archive", "content_category", category_id, {"name": row["name"] if row else None}) + return redirect("/workers") + + +@app.post("/workers/{worker_name}/toggle") +async def worker_toggle(request: Request, worker_name: str, csrf_token: str = Form(...)): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + pool = await get_pool() + async with pool.acquire() as conn: + async with conn.transaction(): + row = await conn.fetchrow( + """ + UPDATE worker_controls + SET enabled=NOT enabled, + updated_by=$2, + updated_at=NOW() + WHERE name=$1 + RETURNING enabled + """, + worker_name, + user["id"], + ) + if not row: + return redirect("/workers") + enabled = bool(row["enabled"]) + setting_key = AI_WORKER_ENABLED_SETTINGS.get(worker_name) + if setting_key: + await conn.execute( + """ + UPDATE app_settings + SET value_json=$2::jsonb, + updated_by=$3, + updated_at=NOW() + WHERE key=$1 + """, + setting_key, + json.dumps(enabled), + user["id"], + ) + await audit( + user["id"], + "worker.toggle", + "worker", + None, + { + "name": worker_name, + "enabled": enabled, + "synced_setting": AI_WORKER_ENABLED_SETTINGS.get(worker_name), + }, + ) + return redirect("/workers") + + +@app.post("/settings/save") +async def settings_save(request: Request, csrf_token: str = Form(...), key: str = Form(...), value: str = Form(...)): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + pool = await get_pool() + row = await pool.fetchrow("SELECT value_type FROM app_settings WHERE key=$1", key) + if not row: + return redirect("/workers") + value_type = str(row["value_type"]) + if value_type == "int": + parsed: Any = int(value) + elif value_type == "float": + parsed = float(value) + elif value_type == "bool": + parsed = str(value).lower() in {"1", "true", "yes", "on"} + elif value_type == "secret": + if not value.strip(): + return redirect("/workers") + parsed = value.strip() + elif value_type == "json": + parsed = json.loads(value) + else: + parsed = value + if key in {"ai_qualifier_prompt", "ai_qualifier_contract", "ai_writer_prompt", "ai_writer_contract"}: + parsed = parsed.replace("\\r\\n", "\n").replace("\\n", "\n") + await pool.execute( + """ + UPDATE app_settings + SET value_json=$2::jsonb, + updated_by=$3, + updated_at=NOW() + WHERE key=$1 + """, + key, + json.dumps(parsed, ensure_ascii=False), + user["id"], + ) + await audit(user["id"], "setting.update", "setting", None, {"key": key, "value": parsed}) + return redirect("/workers") + + +@app.get("/logs", response_class=HTMLResponse) +async def logs_page( + request: Request, + q: str = "", + action: str = "", + actor_id: int = 0, + entity_type: str = "", + date_from: str = "", + date_to: str = "", + page: int = 1, + per_page: int = 50, +): + user = await get_current_user(request) + if not user: + return redirect("/login") + pool = await get_pool() + await prune_audit_log() + where_parts: list[str] = [] + args: list[Any] = [] + if q.strip(): + add_where( + where_parts, + args, + "(al.action ILIKE ${i} OR al.entity_type ILIKE ${i} OR COALESCE(al.after_json::text, '') ILIKE ${i})", + f"%{q.strip()}%", + ) + if action.strip(): + add_where(where_parts, args, "al.action=${i}", action.strip()) + if actor_id > 0: + add_where(where_parts, args, "al.actor_id=${i}", actor_id) + if entity_type.strip(): + add_where(where_parts, args, "al.entity_type=${i}", entity_type.strip()) + parsed_date_from = parse_date_filter(date_from) + parsed_date_to = parse_date_filter(date_to) + if parsed_date_from: + add_where(where_parts, args, "timezone('Asia/Yekaterinburg', al.created_at)::date >= ${i}::date", parsed_date_from) + if parsed_date_to: + add_where(where_parts, args, "timezone('Asia/Yekaterinburg', al.created_at)::date <= ${i}::date", parsed_date_to) + where_sql = " AND ".join(where_parts) if where_parts else "TRUE" + total = int( + await pool.fetchval( + f""" + SELECT COUNT(*) + FROM audit_log al + LEFT JOIN admin_users u ON u.id=al.actor_id + WHERE {where_sql} + """, + *args, + ) + or 0 + ) + pager = pagination(page, per_page, total) + limit_index = len(args) + 1 + offset_index = len(args) + 2 + rows = await pool.fetch( + f""" + SELECT al.*, u.login AS actor_login + FROM audit_log al + LEFT JOIN admin_users u ON u.id=al.actor_id + WHERE {where_sql} + ORDER BY al.created_at DESC, al.id DESC + LIMIT ${limit_index} OFFSET ${offset_index} + """, + *args, + pager["per_page"], + pager["offset"], + ) + action_rows = await pool.fetch( + """ + SELECT action, COUNT(*) AS count + FROM audit_log + WHERE created_at >= NOW() - INTERVAL '30 days' + GROUP BY action + ORDER BY COUNT(*) DESC, action + LIMIT 80 + """ + ) + actor_rows = await pool.fetch( + """ + SELECT u.id, u.login, COUNT(al.id) AS count + FROM audit_log al + JOIN admin_users u ON u.id=al.actor_id + WHERE al.created_at >= NOW() - INTERVAL '30 days' + GROUP BY u.id, u.login + ORDER BY COUNT(al.id) DESC, u.login + """ + ) + entity_rows = await pool.fetch( + """ + SELECT entity_type, COUNT(*) AS count + FROM audit_log + WHERE created_at >= NOW() - INTERVAL '30 days' + GROUP BY entity_type + ORDER BY COUNT(*) DESC, entity_type + LIMIT 50 + """ + ) + return templates.TemplateResponse( + "logs.html", + base_context( + request, + user, + logs=[prepare_audit_row(row) for row in rows], + q=q, + action=action, + actor_id=actor_id, + entity_type=entity_type, + date_from=date_from, + date_to=date_to, + actions=[dict(row) for row in action_rows], + actors=[dict(row) for row in actor_rows], + entity_types=[dict(row) for row in entity_rows], + preserved_query=preserved_query(request, {"page", "per_page"}), + pagination=pager, + ), + ) + + +@app.get("/users", response_class=HTMLResponse) +async def users(request: Request, page: int = 1, per_page: int = 50): + user = await get_current_user(request) + if not user: + return redirect("/login") + pool = await get_pool() + total = int(await pool.fetchval("SELECT COUNT(*) FROM admin_users") or 0) + pager = pagination(page, per_page, total) + rows = await pool.fetch( + "SELECT id, login, role, is_active, created_at FROM admin_users ORDER BY id LIMIT $1 OFFSET $2", + pager["per_page"], + pager["offset"], + ) + users_rows = [] + for row in rows: + item = dict(row) + item["created_at_fmt"] = format_dt(item.get("created_at")) + users_rows.append(item) + return templates.TemplateResponse("users.html", base_context(request, user, users=users_rows, pagination=pager)) + + +@app.post("/users/create") +async def user_create( + request: Request, + csrf_token: str = Form(...), + login: str = Form(...), + password: str = Form(...), + role: str = Form("editor"), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + if user["role"] != "admin": + return redirect("/users") + role = role if role in {"admin", "editor", "viewer"} else "editor" + pool = await get_pool() + await pool.execute( + "INSERT INTO admin_users(login, password_hash, role) VALUES($1, $2, $3)", + login.strip().lower(), + hash_password(password), + role, + ) + await audit(user["id"], "user.create", "user", None, {"login": login, "role": role}) + return redirect("/users") + + +@app.post("/users/{user_id}/toggle") +async def user_toggle(request: Request, user_id: int, csrf_token: str = Form(...)): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + if user["role"] != "admin" or int(user["id"]) == user_id: + return redirect("/users") + pool = await get_pool() + # Prevent deactivating the bootstrap admin account (managed via env secrets). + bootstrap_login = settings.admin_bootstrap_login.strip().lower() + target_login = await pool.fetchval("SELECT login FROM admin_users WHERE id=$1", user_id) + if target_login and target_login == bootstrap_login: + logger.warning("Attempt to deactivate bootstrap admin '{}' was blocked", bootstrap_login) + return redirect("/users") + await pool.execute("UPDATE admin_users SET is_active=NOT is_active, updated_at=NOW() WHERE id=$1", user_id) + await audit(user["id"], "user.toggle", "user", user_id) + return redirect("/users") + + +@app.post("/users/{user_id}/delete") +async def user_delete(request: Request, user_id: int, csrf_token: str = Form(...)): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + if user["role"] != "admin" or int(user["id"]) == user_id: + return redirect("/users") + pool = await get_pool() + bootstrap_login = settings.admin_bootstrap_login.strip().lower() + target = await pool.fetchrow("SELECT id, login, role FROM admin_users WHERE id=$1", user_id) + if not target: + return redirect("/users") + if str(target["login"]).strip().lower() == bootstrap_login: + logger.warning("Attempt to delete bootstrap admin '{}' was blocked", bootstrap_login) + return redirect("/users") + await pool.execute("DELETE FROM admin_users WHERE id=$1", user_id) + await audit(user["id"], "user.delete", "user", user_id, None, {"login": target["login"], "role": target["role"]}) + return redirect("/users") diff --git a/src/vk_parser_app/config.py b/src/vk_parser_app/config.py new file mode 100644 index 0000000..d83bda6 --- /dev/null +++ b/src/vk_parser_app/config.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + app_env: str = "production" + app_secret_key: str = "change-me" + admin_site_title: str = "" + admin_app_title: str = "Редакторская" + admin_bootstrap_login: str = "admin" + admin_bootstrap_password: str = "" + + db_host: str = "localhost" + db_port: int = 5432 + db_name: str = "vk_parser" + db_user: str = "vk_parser_user" + db_password: str = "" + + vk_access_token: str = "" + vk_group_access_token: str = "" + vk_api_version: str = "5.199" + vk_storage_group_id: int = 0 + + tg_bot_token: str = "" + tg_media_channel_id: str = "" + local_bot_api_url: str = "" + + admin_host: str = "0.0.0.0" + admin_port: int = 8080 + log_level: str = "INFO" + + model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") + + @property + def db_dsn(self) -> str: + return ( + f"postgresql://{self.db_user}:{self.db_password}" + f"@{self.db_host}:{self.db_port}/{self.db_name}" + ) + + @property + def display_admin_title(self) -> str: + return self.admin_app_title.strip() or "Редакторская" + + @property + def display_site_title(self) -> str: + return self.admin_site_title.strip() or "Редакторская" + + @property + def vk_storage_owner_id(self) -> int: + return -abs(int(self.vk_storage_group_id)) + + +settings = Settings() diff --git a/src/vk_parser_app/constants.py b/src/vk_parser_app/constants.py new file mode 100644 index 0000000..35d313f --- /dev/null +++ b/src/vk_parser_app/constants.py @@ -0,0 +1,40 @@ +PLATFORM_VK = "vk" + +SOURCE_STATUS_NEW = "new" +SOURCE_STATUS_OK = "ok" +SOURCE_STATUS_ERROR = "error" +SOURCE_STATUS_PAUSED = "paused" + +POST_STATUS_RAW_SAVED = "raw_saved" +POST_STATUS_STORAGE_PENDING = "storage_pending" +POST_STATUS_STORAGE_READY = "storage_ready" +POST_STATUS_SKIPPED = "skipped" +POST_STATUS_FAILED = "failed" + +PUBLICATION_STATUS_PENDING = "pending" +PUBLICATION_STATUS_PUBLISHED = "published" +PUBLICATION_STATUS_FAILED = "publish_failed" + +MEDIA_STATUS_PENDING = "pending" +MEDIA_STATUS_UPLOADED = "uploaded" +MEDIA_STATUS_LINK_ONLY = "link_only" +MEDIA_STATUS_FAILED = "failed" + +JOB_STATUS_PENDING = "pending" +JOB_STATUS_IN_PROGRESS = "in_progress" +JOB_STATUS_RETRY = "retry" +JOB_STATUS_DONE = "done" +JOB_STATUS_DEAD = "dead" + +JOB_TYPE_VK_STORAGE_COPY = "vk.storage.copy" + +WORKER_PARSER = "vk-parser" +WORKER_STORAGE_UPLOADER = "vk-storage-uploader" +WORKER_AI_QUALIFIER = "ai-qualifier" +WORKER_AI_WRITER = "ai-writer" +WORKER_TG_POSTER = "tg-poster" +WORKER_TG_REACTOR = "tg-reactor" +WORKER_VK_POSTER = "vk-poster" +WORKER_MAX_POSTER = "max-poster" +WORKER_SITE_POSTER = "site-poster" +WORKER_DAILY_REPORT = "daily-report" diff --git a/src/vk_parser_app/db.py b/src/vk_parser_app/db.py new file mode 100644 index 0000000..deadc9f --- /dev/null +++ b/src/vk_parser_app/db.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import asyncpg +from loguru import logger + +from .config import settings + +_pool: asyncpg.Pool | None = None + + +async def get_pool() -> asyncpg.Pool: + global _pool + if _pool is None: + _pool = await asyncpg.create_pool( + host=settings.db_host, + port=settings.db_port, + database=settings.db_name, + user=settings.db_user, + password=settings.db_password, + min_size=1, + max_size=10, + command_timeout=60, + ) + logger.info("Database pool created: {}:{}/{}", settings.db_host, settings.db_port, settings.db_name) + return _pool + + +async def close_pool() -> None: + global _pool + if _pool is not None: + await _pool.close() + _pool = None + + +async def fetch_setting(key: str, default: Any = None) -> Any: + pool = await get_pool() + row = await pool.fetchrow("SELECT value_json FROM app_settings WHERE key=$1", key) + if not row: + return default + value = row["value_json"] + if isinstance(value, str): + try: + return json.loads(value) + except json.JSONDecodeError: + return value + return value + + +async def fetch_int_setting(key: str, default: int) -> int: + try: + return int(await fetch_setting(key, default)) + except Exception: + return default + + +async def fetch_float_setting(key: str, default: float) -> float: + try: + return float(await fetch_setting(key, default)) + except Exception: + return default + + +async def fetch_bool_setting(key: str, default: bool) -> bool: + try: + value = await fetch_setting(key, default) + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() in {"1", "true", "yes", "on"} + return bool(value) + except Exception: + return default + + +async def apply_migrations(migrations_dir: Path) -> None: + pool = await get_pool() + async with pool.acquire() as conn: + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ) + """ + ) + for path in sorted(migrations_dir.glob("*.sql")): + version = path.name + exists = await conn.fetchval("SELECT 1 FROM schema_migrations WHERE version=$1", version) + if exists: + continue + sql = path.read_text(encoding="utf-8") + async with conn.transaction(): + await conn.execute(sql) + await conn.execute("INSERT INTO schema_migrations(version) VALUES($1)", version) + logger.info("Applied migration {}", version) diff --git a/src/vk_parser_app/heartbeat.py b/src/vk_parser_app/heartbeat.py new file mode 100644 index 0000000..8634cc0 --- /dev/null +++ b/src/vk_parser_app/heartbeat.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import json +import time +from typing import Any + + +class HeartbeatReporter: + def __init__(self, name: str, interval_sec: int = 30) -> None: + self.name = name + self.interval_sec = max(5, int(interval_sec)) + self._last = 0.0 + + async def beat( + self, + pool, + status: str = "running", + current_job_id: int | None = None, + meta: dict[str, Any] | None = None, + force: bool = False, + ) -> None: + now = time.monotonic() + if not force and now - self._last < self.interval_sec: + return + self._last = now + try: + async with pool.acquire() as conn: + await conn.execute( + """ + INSERT INTO worker_heartbeats(name, heartbeat_at, status, current_job_id, meta_json, updated_at) + VALUES ($1, NOW(), $2, $3, $4::jsonb, NOW()) + ON CONFLICT (name) DO UPDATE + SET heartbeat_at=NOW(), + status=EXCLUDED.status, + current_job_id=EXCLUDED.current_job_id, + meta_json=EXCLUDED.meta_json, + updated_at=NOW() + """, + self.name, + status, + current_job_id, + json.dumps(meta or {}, ensure_ascii=False), + ) + except Exception: + return diff --git a/src/vk_parser_app/jobs.py b/src/vk_parser_app/jobs.py new file mode 100644 index 0000000..a432849 --- /dev/null +++ b/src/vk_parser_app/jobs.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from .constants import JOB_STATUS_IN_PROGRESS, JOB_STATUS_PENDING, JOB_STATUS_RETRY + + +async def is_worker_enabled(pool, name: str) -> bool: + value = await pool.fetchval("SELECT enabled FROM worker_controls WHERE name=$1", name) + return bool(value) + + +async def claim_job(pool, job_type: str, worker_id: str) -> dict | None: + row = await pool.fetchrow( + """ + WITH cte AS ( + SELECT id + FROM jobs + WHERE type=$1 + AND status IN ($2, $3) + AND next_run_at <= NOW() + ORDER BY next_run_at ASC, id ASC + FOR UPDATE SKIP LOCKED + LIMIT 1 + ) + UPDATE jobs j + SET status=$4, + locked_by=$5, + locked_at=NOW(), + updated_at=NOW() + FROM cte + WHERE j.id=cte.id + RETURNING j.* + """, + job_type, + JOB_STATUS_PENDING, + JOB_STATUS_RETRY, + JOB_STATUS_IN_PROGRESS, + worker_id, + ) + return dict(row) if row else None + + +async def ack_done(pool, job_id: int) -> None: + await pool.execute( + """ + UPDATE jobs + SET status='done', + locked_by=NULL, + locked_at=NULL, + last_error=NULL, + updated_at=NOW() + WHERE id=$1 + """, + job_id, + ) + + +async def ack_retry(pool, job: dict, error: str, delay_sec: int = 60) -> None: + attempt = int(job.get("attempts") or 0) + 1 + max_attempts = int(job.get("max_attempts") or 5) + status = "dead" if attempt >= max_attempts else "retry" + await pool.execute( + """ + UPDATE jobs + SET status=$2, + attempts=$3, + next_run_at=CASE WHEN $2='retry' THEN NOW() + ($4 * INTERVAL '1 second') ELSE next_run_at END, + locked_by=NULL, + locked_at=NULL, + last_error=$5, + updated_at=NOW() + WHERE id=$1 + """, + int(job["id"]), + status, + attempt, + int(delay_sec), + error[:1000], + ) + + +async def recover_stale_jobs(pool, job_type: str, stale_minutes: int = 20) -> int: + result = await pool.execute( + """ + UPDATE jobs + SET status='retry', + locked_by=NULL, + locked_at=NULL, + next_run_at=NOW(), + last_error=COALESCE(last_error, 'recovered stale lock'), + updated_at=NOW() + WHERE type=$1 + AND status='in_progress' + AND locked_at < NOW() - ($2 * INTERVAL '1 minute') + """, + job_type, + stale_minutes, + ) + return int(str(result).split()[-1]) diff --git a/src/vk_parser_app/main.py b/src/vk_parser_app/main.py new file mode 100644 index 0000000..8bb4ef6 --- /dev/null +++ b/src/vk_parser_app/main.py @@ -0,0 +1,3 @@ +from .admin import app + +__all__ = ["app"] diff --git a/src/vk_parser_app/security.py b/src/vk_parser_app/security.py new file mode 100644 index 0000000..d5b875a --- /dev/null +++ b/src/vk_parser_app/security.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import base64 +import hashlib +import secrets + + +def hash_password(password: str, iterations: int = 390_000) -> str: + salt = secrets.token_bytes(16) + digest = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, iterations) + return "pbkdf2_sha256${}${}${}".format( + iterations, + base64.urlsafe_b64encode(salt).decode("ascii"), + base64.urlsafe_b64encode(digest).decode("ascii"), + ) + + +def verify_password(password: str, encoded: str) -> bool: + try: + algo, iterations_raw, salt_raw, digest_raw = encoded.split("$", 3) + if algo != "pbkdf2_sha256": + return False + iterations = int(iterations_raw) + salt = base64.urlsafe_b64decode(salt_raw.encode("ascii")) + expected = base64.urlsafe_b64decode(digest_raw.encode("ascii")) + except Exception: + return False + actual = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, iterations) + return secrets.compare_digest(actual, expected) + + +def token_hash(token: str, secret: str) -> str: + return hashlib.sha256((secret + ":" + token).encode("utf-8")).hexdigest() + + +def new_token() -> str: + return secrets.token_urlsafe(32) diff --git a/src/vk_parser_app/templates/base.html b/src/vk_parser_app/templates/base.html new file mode 100644 index 0000000..2d2646c --- /dev/null +++ b/src/vk_parser_app/templates/base.html @@ -0,0 +1,340 @@ + + + + + + + {{ title or site_title or "Редакторская" }} + + + + + + + + + + + + + + + + + + + + + + + + + + {% if user %} + + + + +
+
+ {{ app_title or "Редакторская" }} +
+
+ + +
+
+ + + + {% endif %} + + +
+
+ {% block body %}{% endblock %} +
+
+ + + + diff --git a/src/vk_parser_app/templates/editor.html b/src/vk_parser_app/templates/editor.html new file mode 100644 index 0000000..bdb2d23 --- /dev/null +++ b/src/vk_parser_app/templates/editor.html @@ -0,0 +1,33 @@ +{% extends "base.html" %} +{% block body %} +
+
+

+ + Редакторская +

+
Проверка, правка и публикация сгенерированных постов.
+
+ + +
+
+ {% for st in status_options %} + + {{ st.label }} + + {{ counts.get(st.value, 0) }} + + + {% endfor %} +
+
+
+ +
+ {% include "editor_content.html" %} +
+ + + +{% endblock %} diff --git a/src/vk_parser_app/templates/editor_content.html b/src/vk_parser_app/templates/editor_content.html new file mode 100644 index 0000000..2320507 --- /dev/null +++ b/src/vk_parser_app/templates/editor_content.html @@ -0,0 +1,128 @@ +
+ + + + +
+
+
+ + Найдено: {{ pagination.total }} +
+
+ На странице: + +
+
+ +
+ {% for p in posts %} + {% include "editor_post.html" %} + {% else %} +
+ +

Ничего не найдено

+

В этой очереди пока нет постов, подходящих под фильтры.

+
+ {% endfor %} +
+ + + {% if pagination.pages > 1 %} +
+
+ + + + + {{ pagination.page }} из {{ pagination.pages }} + + + + +
+
+ {% endif %} +
+
diff --git a/src/vk_parser_app/templates/editor_post.html b/src/vk_parser_app/templates/editor_post.html new file mode 100644 index 0000000..cc04e0f --- /dev/null +++ b/src/vk_parser_app/templates/editor_post.html @@ -0,0 +1,213 @@ +
+ + +
+ +
+
+ #{{ p.id }} + {% include "editor_post_status.html" %} +
+ +
+
+ +
+
+ {{ p.source_name }} +
{{ p.posted_at_fmt or p.created_at_fmt }}
+
+
+ +
+
+ AI Оценка: + {{ p.qualification_score or "?" }}/10 +
+
+ + {% if p.publication_platforms %} +
+
+
Куда публикуем:
+ {% for pub in p.publication_platforms %} + {% if pub.url %} + + {{ pub.label }} + + {% elif pub.status == 'published' %} + + {{ pub.label }} + + {% else %} + + {{ pub.label }} + + {% endif %} + {% endfor %} +
+
+ {% endif %} +
+ + +
+ + + {% if p.media_items %} + {% set photo_urls = [] %} + {% for m in p.media_items %} + {% if m.type == 'photo' and m.url %} + {% set _ = photo_urls.append(m.url) %} + {% endif %} + {% endfor %} + +
+ +
+ {% for m in p.media_items %} + {% if m.type == "photo" and m.url %} + {% set photo_idx = photo_urls.index(m.url) %} +
+ + + + +
+ {% elif m.url %} +
+ + + + + + +
+ {% endif %} + {% endfor %} +
+ + + +
+ {% endif %} + + +
+ + + ОРИГИНАЛЬНЫЙ ТЕКСТ + +
+
{{ p.raw_text }}
+
+
+ + + + + + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + + AI ЗАМЕТКИ / РЕЗОЛЮЦИЯ + +
+
+
+
AI заметки / резолюция
+
{{ p.rewrite_notes or p.qualification_reason or "—" }}
+
+
+
AI рерайт
+
{{ p.rewritten_text or "—" }}
+
+
+ + +
+
+
+
+
+ + + {% include "editor_post_actions.html" %} +
+
+
diff --git a/src/vk_parser_app/templates/editor_post_actions.html b/src/vk_parser_app/templates/editor_post_actions.html new file mode 100644 index 0000000..dda31bb --- /dev/null +++ b/src/vk_parser_app/templates/editor_post_actions.html @@ -0,0 +1,36 @@ +{% if p.editorial_status != 'published' %} +
+
+ {% if action_message %} +
+ + {{ action_message }} +
+ {% endif %} + {% if p.editorial_status in ['rejected', 'accepted', 'publish_failed'] %} + + {% else %} + + {% endif %} +
+ + {% if p.editorial_status not in ['rejected', 'accepted', 'publish_failed'] %} +
+ + +
+ {% endif %} +
+{% endif %} diff --git a/src/vk_parser_app/templates/editor_post_status.html b/src/vk_parser_app/templates/editor_post_status.html new file mode 100644 index 0000000..4db7ee6 --- /dev/null +++ b/src/vk_parser_app/templates/editor_post_status.html @@ -0,0 +1,11 @@ +
+ {% if p.editorial_status == 'accepted' or p.editorial_status == 'published' %} + Принят + {% elif p.editorial_status == 'rejected' or p.editorial_status == 'publish_failed' %} + Отклонен + {% elif p.editorial_status == 'regenerating' %} + В работе + {% else %} + Ожидает + {% endif %} +
diff --git a/src/vk_parser_app/templates/editor_post_update.html b/src/vk_parser_app/templates/editor_post_update.html new file mode 100644 index 0000000..c56bd8e --- /dev/null +++ b/src/vk_parser_app/templates/editor_post_update.html @@ -0,0 +1,3 @@ +{% include "editor_post_actions.html" %} +{% set status_oob = true %} +{% include "editor_post_status.html" %} diff --git a/src/vk_parser_app/templates/login.html b/src/vk_parser_app/templates/login.html new file mode 100644 index 0000000..25ad63f --- /dev/null +++ b/src/vk_parser_app/templates/login.html @@ -0,0 +1,30 @@ +{% extends "base.html" %} +{% block body %} +
+
+
+

Вход в систему

+ + {% if error %} +
+ {{ error }} +
+ {% endif %} + +
+
+ + +
+
+ + +
+
+ +
+
+
+
+
+{% endblock %} diff --git a/src/vk_parser_app/templates/logs.html b/src/vk_parser_app/templates/logs.html new file mode 100644 index 0000000..71dee9d --- /dev/null +++ b/src/vk_parser_app/templates/logs.html @@ -0,0 +1,160 @@ +{% extends "base.html" %} +{% block body %} +
+

+ + Логи +

+
Журнал действий и запросов админки. События старше 30 дней удаляются автоматически.
+
+ +
+
+
+
+ +
+ + +
+
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ +
+ + +
+
+ +
+ + Сбросить +
+
+
+ +
+
+ + Всего записей: {{ pagination.total }} +
+
+ На странице: +
+ {% for item in preserved_query %} + {% if item.key != "per_page" %} + + {% endif %} + {% endfor %} + +
+
+
+ +
+ + + + + + + + + + + + {% for item in logs %} + + + + + + + + {% else %} + + + + {% endfor %} + +
ВремяПользовательДействиеСущностьДетали
{{ item.created_at_fmt }}{{ item.actor_label }}{{ item.action }} + {{ item.entity_type }} + {% if item.entity_id %} + #{{ item.entity_id }} + {% endif %} + + {% if item.details_pretty %} +
+ + + JSON Payload + +
+
+ 
{{ item.details_pretty }}
+
+
+
+ {% else %} + — + {% endif %} +
+
+ +

Записей по заданным фильтрам не найдено

+
+
+
+ + {% if pagination.pages > 1 %} +
+
+ + + + + {{ pagination.page }} из {{ pagination.pages }} + + + + +
+
+ {% endif %} +
+{% endblock %} diff --git a/src/vk_parser_app/templates/pagination.html b/src/vk_parser_app/templates/pagination.html new file mode 100644 index 0000000..9af310f --- /dev/null +++ b/src/vk_parser_app/templates/pagination.html @@ -0,0 +1,37 @@ +{% if pagination and pagination.pages > 1 %} + +{% else %} + {% if pagination %}{% endif %} +{% endif %} diff --git a/src/vk_parser_app/templates/prompt_test.html b/src/vk_parser_app/templates/prompt_test.html new file mode 100644 index 0000000..f7d0e21 --- /dev/null +++ b/src/vk_parser_app/templates/prompt_test.html @@ -0,0 +1,163 @@ +{% extends "base.html" %} +{% block body %} +
+
+

Тест промпта райтера

+
Песочница для свободной части ai_writer_prompt. Посты и статусы в БД не меняются.
+
+ Настройки AI +
+ +
+ +
Как пользоваться
+
+
+
Что тестируем
+

Отправляется только текст из поля ниже как system prompt. ai_writer_contract специально не добавляется, чтобы проверять стиль и структуру.

+
+
+
Что уйдёт в модель
+

Один выбранный raw-пост в формате: categories, producer, ссылка, оценка, медиа и обрезанный исходный текст.

+
+
+
Что безопасно менять
+

Голос канала, структуру текста, длину, правила про эмоджи, запрет выдумок, примеры формулировок.

+
+
+
Что не происходит
+

Рерайт не сохраняется, batch не создаётся. Тест только тратит токены текущей модели райтера.

+
+
+
+ +
+ + +
+
+
+ +
+
+ + +
+
+ + +
+
+ +
+ Модель: + {{ provider }} + {{ model or "не выбрана" }} +
+
+
+ + {% if selected %} +
+
+
+ #{{ selected.id }} · {{ selected.source_name }} + Оригинал +
+
+ {{ selected.posted_at_fmt or selected.created_at_fmt }} · {{ selected.media_count }} медиа · оценка {{ selected.qualification_score or "—" }} +
+ 
{{ selected.raw_text }}
+
+
+ {% endif %} + +
+
+
+ + +
+
+ +
+
+
+
+ + +
+ + +{% endblock %} diff --git a/src/vk_parser_app/templates/raw_post_detail.html b/src/vk_parser_app/templates/raw_post_detail.html new file mode 100644 index 0000000..6dd9ae8 --- /dev/null +++ b/src/vk_parser_app/templates/raw_post_detail.html @@ -0,0 +1,194 @@ +{% extends "base.html" %} +{% block body %} +
+
+ ← Raw-посты +

Raw #{{ post.id }}

+
+ {{ post.source_name }} + · {{ post.source_platform }} + · #{{ post.source_tag or "—" }} + {% if post.posted_at_fmt %}· пост VK: {{ post.posted_at_fmt }}{% endif %} +
+
+ {{ post.stage.label }} +
+ +
+
+
+ {% for m in post.media_items %} + {% if m.type == "photo" and m.url %} + + {% elif m.url %} + + ▶ + + {% endif %} + {% endfor %} +
+ {% if post.media_count %}
{{ post.media_count }} медиа
{% endif %} +
{{ post.raw_text }}
+
Загружен парсером: {{ post.created_at_fmt }}
+
+ + +
+ +
+
+
+

Рерайт

+
Готовый текст для будущей редакторской проверки.
+
+ + {{ post.rewrite_status or "pending" }} + +
+ {% if post.rewritten_text %} +
{{ post.rewritten_text }}
+ {% else %} +
Рерайт ещё не готов.
+ {% endif %} +
+
Категория
{{ post.rewrite_category or "—" }}
+
Хэштеги
+
+ {% if post.rewrite_category_tag or post.rewrite_source_tag %} + #{{ post.rewrite_category_tag or "—" }} #{{ post.rewrite_source_tag or "—" }} + {% else %} + — + {% endif %} +
+
Заметка
{{ post.rewrite_notes or "—" }}
+
Модель
{{ post.rewrite_model or post.writer_batch_model or "—" }}
+
Готов
{{ post.rewritten_at_fmt or "—" }}
+
Batch
{% if post.rewrite_batch_id %}#{{ post.rewrite_batch_id }}{% else %}—{% endif %}
+
Токены
{{ post.writer_total_tokens or "—" }}
+
Стоимость
{% if post.writer_estimated_cost_usd %}${{ post.writer_estimated_cost_usd }}{% else %}—{% endif %}
+
+
+ +{% if post.qualification_batch_id %} +
+ Технический ответ AI +
+
Provider
{{ post.qualification_provider or "—" }}
+
Batch status
{{ post.batch_status or "—" }}
+
Создан
{{ post.batch_created_at_fmt or "—" }}
+
Завершён
{{ post.batch_completed_at_fmt or "—" }}
+
Постов в batch
{{ post.batch_posts_count or "—" }}
+
Accepted / rejected / maybe
+
{{ post.batch_accepted_count or 0 }} / {{ post.batch_rejected_count or 0 }} / {{ post.batch_maybe_count or 0 }}
+
Токены
+
{{ post.batch_prompt_tokens or 0 }} / {{ post.batch_completion_tokens or 0 }} / {{ post.batch_total_tokens or 0 }}
+
Стоимость
{% if post.batch_estimated_cost_usd %}${{ post.batch_estimated_cost_usd }}{% else %}—{% endif %}
+
Ошибка
{{ post.batch_error or "—" }}
+
+ {% if post.batch_prompt_text %} +
+ Использованный prompt квалификатора + 
{{ post.batch_prompt_text }}
+
+ {% endif %} + 
{{ post.batch_response_pretty or "{}" }}
+
+{% endif %} + +{% if post.rewrite_batch_id %} +
+ Технический ответ AI-райтера +
+
Provider
{{ post.rewrite_provider or "—" }}
+
Batch status
{{ post.writer_batch_status or "—" }}
+
Создан
{{ post.writer_batch_created_at_fmt or "—" }}
+
Завершён
{{ post.writer_batch_completed_at_fmt or "—" }}
+
Постов в batch
{{ post.writer_batch_posts_count or "—" }}
+
Ready / failed
{{ post.writer_batch_ready_count or 0 }} / {{ post.writer_batch_failed_count or 0 }}
+
Токены
+
{{ post.writer_prompt_tokens or 0 }} / {{ post.writer_completion_tokens or 0 }} / {{ post.writer_total_tokens or 0 }}
+
Стоимость
{% if post.writer_estimated_cost_usd %}${{ post.writer_estimated_cost_usd }}{% else %}—{% endif %}
+
Ошибка
{{ post.writer_batch_error or "—" }}
+
+ {% if post.writer_batch_prompt_text %} +
+ Использованный prompt райтера + 
{{ post.writer_batch_prompt_text }}
+
+ {% endif %} + 
{{ post.writer_batch_response_pretty or "{}" }}
+
+{% endif %} + + + +{% endblock %} diff --git a/src/vk_parser_app/templates/raw_posts.html b/src/vk_parser_app/templates/raw_posts.html new file mode 100644 index 0000000..b015297 --- /dev/null +++ b/src/vk_parser_app/templates/raw_posts.html @@ -0,0 +1,17 @@ +{% extends "base.html" %} + +{% block body %} +
+
+

+ + Сырые посты +

+
Этапы обработки исходных постов и AI-квалификация.
+
+
+ +
+ {% include "raw_posts_content.html" %} +
+{% endblock %} diff --git a/src/vk_parser_app/templates/raw_posts_content.html b/src/vk_parser_app/templates/raw_posts_content.html new file mode 100644 index 0000000..c4ce3eb --- /dev/null +++ b/src/vk_parser_app/templates/raw_posts_content.html @@ -0,0 +1,223 @@ +
+ + + + +
+
+
+ + Всего: {{ pagination.total }} +
+
+ На странице: + +
+
+ +
+
+ {% for p in posts %} + +
+
+
#{{ p.id }}
+
{{ p.source_name }}
+
+ {{ p.status }} + {% if p.qualification_status %} + Квалиф: {{ p.qualification_status }} + {% endif %} + {% if p.rewrite_status %} + Рерайт: {{ p.rewrite_status }} + {% endif %} +
+
+
+ {% if p.qualification_score is not none %} +
{{ p.qualification_score }}
+ {% else %} + — + {% endif %} +
+
+
{{ p.raw_text }}
+
{{ p.posted_at_fmt or p.created_at_fmt }}
+ + {% else %} +
+
+ +

Нет данных по заданным фильтрам

+
+
+ {% endfor %} +
+ +
+ + + {% if pagination.pages > 1 %} +
+
+ + + + + {{ pagination.page }} из {{ pagination.pages }} + + + + +
+
+ {% endif %} +
+
diff --git a/src/vk_parser_app/templates/source_form.html b/src/vk_parser_app/templates/source_form.html new file mode 100644 index 0000000..3010cdd --- /dev/null +++ b/src/vk_parser_app/templates/source_form.html @@ -0,0 +1,57 @@ +{% extends "base.html" %} +{% block body %} +
+
+

{{ title }}

+
+ Назад +
+ +
+
+
+ + +
+
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+
+ +
+ +
+ +
+ +
+
+
+
+{% endblock %} diff --git a/src/vk_parser_app/templates/source_row.html b/src/vk_parser_app/templates/source_row.html new file mode 100644 index 0000000..16597dc --- /dev/null +++ b/src/vk_parser_app/templates/source_row.html @@ -0,0 +1,63 @@ + + {{ s.id }} + {{ s.platform }} + +
+ {{ s.name }} +
+
#{{ s.id }}
+
{{ s.external_id or "" }}
+
#{{ s.tag or "—" }}
+ {{ s.url }} + + #{{ s.tag or "—" }} + + {{ s.url }} + + +
+
+ {% if s.active %} + Работает + {% else %} + На паузе + {% endif %} +
+
+ Парсер: + {{ s.status }} +
+ {% if s.status_msg %} + {{ s.status_msg }} + {% endif %} +
+ + +
{{ s.posts_count }}
+
+{{ s.posts_24h }}
+ + {{ s.last_parsed_at_fmt or "Никогда" }} + +
+ + + +
+ + +
+
+ + +
+
+ + diff --git a/src/vk_parser_app/templates/sources.html b/src/vk_parser_app/templates/sources.html new file mode 100644 index 0000000..e4b38bf --- /dev/null +++ b/src/vk_parser_app/templates/sources.html @@ -0,0 +1,146 @@ +{% extends "base.html" %} +{% block body %} +
+

+ + Источники +

+
VK-источники для парсинга. Название идёт в prompt, тэг — в будущие хэштеги.
+
+ +
+ +
+
+ +
+ Добавить источники +
+ + + +
+
+
+ + + + + + +
+ + +
+ + + + +
+
+ + {% if source_preview %} +
+

Результаты проверки

+ +
+ + + + + + + + + + + + + {% for item in source_preview %} + + + + + + + + + {% endfor %} + +
# Стр.НазваниеТэгСсылкаVK IDСтатус
{{ item.line_no }}{{ item.name or "—" }}#{{ item.tag or "—" }}{{ item.url }} + {{ item.external_id }}
+ {{ item.external_owner_id or "" }} + 		+ {% if item.ok %} + Готов + {% else %} + Ошибка + {% endif %} +
+
+ +
+ + + +
+ + Строки с ошибками и дубликаты будут автоматически пропущены. +
+
+
+ {% endif %} +
+
+ +
+
+
+
+ +
+ + +
+
+ +
+ + +
+ +
+ + +
+ +
+ +
+
+
+ +
+ {% include "sources_content.html" %} +
+
+{% endblock %} diff --git a/src/vk_parser_app/templates/sources_content.html b/src/vk_parser_app/templates/sources_content.html new file mode 100644 index 0000000..fcfe800 --- /dev/null +++ b/src/vk_parser_app/templates/sources_content.html @@ -0,0 +1,48 @@ +
+ + + + + + + + + + + + + + + + {% for s in sources %} + {% include "source_row.html" %} + {% else %} + + + + {% endfor %} + +
НазваниеСтатусДействия
+
+ +

Нет источников, удовлетворяющих фильтрам.

+
+
+
+ + +{% if pagination.pages > 1 %} +
+
+ + + + + {{ pagination.page }} из {{ pagination.pages }} + + + + +
+
+{% endif %} diff --git a/src/vk_parser_app/templates/table_tools.html b/src/vk_parser_app/templates/table_tools.html new file mode 100644 index 0000000..6f7ed29 --- /dev/null +++ b/src/vk_parser_app/templates/table_tools.html @@ -0,0 +1,49 @@ +
+ +
Расширенные фильтры и сортировка
+
+
+
Фильтры
+
+ {% for item in table_filters %} +
+ + + +
+ {% endfor %} +
+
Пустые строки игнорируются. Для дат используй YYYY-MM-DD, для булевых значений — true/false.
+
+ +
+
Сортировка
+
+ {% for item in table_sorts %} +
+ + +
+ {% endfor %} +
+
Сортировки применяются сверху вниз. Заменяют базовую сортировку страницы.
+
+
+
diff --git a/src/vk_parser_app/templates/users.html b/src/vk_parser_app/templates/users.html new file mode 100644 index 0000000..061956b --- /dev/null +++ b/src/vk_parser_app/templates/users.html @@ -0,0 +1,126 @@ +{% extends "base.html" %} +{% block body %} +
+

+ + Пользователи +

+
Управление доступом к админ-панели.
+
+ +
+
+ + + + + + + + + + + + + {% for u in users %} + + + + + + + + + {% endfor %} + +
IDЛогинРольСозданСтатусДействия
{{ u.id }}{{ u.login }} + {{ u.role }} + {{ u.created_at_fmt }} + {% if u.is_active %} + + Активен + + {% else %} + + Заблокирован + + {% endif %} + + {% if user.role == "admin" and user.id != u.id %} +
+
+ + +
+
+ + +
+
+ {% endif %} +
+
+ {% if pagination.pages > 1 %} +
+
+ + + + + {{ pagination.page }} из {{ pagination.pages }} + + + + +
+
+ {% endif %} +
+ +{% if user.role == "admin" %} +
+ +
+
+ +
+ Добавить пользователя +
+ + + +
+
+ + +
+ + +
+ +
+ + +
+ +
+ + +
+ + +
+
+
+{% endif %} +{% endblock %} diff --git a/src/vk_parser_app/templates/vk_oauth_callback.html b/src/vk_parser_app/templates/vk_oauth_callback.html new file mode 100644 index 0000000..8d96bee --- /dev/null +++ b/src/vk_parser_app/templates/vk_oauth_callback.html @@ -0,0 +1,45 @@ +{% extends "base.html" %} +{% block body %} +
+

VK OAuth

+ {% if result %} + {% if result.ok %} +

Токен VK-постера сохранён

+
+
Сообщество
{{ result.owner_id }}
+
User ID
{{ result.user_id or "—" }}
+
Живёт, сек
{{ result.expires_in or "—" }}
+
Истекает
{{ result.expires_at or "—" }}
+
Refresh token
{% if result.has_refresh_token %}OK{% else %}Не получен{% endif %}
+
wall.get
{% if result.wall_get_ok %}OK{% else %}Ошибка: {{ result.wall_get_error }}{% endif %}
+
Фото upload
+
+ {% if result.photo_upload_server_ok %} + OK + {% else %} + Недоступно: {{ result.photo_upload_server_error }} +
VK выдаёт право photos для upload API отдельно; постер всё равно опубликует текст и существующие VK-вложения.
+ {% endif %} +
+
+

Вернуться к воркерам

+ {% else %} +

VK OAuth не завершился

+

{{ result.error_description or result.error or "Неизвестная ошибка" }}

+

Вернуться к воркерам Повторить

+ {% endif %} + {% elif code %} +

VK вернул code. Новый обработчик должен обменивать его автоматически через /vk-oauth/callback.

+ + + {% if state and expected_state and state != expected_state %} +

state не совпал, такой code лучше не использовать.

+ {% endif %} + {% elif error %} +

{{ error }}

+

{{ error_description }}

+ {% else %} +

VK вернул callback без данных.

+ {% endif %} +
+{% endblock %} diff --git a/src/vk_parser_app/templates/workers.html b/src/vk_parser_app/templates/workers.html new file mode 100644 index 0000000..838a33f --- /dev/null +++ b/src/vk_parser_app/templates/workers.html @@ -0,0 +1,396 @@ +{% extends "base.html" %} +{% block body %} +
+

+ + Система и Воркеры +

+
Управление фоновыми процессами, категориями, расписанием и настройками AI.
+
+ + +
+
+ + + + + + + + + + + + + {% for w in workers %} + + + + + + + + + {% endfor %} + +
ВоркерСостояниеHeartbeatСтатусТекущая задачаДействия
{{ w.name }} + {% if w.enabled %} + + Включен + + {% else %} + + Выключен + + {% endif %} + {{ w.heartbeat_at or "—" }} +
{{ w.status or "—" }}
+ {% if w.meta_json and w.meta_json.error %} +
{{ w.meta_json.error }}
+ {% endif %} + 		{{ w.current_job_id or "—" }} +
+ + +
+
+
+
+ +
+ +
+ +
+ Категории публикаций +
+ + +
+
+ Название описывает категорию для AI, тэг используется в соцсетях. +
+ +
+
Добавить категорию
+ +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+ + +
+
+ +
+ + + + + + + + + + + + {% for c in categories %} + + + + + + + + {% endfor %} + +
Название / ТэгСайт / SlugIDСтатус
+
+ +
+
+ + +
+ 		+
+ + +
+ 		{{ c.sort_order }} +
+ {% if c.is_active %}Активна{% else %}Выкл{% endif %} + +
+ 		+
+ +
+ + +
+
+ + +
+
+
+
+
+
+ +
+ +
+ +
+ Расписание TG-постера +
+ + +
+
+ Время по Екб. Нужны: воркер tg-poster и настройка tg_poster_enabled. +
+ + +
+ {% for row in tg_schedule %} + {% if row.enabled %} +
+ {{ row.time[:2] }}:00 + {{ row.count }} шт. +
+ + +
+
+ {% endif %} + {% else %} +
Расписание пусто
+ {% endfor %} +
+ +
+ + +
+ + +
+
+ + +
+ +
+
+
+ + +
+ +
+ Расписание VK-постера +
+ + +
+
+ Время по Екб. Нужны: воркер vk-poster и vk_poster_enabled. +
+ + + +
+ {% for row in vk_schedule %} + {% if row.enabled %} +
+ {{ row.time[:2] }}:00 + {{ row.count }} шт. +
+ + +
+
+ {% endif %} + {% else %} +
Расписание пусто
+ {% endfor %} +
+ +
+ + +
+ + +
+
+ + +
+ +
+
+
+
+
+ +
+
+

+ Глобальные настройки +

+
+
+ +
+ {% for category, rows in settings|groupby("category") %} +
+ +
{{ category_titles.get(category, category) }}
+ + +
+ +
+ {% for s in rows %} +
+ +
+
{{ s.key }}
+
{{ s.title }}
+
{{ s.description }}
+
+ + +
+
+ + + +
+
+ {% if s.value_type == "bool" %} + + + {% elif s.key in ["ai_qualifier_provider", "ai_writer_provider"] %} + + + {% elif s.key == "site_poster_provider" %} + + + {% elif s.key == "ai_qualifier_model" %} + + + {% elif s.key == "ai_writer_model" %} + + + {% elif s.value_type == "secret" %} + + + {% elif s.value_type == "text" %} +
+ {% if prompt_hints and s.key in prompt_hints %} +
+ {{ prompt_hints[s.key].title }} +
{{ prompt_hints[s.key].body }}
+
+
Как безопасно менять:
{{ prompt_hints[s.key].safe }}
+
На вход:
{{ prompt_hints[s.key].input }}
+
+
+ {% endif %} + +
+ + {% elif s.value_type == "json" %} + + + {% else %} + + {% endif %} +
+ + +
+
+
+
+ {% endfor %} +
+ +
+
+ {% endfor %} +
+{% endblock %} diff --git a/src/vk_parser_app/text_utils.py b/src/vk_parser_app/text_utils.py new file mode 100644 index 0000000..a06128c --- /dev/null +++ b/src/vk_parser_app/text_utils.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +import re + + +def normalize_hash_tag(value: str, fallback: str = "source") -> str: + tag = (value or "").strip().lstrip("#").lower() + tag = re.sub(r"\s+", "_", tag) + tag = re.sub(r"[^\wа-яё_]+", "_", tag, flags=re.IGNORECASE) + tag = re.sub(r"_+", "_", tag).strip("_") + return tag or fallback + + +def strip_trailing_hashtag_line(text: str) -> str: + lines = str(text or "").rstrip().splitlines() + while lines and not lines[-1].strip(): + lines.pop() + if not lines: + return "" + parts = lines[-1].split() + if parts and all(part.startswith("#") and normalize_hash_tag(part, "") for part in parts): + lines.pop() + return "\n".join(lines).rstrip() + + +def publication_hashtags(category_tag: str, source_tag: str) -> str: + tags = [ + normalize_hash_tag(category_tag, "category"), + normalize_hash_tag(source_tag, "source"), + ] + return " ".join(f"#{tag}" for tag in tags if tag) + + +import html + + +def build_publication_text( + text: str, + category_tag: str, + source_tag: str, + format_title: bool = False, + parse_mode: str | None = None, +) -> str: + base = strip_trailing_hashtag_line(text) + lines = base.splitlines() + sep_pattern = re.compile(r"^\s*[━—─\-=\*\#_]{2,}\s*$") + + cleaned_lines = [] + for line in lines: + if sep_pattern.match(line): + cleaned_lines.append("") + else: + cleaned_lines.append(line) + + title_idx = None + for idx, line in enumerate(cleaned_lines): + if line.strip(): + title_idx = idx + break + + if title_idx is None: + return publication_hashtags(category_tag, source_tag) + + formatted_lines = [] + for idx, line in enumerate(cleaned_lines): + if idx == title_idx and format_title: + if parse_mode == "html": + escaped_title = html.escape(line.strip()) + formatted_lines.append(f"{escaped_title}") + elif parse_mode == "markdown": + formatted_lines.append(f"**{line.strip()}**") + else: + formatted_lines.append(line.strip()) + + # Ensure a newline break (blank line) after the bold title + if idx + 1 < len(cleaned_lines) and cleaned_lines[idx + 1].strip() != "": + formatted_lines.append("") + else: + if parse_mode == "html": + formatted_lines.append(html.escape(line)) + else: + formatted_lines.append(line) + + raw_body = "\n".join(formatted_lines) + # Collapse multiple consecutive blank lines to at most two newlines (\n\n) + normalized_body = re.sub(r"\n{3,}", "\n\n", raw_body).strip() + hashtags = publication_hashtags(category_tag, source_tag) + if hashtags: + return f"{normalized_body}\n\n{hashtags}" if normalized_body else hashtags + return normalized_body + + +def parse_categories(value: object) -> list[str]: + if isinstance(value, list): + raw_items = [str(item) for item in value] + else: + text = str(value or "") + raw_items = re.split(r"[\n,|]+", text) + categories: list[str] = [] + seen: set[str] = set() + for item in raw_items: + category = item.strip().strip("#") + key = category.lower() + if category and key not in seen: + categories.append(category) + seen.add(key) + return categories diff --git a/src/vk_parser_app/vk_api.py b/src/vk_parser_app/vk_api.py new file mode 100644 index 0000000..8718add --- /dev/null +++ b/src/vk_parser_app/vk_api.py @@ -0,0 +1,304 @@ +from __future__ import annotations + +import asyncio +import json +import re +import time +from dataclasses import dataclass +from typing import Any + +import aiohttp +from loguru import logger + +from .config import settings + + +class VKAPIError(RuntimeError): + def __init__(self, code: int | None, message: str) -> None: + self.code = code + super().__init__(f"VK API error {code}: {message}") + + +class VKRateLimiter: + def __init__(self, rps: int = 3) -> None: + self.rps = max(1, int(rps)) + self.interval = 1.0 / self.rps + self._last = 0.0 + self._lock = asyncio.Lock() + + async def acquire(self) -> None: + async with self._lock: + now = time.monotonic() + wait = self.interval - (now - self._last) + if wait > 0: + await asyncio.sleep(wait) + self._last = time.monotonic() + + +class VKAPIClient: + base_url = "https://api.vk.com/method" + + def __init__( + self, + token: str | None = None, + version: str | None = None, + rps: int = 3, + timeout_total_sec: int = 60, + timeout_connect_sec: int = 10, + rate_limit_sleep_sec: float = 1.0, + retry_attempts: int = 3, + retry_min_delay_sec: float = 2.0, + retry_max_delay_sec: float = 10.0, + ) -> None: + self.token = token or settings.vk_access_token + self.version = version or settings.vk_api_version + self.limiter = VKRateLimiter(rps) + self.timeout_total_sec = max(1, int(timeout_total_sec)) + self.timeout_connect_sec = max(1, int(timeout_connect_sec)) + self.rate_limit_sleep_sec = max(0.1, float(rate_limit_sleep_sec)) + self.retry_attempts = max(1, int(retry_attempts)) + self.retry_min_delay_sec = max(0.1, float(retry_min_delay_sec)) + self.retry_max_delay_sec = max(self.retry_min_delay_sec, float(retry_max_delay_sec)) + self.session: aiohttp.ClientSession | None = None + + async def __aenter__(self) -> "VKAPIClient": + self.session = aiohttp.ClientSession( + timeout=aiohttp.ClientTimeout(total=self.timeout_total_sec, connect=self.timeout_connect_sec) + ) + return self + + async def __aexit__(self, *args) -> None: + if self.session: + await self.session.close() + + async def call(self, method: str, **params: Any) -> dict | list: + if not self.session: + raise RuntimeError("VKAPIClient is not initialized") + if not self.token: + raise RuntimeError("VK_ACCESS_TOKEN is empty") + + payload = dict(params) + payload["access_token"] = self.token + payload["v"] = self.version + + for attempt in range(1, self.retry_attempts + 1): + try: + await self.limiter.acquire() + async with self.session.post(f"{self.base_url}/{method}", data=payload) as resp: + resp.raise_for_status() + data = await resp.json(content_type=None) + if "error" not in data: + return data.get("response", {}) + + err = data["error"] + code = err.get("error_code") + message = err.get("error_msg", "unknown") + if code == 6 and attempt < self.retry_attempts: + logger.warning("VK rate limit hit, sleeping {}s", self.rate_limit_sleep_sec) + await asyncio.sleep(self.rate_limit_sleep_sec) + continue + raise VKAPIError(code, message) + except VKAPIError: + raise + except Exception: + if attempt >= self.retry_attempts: + raise + delay = min(self.retry_min_delay_sec * (2 ** (attempt - 1)), self.retry_max_delay_sec) + await asyncio.sleep(delay) + + raise VKAPIError(None, "retry exhausted") + + async def resolve_group(self, input_value: str) -> tuple[str, int, str]: + external_id = normalize_vk_source(input_value) + if external_id.lstrip("-").isdigit(): + group_id = abs(int(external_id)) + response = await self.call("groups.getById", group_id=str(group_id)) + else: + response = await self.call("groups.getById", group_id=external_id) + items = response if isinstance(response, list) else response.get("groups", []) + if not items: + raise VKAPIError(None, f"cannot resolve VK group: {input_value}") + group = items[0] + group_id = int(group["id"]) + screen_name = str(group.get("screen_name") or external_id) + name = str(group.get("name") or screen_name) + return screen_name, -group_id, name + + async def get_wall_posts(self, owner_id: int, count: int, offset: int = 0) -> dict: + response = await self.call("wall.get", owner_id=owner_id, count=count, offset=offset, filter="owner") + if not isinstance(response, dict): + raise VKAPIError(None, "wall.get returned non-object response") + return response + + async def get_wall_upload_server(self, group_id: int) -> str: + response = await self.call("photos.getWallUploadServer", group_id=abs(int(group_id))) + if not isinstance(response, dict) or not response.get("upload_url"): + raise VKAPIError(None, "photos.getWallUploadServer returned no upload_url") + return str(response["upload_url"]) + + async def upload_wall_photo_bytes(self, group_id: int, data: bytes, filename: str = "photo.jpg") -> dict: + if not self.session: + raise RuntimeError("VKAPIClient is not initialized") + upload_url = await self.get_wall_upload_server(group_id) + form = aiohttp.FormData() + form.add_field("photo", data, filename=filename, content_type="image/jpeg") + async with self.session.post(upload_url, data=form) as resp: + uploaded = await resp.json(content_type=None) + saved = await self.call( + "photos.saveWallPhoto", + group_id=abs(int(group_id)), + photo=uploaded.get("photo"), + server=uploaded.get("server"), + hash=uploaded.get("hash"), + ) + if not isinstance(saved, list) or not saved: + raise VKAPIError(None, f"photos.saveWallPhoto returned invalid response: {json.dumps(saved)[:300]}") + return saved[0] + + async def upload_wall_photo_url(self, group_id: int, url: str) -> dict: + if not self.session: + raise RuntimeError("VKAPIClient is not initialized") + async with self.session.get(url) as resp: + resp.raise_for_status() + content_type = resp.headers.get("content-type") or "image/jpeg" + data = await resp.read() + suffix = ".jpg" + if "png" in content_type: + suffix = ".png" + elif "webp" in content_type: + suffix = ".webp" + return await self.upload_wall_photo_bytes(group_id, data, filename=f"photo{suffix}") + + async def create_wall_post( + self, + owner_id: int, + message: str, + attachments: list[str], + from_group: bool = True, + ) -> int: + response = await self.call( + "wall.post", + owner_id=int(owner_id), + from_group=1 if from_group else 0, + message=message or "", + attachments=",".join(attachments) if attachments else "", + ) + if not isinstance(response, dict) or "post_id" not in response: + raise VKAPIError(None, f"wall.post returned invalid response: {response}") + return int(response["post_id"]) + + +def normalize_vk_source(value: str) -> str: + raw = str(value or "").strip() + if not raw: + return "" + if "vk.com" in raw: + raw = raw.split("vk.com", 1)[1] + raw = raw.lstrip("/") + raw = raw.split("?", 1)[0].split("#", 1)[0].strip() + raw = re.sub(r"^(club|public)", "", raw, flags=re.IGNORECASE) + return raw.lower() + + +def is_repost(post: dict) -> bool: + return bool(post.get("copy_history")) + + +def is_deleted_or_invalid(post: dict) -> bool: + if post.get("is_deleted") or post.get("deleted"): + return True + if not post.get("id") or not post.get("date"): + return True + return False + + +def is_fatal_source_error(error: Exception | str) -> bool: + message = str(error or "").lower() + fatal_patterns = ( + "vk api error 15:", + "vk api error 18:", + "vk api error 19:", + "vk api error 100:", + "vk api error 113:", + "vk api error 1051:", + "vk api error 200:", + "vk api error 201:", + "vk api error 203:", + "[15]", + "[18]", + "[19]", + "[100]", + "[113]", + "[1051]", + "[200]", + "[201]", + "[203]", + "access denied", + "private profile", + "cannot resolve vk group", + "cannot resolve screen_name", + "method is unavailable with current profile type", + "wall is disabled", + ) + return any(pattern in message for pattern in fatal_patterns) + + +def post_vk_url(owner_id: int, post_id: int | str) -> str: + return f"https://vk.com/wall{int(owner_id)}_{post_id}" + + +@dataclass +class ExtractedMedia: + media_type: str + original_url: str | None + attachment_id: str + width: int | None = None + height: int | None = None + duration_sec: int | None = None + sort_order: int = 0 + + +def extract_media(post: dict) -> list[ExtractedMedia]: + out: list[ExtractedMedia] = [] + for idx, att in enumerate(post.get("attachments", []) or []): + att_type = att.get("type") + if att_type == "photo": + photo = att.get("photo") or {} + sizes = sorted( + photo.get("sizes", []) or [], + key=lambda s: int(s.get("width") or 0) * int(s.get("height") or 0), + reverse=True, + ) + best = sizes[0] if sizes else {} + owner_id = photo.get("owner_id") + media_id = photo.get("id") + if owner_id is None or media_id is None: + continue + out.append( + ExtractedMedia( + media_type="photo", + original_url=best.get("url"), + attachment_id=f"photo{owner_id}_{media_id}", + width=best.get("width"), + height=best.get("height"), + sort_order=idx, + ) + ) + elif att_type == "video": + video = att.get("video") or {} + owner_id = video.get("owner_id") + media_id = video.get("id") + if owner_id is None or media_id is None: + continue + out.append( + ExtractedMedia( + media_type="video", + original_url=f"https://vk.com/video{owner_id}_{media_id}", + attachment_id=f"video{owner_id}_{media_id}", + width=video.get("width"), + height=video.get("height"), + duration_sec=video.get("duration"), + sort_order=idx, + ) + ) + return out diff --git a/src/vk_parser_app/workers/__init__.py b/src/vk_parser_app/workers/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/vk_parser_app/workers/__init__.py @@ -0,0 +1 @@ + diff --git a/src/vk_parser_app/workers/ai_alerts.py b/src/vk_parser_app/workers/ai_alerts.py new file mode 100644 index 0000000..b687b01 --- /dev/null +++ b/src/vk_parser_app/workers/ai_alerts.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import asyncio +import json +import re +from typing import Any + +from aiogram import Bot +from aiogram.exceptions import TelegramRetryAfter +from loguru import logger + +from ..config import settings +from ..db import fetch_setting + + +def parse_recipients(value: Any) -> list[int]: + if isinstance(value, list): + raw_items = value + elif isinstance(value, str): + raw = value.strip() + if not raw: + return [] + try: + parsed = json.loads(raw) + raw_items = parsed if isinstance(parsed, list) else [parsed] + except json.JSONDecodeError: + raw_items = re.split(r"[\s,;]+", raw) + else: + raw_items = [value] + + recipients: list[int] = [] + for item in raw_items: + try: + recipient_id = int(str(item).strip()) + except Exception: + continue + if recipient_id not in recipients: + recipients.append(recipient_id) + return recipients + + +async def send_ai_worker_error_alert(worker_name: str, model: str, post_ids: list[int], error: str) -> None: + token = ( + str(await fetch_setting("daily_report_bot_token", "") or "").strip() + or str(await fetch_setting("tg_poster_bot_token", "") or "").strip() + or settings.tg_bot_token + ) + recipients = parse_recipients(await fetch_setting("daily_report_recipient_ids", [442509142])) + if not token or not recipients: + logger.warning("AI worker alert skipped: token or recipients are empty") + return + + post_part = ", ".join(str(post_id) for post_id in post_ids) if post_ids else "-" + text = ( + "AI worker batch failed\n" + f"worker: {worker_name}\n" + f"model: {model or '-'}\n" + f"posts: {post_part}\n" + f"error: {error[:1000]}" + ) + bot = Bot(token=token) + try: + for recipient_id in recipients: + while True: + try: + await bot.send_message(recipient_id, text, disable_web_page_preview=True) + break + except TelegramRetryAfter as exc: + await asyncio.sleep(float(exc.retry_after) + 1) + finally: + await bot.session.close() diff --git a/src/vk_parser_app/workers/ai_qualifier.py b/src/vk_parser_app/workers/ai_qualifier.py new file mode 100644 index 0000000..855d6cc --- /dev/null +++ b/src/vk_parser_app/workers/ai_qualifier.py @@ -0,0 +1,479 @@ +from __future__ import annotations + +import asyncio +import hashlib +import json +from datetime import datetime, timezone +from decimal import Decimal +from typing import Any + +import litellm +from loguru import logger + +from ..config import settings +from ..constants import WORKER_AI_QUALIFIER +from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool +from ..heartbeat import HeartbeatReporter +from ..jobs import is_worker_enabled +from .ai_alerts import send_ai_worker_error_alert + + +def now_utc() -> datetime: + return datetime.now(timezone.utc) + + +def prompt_hash(prompt: str) -> str: + return hashlib.sha256((prompt or "").encode("utf-8")).hexdigest() + + +def normalize_prompt(prompt: str) -> str: + return (prompt or "").replace("\\r\\n", "\n").replace("\\n", "\n").strip() + + +DEFAULT_CONTRACT_PROMPT = """ +OUTPUT SCHEMA (return array matching input order): +{"results":[{"id":123,"score":8,"decision":"accepted","reason":"до 10 слов на русском","reject_tag":null}]} + +Rules: +- Input is a JSON array of posts. +- Return JSON only. No markdown. No text outside JSON. +- score must be integer 1..10. +- decision must be exactly one of: "accepted", "rejected", "maybe". +- reject_tag (rejected/maybe only) must be one of: "meme", "no_product", "politics", "discount_only", "off_topic", "vacancy", "low_content", "wrong_language", "injection", "weapon", null. +- Return one result for every input post id. +""" + + +def build_system_prompt(user_prompt: str, contract_prompt: str) -> str: + parts = [normalize_prompt(user_prompt), normalize_prompt(contract_prompt or DEFAULT_CONTRACT_PROMPT)] + return "\n\n".join(part for part in parts if part).strip() + + +def normalize_model(provider: str, model: str) -> str: + provider = (provider or "").strip().lower() + model = (model or "").strip() + if not model: + return model + if "/" in model: + return model + if provider in {"openrouter", "anthropic", "gemini", "vertex_ai", "bedrock"}: + return f"{provider}/{model}" + return model + + +def parse_ai_json(content: str) -> dict: + raw = (content or "").strip() + if raw.startswith("```"): + raw = raw.strip("`") + if raw.lower().startswith("json"): + raw = raw[4:].strip() + return json.loads(raw) + + +def response_usage(response: Any) -> dict[str, Any]: + usage = getattr(response, "usage", None) + if usage is None and isinstance(response, dict): + usage = response.get("usage") + + def get(name: str) -> int | None: + if usage is None: + return None + value = getattr(usage, name, None) + if value is None and isinstance(usage, dict): + value = usage.get(name) + try: + return int(value) if value is not None else None + except Exception: + return None + + cost = getattr(response, "_hidden_params", None) + if isinstance(cost, dict): + cost = cost.get("response_cost") + else: + cost = None + try: + cost_value = Decimal(str(cost)) if cost is not None else None + except Exception: + cost_value = None + + return { + "prompt_tokens": get("prompt_tokens"), + "completion_tokens": get("completion_tokens"), + "total_tokens": get("total_tokens"), + "estimated_cost_usd": cost_value, + } + + +def validate_results(data: dict, expected_ids: set[int], min_score: int) -> list[dict]: + results = data.get("results") + if not isinstance(results, list) and isinstance(data.get("data"), dict): + results = data["data"].get("results") + if not isinstance(results, list): + raise ValueError("AI response has no results list") + + out: list[dict] = [] + seen: set[int] = set() + for item in results: + if not isinstance(item, dict): + continue + post_id = int(item.get("id")) + if post_id not in expected_ids: + raise ValueError(f"AI returned unexpected id={post_id}") + score = max(1, min(10, int(item.get("score")))) + decision = str(item.get("decision") or "").strip().lower() + if decision not in {"accepted", "rejected", "maybe"}: + decision = "accepted" if score >= min_score else "rejected" + reason = str(item.get("reason") or "").strip()[:1000] + reject_tag = item.get("reject_tag") + reject_tag = str(reject_tag).strip().lower()[:80] if reject_tag not in {None, ""} else None + seen.add(post_id) + out.append({"id": post_id, "score": score, "decision": decision, "reason": reason, "reject_tag": reject_tag}) + + missing = expected_ids - seen + if missing: + raise ValueError(f"AI response missing ids: {sorted(missing)[:10]}") + return out + + +class AIQualifierWorker: + def __init__(self) -> None: + self.pool = None + self.heartbeat = HeartbeatReporter(WORKER_AI_QUALIFIER, 30) + + async def init(self) -> None: + self.pool = await get_pool() + + async def claim_posts(self, batch_size: int) -> list[dict]: + async with self.pool.acquire() as conn: + async with conn.transaction(): + count = await conn.fetchval( + """ + SELECT COUNT(*) + FROM raw_posts + WHERE status='storage_ready' + AND COALESCE(qualification_status, 'pending') = 'pending' + AND EXISTS ( + SELECT 1 + FROM raw_post_media m + WHERE m.raw_post_id=raw_posts.id + AND COALESCE(m.editor_hidden, FALSE)=FALSE + AND m.media_type IN ('photo', 'video') + AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') <> '' + ) + AND NOT EXISTS ( + SELECT 1 + FROM raw_post_media m + WHERE m.raw_post_id=raw_posts.id + AND COALESCE(m.editor_hidden, FALSE)=FALSE + AND m.media_type IN ('photo', 'video') + AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = '' + ) + """ + ) + if int(count or 0) < batch_size: + return [] + + rows = await conn.fetch( + """ + WITH cte AS ( + SELECT id + FROM raw_posts + WHERE status='storage_ready' + AND COALESCE(qualification_status, 'pending') = 'pending' + AND EXISTS ( + SELECT 1 + FROM raw_post_media m + WHERE m.raw_post_id=raw_posts.id + AND COALESCE(m.editor_hidden, FALSE)=FALSE + AND m.media_type IN ('photo', 'video') + AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') <> '' + ) + AND NOT EXISTS ( + SELECT 1 + FROM raw_post_media m + WHERE m.raw_post_id=raw_posts.id + AND COALESCE(m.editor_hidden, FALSE)=FALSE + AND m.media_type IN ('photo', 'video') + AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = '' + ) + ORDER BY created_at ASC, id ASC + FOR UPDATE SKIP LOCKED + LIMIT $1 + ) + UPDATE raw_posts rp + SET qualification_status='processing', + updated_at=NOW() + FROM cte + WHERE rp.id=cte.id + RETURNING rp.id, rp.raw_text, rp.original_url, rp.storage_post_url, rp.created_at, + rp.source_id, + (SELECT s.name FROM sources s WHERE s.id=rp.source_id) AS source_name, + (SELECT COUNT(*) FROM raw_post_media m WHERE m.raw_post_id=rp.id) AS media_count, + (SELECT ARRAY_AGG(DISTINCT m.media_type ORDER BY m.media_type) + FROM raw_post_media m WHERE m.raw_post_id=rp.id) AS media_types + """, + batch_size, + ) + return [dict(r) for r in rows] + + async def create_batch(self, provider: str, model: str, prompt: str, payload: dict) -> int: + return int( + await self.pool.fetchval( + """ + INSERT INTO ai_qualification_batches(provider, model, prompt_hash, prompt_text, posts_count, request_json) + VALUES($1, $2, $3, $4, $5, $6::jsonb) + RETURNING id + """, + provider, + model, + prompt_hash(prompt), + prompt, + len(payload), + json.dumps(payload, ensure_ascii=False), + ) + ) + + async def finish_batch( + self, + batch_id: int, + status: str, + response: Any = None, + error: str | None = None, + usage: dict[str, Any] | None = None, + ) -> None: + counts = {"accepted": 0, "rejected": 0, "maybe": 0} + if isinstance(response, dict) and isinstance(response.get("results"), list): + for item in response["results"]: + decision = str(item.get("decision") or "") + if decision in counts: + counts[decision] += 1 + usage = usage or {} + await self.pool.execute( + """ + UPDATE ai_qualification_batches + SET status=$2, + response_json=$3::jsonb, + error=$4, + accepted_count=$5, + rejected_count=$6, + maybe_count=$7, + prompt_tokens=$8, + completion_tokens=$9, + total_tokens=$10, + estimated_cost_usd=$11, + completed_at=NOW() + WHERE id=$1 + """, + batch_id, + status, + json.dumps(response, ensure_ascii=False) if response is not None else None, + error[:2000] if error else None, + counts["accepted"], + counts["rejected"], + counts["maybe"], + usage.get("prompt_tokens"), + usage.get("completion_tokens"), + usage.get("total_tokens"), + usage.get("estimated_cost_usd"), + ) + + async def mark_posts_after_failed_batch(self, post_ids: list[int], error: str, max_attempts: int = 2) -> tuple[list[int], list[int]]: + if not post_ids: + return [], [] + rows = await self.pool.fetch( + """ + WITH target AS ( + SELECT unnest($1::bigint[]) AS id + ), + attempts AS ( + SELECT t.id, COUNT(b.id) AS failed_batches + FROM target t + LEFT JOIN ai_qualification_batches b + ON b.status='failed' + AND b.request_json @> jsonb_build_array(jsonb_build_object('id', t.id)) + GROUP BY t.id + ) + UPDATE raw_posts + SET qualification_status=CASE WHEN attempts.failed_batches < $3 THEN 'pending' ELSE 'failed' END, + qualification_reason=CASE + WHEN attempts.failed_batches < $3 THEN $2 || ' (retry once)' + ELSE $2 + END, + updated_at=NOW() + FROM attempts + WHERE raw_posts.id=attempts.id + RETURNING raw_posts.id, raw_posts.qualification_status + """, + post_ids, + error[:900], + max_attempts, + ) + retry_ids = [int(row["id"]) for row in rows if row["qualification_status"] == "pending"] + failed_ids = [int(row["id"]) for row in rows if row["qualification_status"] == "failed"] + return retry_ids, failed_ids + + async def report_failed_batch(self, model: str, post_ids: list[int], error: str) -> None: + meta = {"error": error[:300], "post_ids": post_ids, "model": model} + await self.heartbeat.beat(self.pool, status="error", meta=meta, force=True) + try: + await send_ai_worker_error_alert(WORKER_AI_QUALIFIER, model, post_ids, error) + except Exception as alert_exc: + logger.warning("AI qualifier alert failed: {}", alert_exc) + + async def apply_results(self, batch_id: int, results: list[dict], model: str, prompt: str, min_score: int) -> None: + for item in results: + score = int(item["score"]) + decision = str(item["decision"]) + model_decision = decision + if score >= min_score and decision == "maybe": + decision = "accepted" + if score < min_score and decision == "accepted": + decision = "rejected" + status = "accepted" if score >= min_score and decision == "accepted" else "rejected" + await self.pool.execute( + """ + UPDATE raw_posts + SET qualification_status=$2, + qualification_score=$3, + qualification_decision=$4, + qualification_reason=$5, + qualification_model=$6, + qualification_prompt_hash=$7, + qualification_batch_id=$8, + qualification_reject_tag=$9, + qualification_model_decision=$10, + qualified_at=NOW(), + updated_at=NOW() + WHERE id=$1 + """, + int(item["id"]), + status, + score, + decision, + item["reason"], + model, + prompt_hash(prompt), + batch_id, + item.get("reject_tag"), + model_decision, + ) + + async def call_ai( + self, + provider: str, + model: str, + api_key: str, + api_base: str, + prompt: str, + payload: dict, + temperature: float, + timeout: int, + ) -> tuple[dict, dict[str, Any]]: + messages = [ + {"role": "system", "content": prompt}, + {"role": "user", "content": json.dumps(payload, ensure_ascii=False)}, + ] + kwargs: dict[str, Any] = { + "model": normalize_model(provider, model), + "messages": messages, + "temperature": temperature, + "timeout": timeout, + "response_format": {"type": "json_object"}, + } + if api_key: + kwargs["api_key"] = api_key + if api_base: + kwargs["api_base"] = api_base + response = await asyncio.to_thread(litellm.completion, **kwargs) + content = response.choices[0].message.content + return parse_ai_json(content), response_usage(response) + + async def run_once(self) -> bool: + enabled = await is_worker_enabled(self.pool, WORKER_AI_QUALIFIER) + setting_enabled = await fetch_bool_setting("ai_qualifier_enabled", False) + if not enabled or not setting_enabled: + await self.heartbeat.beat(self.pool, status="disabled", force=True) + return False + + provider = str(await fetch_setting("ai_qualifier_provider", "openrouter") or "openrouter") + model = str(await fetch_setting("ai_qualifier_model", "") or "").strip() + api_key = str(await fetch_setting("ai_qualifier_api_key", "") or "").strip() + api_base = str(await fetch_setting("ai_qualifier_api_base", "") or "").strip() + batch_size = max(1, await fetch_int_setting("ai_qualifier_batch_size", 30)) + min_score = max(1, min(10, await fetch_int_setting("ai_qualifier_min_score", 7))) + max_text_chars = max(100, await fetch_int_setting("ai_qualifier_max_text_chars", 2000)) + temperature = max(0.0, await fetch_float_setting("ai_qualifier_temperature", 0.0)) + timeout = max(10, await fetch_int_setting("ai_qualifier_timeout_sec", 120)) + prompt = build_system_prompt( + str(await fetch_setting("ai_qualifier_prompt", "") or ""), + str(await fetch_setting("ai_qualifier_contract", DEFAULT_CONTRACT_PROMPT) or DEFAULT_CONTRACT_PROMPT), + ) + + if not model or not api_key or not prompt: + await self.heartbeat.beat(self.pool, status="not_configured", force=True) + return False + + posts = await self.claim_posts(batch_size) + await self.heartbeat.beat(self.pool, meta={"claimed": len(posts), "batch_size": batch_size}) + if not posts: + return False + + payload = [ + { + "id": int(p["id"]), + "source": p.get("source_name") or "", + "original_url": p.get("original_url") or "", + "media_count": int(p.get("media_count") or 0), + "media_types": list(p.get("media_types") or []), + "text": str(p.get("raw_text") or "")[:max_text_chars], + } + for p in posts + ] + post_ids = [int(p["id"]) for p in posts] + batch_id = await self.create_batch(provider, model, prompt, payload) + try: + response, usage = await self.call_ai(provider, model, api_key, api_base, prompt, payload, temperature, timeout) + results = validate_results(response, set(post_ids), min_score) + await self.apply_results(batch_id, results, normalize_model(provider, model), prompt, min_score) + await self.finish_batch(batch_id, "done", {"results": results}, usage=usage) + logger.info("AI qualifier batch done: id={} posts={} usage={}", batch_id, len(results), usage) + return True + except Exception as exc: + await self.finish_batch(batch_id, "failed", error=str(exc)) + retry_ids, failed_ids = await self.mark_posts_after_failed_batch(post_ids, str(exc)) + if failed_ids: + await self.report_failed_batch(normalize_model(provider, model), failed_ids, str(exc)) + else: + await self.heartbeat.beat( + self.pool, + status="retrying_after_error", + meta={"error": str(exc)[:300], "post_ids": retry_ids, "model": normalize_model(provider, model)}, + force=True, + ) + logger.exception("AI qualifier batch failed: id={} error={}", batch_id, exc) + return True + + async def run_loop(self) -> None: + await self.init() + worker_id = f"{WORKER_AI_QUALIFIER}" + logger.info("{} started", worker_id) + while True: + try: + had_work = await self.run_once() + except Exception as exc: + logger.exception("AI qualifier loop error: {}", exc) + had_work = False + interval = max(10, await fetch_int_setting("ai_qualifier_interval_sec", 60)) + if not had_work: + await asyncio.sleep(interval) + + +async def main() -> None: + logger.remove() + logger.add(lambda msg: print(msg, end=""), level=settings.log_level) + worker = AIQualifierWorker() + await worker.run_loop() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/vk_parser_app/workers/ai_writer.py b/src/vk_parser_app/workers/ai_writer.py new file mode 100644 index 0000000..54b0bec --- /dev/null +++ b/src/vk_parser_app/workers/ai_writer.py @@ -0,0 +1,556 @@ +from __future__ import annotations + +import asyncio +import hashlib +import json +import re +from datetime import datetime, timezone +from decimal import Decimal +from typing import Any + +import litellm +from loguru import logger + +from ..config import settings +from ..constants import WORKER_AI_WRITER +from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool +from ..heartbeat import HeartbeatReporter +from ..jobs import is_worker_enabled +from ..text_utils import build_publication_text, normalize_hash_tag, parse_categories +from .ai_alerts import send_ai_worker_error_alert + + +NON_TARGET_CATEGORY_ID = 18 +NON_TARGET_CATEGORY_TAGS = {"нцк", "nck"} +EMOJI_LEADING_MARKER_RE = re.compile( + r"^[ \t]*(?:🧵|🎯|🛡|🧤|📦|💵|⚙\ufe0f?|🎨|🎒|🔧|👕|💪)\s+" +) + + +def now_utc() -> datetime: + return datetime.now(timezone.utc) + + +def prompt_hash(prompt: str) -> str: + return hashlib.sha256((prompt or "").encode("utf-8")).hexdigest() + + +def normalize_prompt(prompt: str) -> str: + return (prompt or "").replace("\\r\\n", "\n").replace("\\n", "\n").strip() + + +def strip_leading_emoji_markers(text: str) -> str: + lines = str(text or "").splitlines() + cleaned: list[str] = [] + for line in lines: + cleaned.append(EMOJI_LEADING_MARKER_RE.sub("", line)) + return "\n".join(cleaned).strip() + + +DEFAULT_CONTRACT_PROMPT = """ +OUTPUT SCHEMA (return array matching input order): +{"rewrites":[{"id":123,"category_id":2,"text":"готовый текст без хэштегов","notes":"короткая заметка для редактора или null"}]} + +Rules: +- Input is a JSON object with key "posts" containing accepted posts. +- Input has key "categories" with objects: id, name, tag. +- Each post includes producer_name and producer_tag. producer_name can be a manufacturer, shop, or publishing source. +- The "text" field for every rewrite must explicitly mention producer_name at least once. Do not satisfy this only through producer_tag, hashtags, notes, or metadata. +- Pick exactly one category from the categories list in input and return its numeric id as category_id. +- If the post is non-target content, choose category_id 18 ("Не целевой контент", tag "НЦК"); the app will reject it automatically. +- Do not add links or hashtags to rewritten text. +- Return JSON only. No markdown. No text outside JSON. +- Never return an empty object. Never omit the "rewrites" key. +- Return one rewrite for every input post id. +- The "text" field must be a ready-to-publish Russian post with normal paragraph line breaks encoded as JSON string newlines. +- The "notes" field must be a short editor note in Russian or null. +""" + + +def build_system_prompt(user_prompt: str, contract_prompt: str) -> str: + parts = [normalize_prompt(user_prompt), normalize_prompt(contract_prompt or DEFAULT_CONTRACT_PROMPT)] + return "\n\n".join(part for part in parts if part).strip() + + +def normalize_model(provider: str, model: str) -> str: + provider = (provider or "").strip().lower() + model = (model or "").strip() + if not model: + return model + if "/" in model: + return model + if provider in {"openrouter", "anthropic", "gemini", "vertex_ai", "bedrock"}: + return f"{provider}/{model}" + return model + + +def parse_ai_json(content: str) -> dict: + raw = (content or "").strip() + if raw.startswith("```"): + raw = raw.strip("`") + if raw.lower().startswith("json"): + raw = raw[4:].strip() + return json.loads(raw) + + +def response_usage(response: Any) -> dict[str, Any]: + usage = getattr(response, "usage", None) + if usage is None and isinstance(response, dict): + usage = response.get("usage") + + def get(name: str) -> int | None: + if usage is None: + return None + value = getattr(usage, name, None) + if value is None and isinstance(usage, dict): + value = usage.get(name) + try: + return int(value) if value is not None else None + except Exception: + return None + + cost = getattr(response, "_hidden_params", None) + if isinstance(cost, dict): + cost = cost.get("response_cost") + else: + cost = None + try: + cost_value = Decimal(str(cost)) if cost is not None else None + except Exception: + cost_value = None + + return { + "prompt_tokens": get("prompt_tokens"), + "completion_tokens": get("completion_tokens"), + "total_tokens": get("total_tokens"), + "estimated_cost_usd": cost_value, + } + + +def validate_rewrites( + data: dict, + expected_ids: set[int], + categories: list[dict[str, Any]], + source_tags_by_id: dict[int, str], +) -> list[dict]: + rewrites = data.get("rewrites") + if not isinstance(rewrites, list) and isinstance(data.get("data"), dict): + rewrites = data["data"].get("rewrites") + if isinstance(rewrites, str): + try: + rewrites = json.loads(rewrites) + except json.JSONDecodeError: + pass + if not isinstance(rewrites, list): + raise ValueError("AI response has no rewrites list") + out: list[dict] = [] + seen: set[int] = set() + for item in rewrites: + if not isinstance(item, dict): + continue + post_id = int(item.get("id")) + if post_id not in expected_ids: + raise ValueError(f"AI returned unexpected id={post_id}") + text = strip_leading_emoji_markers(str(item.get("text") or "").strip()) + by_id = {int(c["id"]): c for c in categories if c.get("id") is not None} + by_name = {str(c["name"]).lower(): c for c in categories if c.get("name")} + if item.get("category_id") is not None: + category_id = int(item["category_id"]) + if category_id not in by_id: + raise ValueError(f"AI returned unknown category_id={category_id!r} for id={post_id}") + category_row = by_id[category_id] + else: + category_name = str(item.get("category") or "").strip().strip("#") + category_row = by_name.get(category_name.lower()) + if not category_row: + raise ValueError(f"AI returned unknown category={category_name!r} for id={post_id}") + notes = str(item.get("notes") or "").strip()[:1000] + seen.add(post_id) + category_tag = normalize_hash_tag(category_row.get("tag") or category_row.get("name") or "", "category") + reject_by_category = int(category_row["id"]) == NON_TARGET_CATEGORY_ID or category_tag.lower() in NON_TARGET_CATEGORY_TAGS + if not reject_by_category and len(text) < 40: + raise ValueError(f"AI returned too short rewrite for id={post_id}") + out.append( + { + "id": post_id, + "category_id": int(category_row["id"]), + "category": str(category_row["name"]), + "category_tag": category_tag, + "source_tag": normalize_hash_tag(source_tags_by_id.get(post_id) or "", "source"), + "text": text, + "notes": notes, + "reject_by_category": reject_by_category, + } + ) + missing = expected_ids - seen + if missing: + raise ValueError(f"AI response missing ids: {sorted(missing)[:10]}") + return out + + +async def load_writer_categories(pool) -> list[dict[str, Any]]: + try: + rows = await pool.fetch( + """ + SELECT sort_order AS id, name, tag + FROM content_categories + WHERE is_active=TRUE + ORDER BY sort_order, name + """ + ) + except Exception: + rows = [] + categories = [{"id": int(row["id"]), "name": str(row["name"]), "tag": str(row["tag"])} for row in rows] + if categories: + return categories + legacy = parse_categories(await fetch_setting("ai_writer_categories", [])) + names = legacy or [ + "защита", + "одежда", + "разгрузка", + "рюкзаки", + "airsoft", + "патчи", + "электроника", + "аксессуары", + "производство", + ] + return [{"id": idx + 1, "name": name, "tag": normalize_hash_tag(name, "category")} for idx, name in enumerate(names)] + + +class AIWriterWorker: + def __init__(self) -> None: + self.pool = None + self.heartbeat = HeartbeatReporter(WORKER_AI_WRITER, 30) + + async def init(self) -> None: + self.pool = await get_pool() + + async def claim_posts(self, batch_size: int) -> list[dict]: + async with self.pool.acquire() as conn: + async with conn.transaction(): + count = await conn.fetchval( + """ + SELECT COUNT(*) + FROM raw_posts + WHERE status='storage_ready' + AND qualification_status='accepted' + AND COALESCE(rewrite_status, 'pending') = 'pending' + AND EXISTS ( + SELECT 1 + FROM raw_post_media m + WHERE m.raw_post_id=raw_posts.id + AND COALESCE(m.editor_hidden, FALSE)=FALSE + AND m.media_type IN ('photo', 'video') + AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') <> '' + ) + AND NOT EXISTS ( + SELECT 1 + FROM raw_post_media m + WHERE m.raw_post_id=raw_posts.id + AND COALESCE(m.editor_hidden, FALSE)=FALSE + AND m.media_type IN ('photo', 'video') + AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = '' + ) + """ + ) + if int(count or 0) < batch_size: + return [] + + rows = await conn.fetch( + """ + WITH cte AS ( + SELECT id + FROM raw_posts + WHERE status='storage_ready' + AND qualification_status='accepted' + AND COALESCE(rewrite_status, 'pending') = 'pending' + AND EXISTS ( + SELECT 1 + FROM raw_post_media m + WHERE m.raw_post_id=raw_posts.id + AND COALESCE(m.editor_hidden, FALSE)=FALSE + AND m.media_type IN ('photo', 'video') + AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') <> '' + ) + AND NOT EXISTS ( + SELECT 1 + FROM raw_post_media m + WHERE m.raw_post_id=raw_posts.id + AND COALESCE(m.editor_hidden, FALSE)=FALSE + AND m.media_type IN ('photo', 'video') + AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = '' + ) + ORDER BY qualified_at ASC NULLS LAST, id ASC + FOR UPDATE SKIP LOCKED + LIMIT $1 + ) + UPDATE raw_posts rp + SET rewrite_status='processing', + updated_at=NOW() + FROM cte + WHERE rp.id=cte.id + RETURNING rp.id, rp.raw_text, rp.original_url, rp.created_at, rp.qualification_score, + rp.qualification_reason, + (SELECT s.name FROM sources s WHERE s.id=rp.source_id) AS source_name, + (SELECT s.tag FROM sources s WHERE s.id=rp.source_id) AS source_tag, + (SELECT COUNT(*) FROM raw_post_media m WHERE m.raw_post_id=rp.id) AS media_count, + (SELECT ARRAY_AGG(DISTINCT m.media_type ORDER BY m.media_type) + FROM raw_post_media m WHERE m.raw_post_id=rp.id) AS media_types + """, + batch_size, + ) + return [dict(r) for r in rows] + + async def create_batch(self, provider: str, model: str, prompt: str, payload: dict) -> int: + return int( + await self.pool.fetchval( + """ + INSERT INTO ai_writer_batches(provider, model, prompt_hash, prompt_text, posts_count, request_json) + VALUES($1, $2, $3, $4, $5, $6::jsonb) + RETURNING id + """, + provider, + model, + prompt_hash(prompt), + prompt, + len(payload["posts"]), + json.dumps(payload, ensure_ascii=False), + ) + ) + + async def finish_batch( + self, + batch_id: int, + status: str, + response: Any = None, + error: str | None = None, + usage: dict[str, Any] | None = None, + ) -> None: + ready_count = 0 + if isinstance(response, dict) and isinstance(response.get("rewrites"), list): + ready_count = len(response["rewrites"]) + usage = usage or {} + await self.pool.execute( + """ + UPDATE ai_writer_batches + SET status=$2, + response_json=$3::jsonb, + error=$4, + ready_count=$5, + failed_count=$6, + prompt_tokens=$7, + completion_tokens=$8, + total_tokens=$9, + estimated_cost_usd=$10, + completed_at=NOW() + WHERE id=$1 + """, + batch_id, + status, + json.dumps(response, ensure_ascii=False) if response is not None else None, + error[:2000] if error else None, + ready_count if status == "done" else 0, + 0 if status == "done" else 1, + usage.get("prompt_tokens"), + usage.get("completion_tokens"), + usage.get("total_tokens"), + usage.get("estimated_cost_usd"), + ) + + async def mark_posts_failed(self, post_ids: list[int], error: str) -> None: + if not post_ids: + return + await self.pool.execute( + """ + UPDATE raw_posts + SET rewrite_status='failed', + rewrite_notes=$2, + updated_at=NOW() + WHERE id=ANY($1::bigint[]) + """, + post_ids, + error[:1000], + ) + + async def report_failed_batch(self, model: str, post_ids: list[int], error: str) -> None: + meta = {"error": error[:300], "post_ids": post_ids, "model": model} + await self.heartbeat.beat(self.pool, status="error", meta=meta, force=True) + try: + await send_ai_worker_error_alert(WORKER_AI_WRITER, model, post_ids, error) + except Exception as alert_exc: + logger.warning("AI writer alert failed: {}", alert_exc) + + async def apply_rewrites(self, batch_id: int, rewrites: list[dict], model: str, prompt: str) -> None: + for item in rewrites: + final_text = build_publication_text(item["text"], item["category_tag"], item["source_tag"]) + editorial_status = "rejected" if item.get("reject_by_category") else "review" + editor_notes = "Отклонено AI: Не целевой контент" if item.get("reject_by_category") else None + await self.pool.execute( + """ + UPDATE raw_posts + SET rewrite_status='ready', + rewritten_text=$2, + rewrite_notes=$3, + rewrite_model=$4, + rewrite_prompt_hash=$5, + rewrite_batch_id=$6, + rewrite_category=$7, + rewrite_category_tag=$8, + rewrite_source_tag=$9, + rewrite_category_id=$10, + editorial_status=$11, + final_text=$12, + final_category=$7, + final_category_tag=$8, + final_source_tag=$9, + final_category_id=$10, + editor_notes=$13, + qualification_status=CASE WHEN $11='rejected' THEN 'rejected' ELSE qualification_status END, + qualification_decision=CASE WHEN $11='rejected' THEN 'rejected' ELSE qualification_decision END, + qualification_model_decision=CASE WHEN $11='rejected' THEN 'rejected' ELSE qualification_model_decision END, + qualification_reason=CASE WHEN $11='rejected' THEN 'AI writer selected non-target category' ELSE qualification_reason END, + qualification_reject_tag=CASE WHEN $11='rejected' THEN 'non_target_content' ELSE qualification_reject_tag END, + reviewed_at=CASE WHEN $11='rejected' THEN NOW() ELSE reviewed_at END, + rewritten_at=NOW(), + updated_at=NOW() + WHERE id=$1 + """, + int(item["id"]), + item["text"], + item["notes"], + model, + prompt_hash(prompt), + batch_id, + item["category"], + item["category_tag"], + item["source_tag"], + item["category_id"], + editorial_status, + final_text, + editor_notes, + ) + + async def call_ai( + self, + provider: str, + model: str, + api_key: str, + api_base: str, + prompt: str, + payload: dict, + temperature: float, + timeout: int, + ) -> tuple[dict, dict[str, Any]]: + messages = [ + {"role": "system", "content": prompt}, + {"role": "user", "content": json.dumps(payload, ensure_ascii=False)}, + ] + kwargs: dict[str, Any] = { + "model": normalize_model(provider, model), + "messages": messages, + "temperature": temperature, + "timeout": timeout, + "response_format": {"type": "json_object"}, + } + if api_key: + kwargs["api_key"] = api_key + if api_base: + kwargs["api_base"] = api_base + response = await asyncio.to_thread(litellm.completion, **kwargs) + content = response.choices[0].message.content + return parse_ai_json(content), response_usage(response) + + async def run_once(self) -> bool: + enabled = await is_worker_enabled(self.pool, WORKER_AI_WRITER) + setting_enabled = await fetch_bool_setting("ai_writer_enabled", False) + if not enabled or not setting_enabled: + await self.heartbeat.beat(self.pool, status="disabled", force=True) + return False + + provider = str(await fetch_setting("ai_writer_provider", "anthropic") or "anthropic") + model = str(await fetch_setting("ai_writer_model", "") or "").strip() + api_key = str(await fetch_setting("ai_writer_api_key", "") or "").strip() + api_base = str(await fetch_setting("ai_writer_api_base", "") or "").strip() + batch_size = max(1, await fetch_int_setting("ai_writer_batch_size", 1)) + max_text_chars = max(200, await fetch_int_setting("ai_writer_max_text_chars", 3500)) + temperature = max(0.0, await fetch_float_setting("ai_writer_temperature", 0.4)) + timeout = max(10, await fetch_int_setting("ai_writer_timeout_sec", 180)) + prompt = build_system_prompt( + str(await fetch_setting("ai_writer_prompt", "") or ""), + str(await fetch_setting("ai_writer_contract", DEFAULT_CONTRACT_PROMPT) or DEFAULT_CONTRACT_PROMPT), + ) + categories = await load_writer_categories(self.pool) + + if not model or not api_key or not prompt or not categories: + await self.heartbeat.beat(self.pool, status="not_configured", force=True) + return False + + posts = await self.claim_posts(batch_size) + await self.heartbeat.beat(self.pool, meta={"claimed": len(posts), "batch_size": batch_size}) + if not posts: + return False + + payload = { + "task": "rewrite_accepted_posts", + "categories": categories, + "posts": [ + { + "id": int(p["id"]), + "producer_name": p.get("source_name") or "", + "producer_tag": normalize_hash_tag(p.get("source_tag") or p.get("source_name") or "", "source"), + "original_url": p.get("original_url") or "", + "qualification_score": p.get("qualification_score"), + "qualification_reason": p.get("qualification_reason") or "", + "media_count": int(p.get("media_count") or 0), + "media_types": list(p.get("media_types") or []), + "text": str(p.get("raw_text") or "")[:max_text_chars], + } + for p in posts + ], + } + post_ids = [int(p["id"]) for p in posts] + source_tags_by_id = { + int(p["id"]): normalize_hash_tag(p.get("source_tag") or p.get("source_name") or "", "source") + for p in posts + } + batch_id = await self.create_batch(provider, model, prompt, payload) + response = None + usage = None + try: + response, usage = await self.call_ai(provider, model, api_key, api_base, prompt, payload, temperature, timeout) + rewrites = validate_rewrites(response, set(post_ids), categories, source_tags_by_id) + await self.apply_rewrites(batch_id, rewrites, normalize_model(provider, model), prompt) + await self.finish_batch(batch_id, "done", {"rewrites": rewrites}, usage=usage) + logger.info("AI writer batch done: id={} posts={} usage={}", batch_id, len(rewrites), usage) + return True + except Exception as exc: + await self.mark_posts_failed(post_ids, str(exc)) + await self.finish_batch(batch_id, "failed", response=response, error=str(exc), usage=usage) + await self.report_failed_batch(normalize_model(provider, model), post_ids, str(exc)) + logger.exception("AI writer batch failed: id={} error={}", batch_id, exc) + return True + + async def run_loop(self) -> None: + await self.init() + logger.info("{} started", WORKER_AI_WRITER) + while True: + try: + had_work = await self.run_once() + except Exception as exc: + logger.exception("AI writer loop error: {}", exc) + had_work = False + interval = max(10, await fetch_int_setting("ai_writer_interval_sec", 60)) + if not had_work: + await asyncio.sleep(interval) + + +async def main() -> None: + logger.remove() + logger.add(lambda msg: print(msg, end=""), level=settings.log_level) + worker = AIWriterWorker() + await worker.run_loop() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/vk_parser_app/workers/daily_report.py b/src/vk_parser_app/workers/daily_report.py new file mode 100644 index 0000000..1abc796 --- /dev/null +++ b/src/vk_parser_app/workers/daily_report.py @@ -0,0 +1,457 @@ +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import re +import sys +from datetime import date, datetime, time, timedelta, timezone +from typing import Any +from zoneinfo import ZoneInfo + +from aiogram import Bot +from aiogram.exceptions import TelegramRetryAfter +from loguru import logger + +from ..config import settings +from ..constants import WORKER_DAILY_REPORT +from ..db import fetch_bool_setting, fetch_int_setting, fetch_setting, get_pool +from ..heartbeat import HeartbeatReporter +from ..jobs import is_worker_enabled + +LOCAL_TZ = ZoneInfo("Asia/Yekaterinburg") +MESSAGE_LIMIT = 4096 + + +def valid_time(value: str) -> bool: + return bool(re.fullmatch(r"(?:[01]\d|2[0-3]):[0-5]\d", str(value or "").strip())) + + +def parse_recipients(value: Any) -> list[int]: + raw_items: list[Any] + if isinstance(value, list): + raw_items = value + elif isinstance(value, str): + raw = value.strip() + if not raw: + return [] + try: + parsed = json.loads(raw) + raw_items = parsed if isinstance(parsed, list) else [parsed] + except json.JSONDecodeError: + raw_items = re.split(r"[\s,;]+", raw) + else: + raw_items = [value] + out: list[int] = [] + for item in raw_items: + try: + recipient_id = int(str(item).strip()) + except Exception: + continue + if recipient_id not in out: + out.append(recipient_id) + return out + + +def day_bounds(report_date: date) -> tuple[datetime, datetime]: + start_local = datetime.combine(report_date, time.min, tzinfo=LOCAL_TZ) + end_local = start_local + timedelta(days=1) + return start_local.astimezone(timezone.utc), end_local.astimezone(timezone.utc) + + +def fmt_int(value: Any) -> str: + try: + return f"{int(value):,}".replace(",", " ") + except Exception: + return "0" + + +def split_chunks(text: str, limit: int = MESSAGE_LIMIT) -> list[str]: + text = str(text or "").strip() + if not text: + return [] + chunks: list[str] = [] + while len(text) > limit: + split_at = text.rfind("\n", 0, limit) + if split_at < limit // 2: + split_at = limit + chunks.append(text[:split_at].strip()) + text = text[split_at:].strip() + if text: + chunks.append(text) + return chunks + + +class DailyReportWorker: + def __init__(self) -> None: + self.pool = None + self.bot: Bot | None = None + self.bot_token = "" + self.recipients: list[int] = [] + self.report_time = "00:04" + self.interval_sec = 60 + self.heartbeat = HeartbeatReporter(WORKER_DAILY_REPORT, 30) + + async def init(self) -> None: + self.pool = await get_pool() + await self.reload_settings() + + async def close(self) -> None: + if self.bot: + await self.bot.session.close() + + async def reload_settings(self) -> None: + token = ( + str(await fetch_setting("daily_report_bot_token", "") or "").strip() + or str(await fetch_setting("tg_poster_bot_token", "") or "").strip() + or settings.tg_bot_token + ) + if not token: + raise RuntimeError("daily_report_bot_token, tg_poster_bot_token and TG_BOT_TOKEN are empty") + if token != self.bot_token: + if self.bot: + await self.bot.session.close() + self.bot = Bot(token=token) + self.bot_token = token + self.recipients = parse_recipients(await fetch_setting("daily_report_recipient_ids", [442509142])) + if not self.recipients: + raise RuntimeError("daily_report_recipient_ids is empty") + report_time = str(await fetch_setting("daily_report_time", "00:04") or "00:04").strip() + self.report_time = report_time if valid_time(report_time) else "00:04" + self.interval_sec = max(10, await fetch_int_setting("daily_report_interval_sec", 60)) + logger.info("Daily report config: time={} recipients={}", self.report_time, self.recipients) + + async def save_last_sent_date(self, report_date: date) -> None: + await self.pool.execute( + """ + UPDATE app_settings + SET value_json=$2::jsonb, + updated_at=NOW() + WHERE key=$1 + """, + "daily_report_last_sent_date", + json.dumps(report_date.isoformat()), + ) + + async def countval(self, sql: str, *args: Any) -> int: + return int(await self.pool.fetchval(sql, *args) or 0) + + async def collect(self, report_date: date) -> dict[str, Any]: + start_utc, end_utc = day_bounds(report_date) + now_utc = datetime.now(timezone.utc) + async with self.pool.acquire() as conn: + raw_collected = await conn.fetchval( + "SELECT COUNT(*) FROM raw_posts WHERE created_at >= $1 AND created_at < $2", + start_utc, + end_utc, + ) + storage_ready = await conn.fetchval( + "SELECT COUNT(*) FROM raw_posts WHERE status='storage_ready' AND updated_at >= $1 AND updated_at < $2", + start_utc, + end_utc, + ) + media_failed = await conn.fetchval( + "SELECT COUNT(*) FROM raw_post_media WHERE status IN ('failed','link_only') AND updated_at >= $1 AND updated_at < $2", + start_utc, + end_utc, + ) + qualification_rows = await conn.fetch( + """ + SELECT COALESCE(qualification_status, 'pending') AS status, COUNT(*) AS count + FROM raw_posts + WHERE qualified_at >= $1 AND qualified_at < $2 + GROUP BY COALESCE(qualification_status, 'pending') + """, + start_utc, + end_utc, + ) + writer_ready = await conn.fetchval( + "SELECT COUNT(*) FROM raw_posts WHERE rewrite_status='ready' AND rewritten_at >= $1 AND rewritten_at < $2", + start_utc, + end_utc, + ) + review_now = await conn.fetchval( + """ + SELECT COUNT(*) + FROM raw_posts + WHERE rewrite_status='ready' + AND COALESCE(editorial_status, 'review')='review' + """ + ) + accepted_today = await conn.fetchval( + """ + SELECT COUNT(*) + FROM raw_posts + WHERE reviewed_at >= $1 AND reviewed_at < $2 + AND editorial_status='accepted' + """, + start_utc, + end_utc, + ) + rejected_today = await conn.fetchval( + """ + SELECT COUNT(*) + FROM raw_posts + WHERE reviewed_at >= $1 AND reviewed_at < $2 + AND editorial_status='rejected' + """, + start_utc, + end_utc, + ) + publication_rows = await conn.fetch( + """ + SELECT platform, status, COUNT(*) AS count + FROM post_publications + WHERE updated_at >= $1 AND updated_at < $2 + GROUP BY platform, status + ORDER BY platform, status + """, + start_utc, + end_utc, + ) + tg_queue = await conn.fetchval( + """ + SELECT COUNT(*) + FROM raw_posts rp + WHERE rp.status='storage_ready' + AND rp.rewrite_status='ready' + AND rp.editorial_status='accepted' + AND COALESCE(rp.publication_status, 'pending')='pending' + AND EXISTS ( + SELECT 1 FROM raw_post_media m + WHERE m.raw_post_id=rp.id + AND COALESCE(m.editor_hidden,FALSE)=FALSE + AND m.media_type IN ('photo','video') + AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') <> '' + ) + AND NOT EXISTS ( + SELECT 1 FROM raw_post_media m + WHERE m.raw_post_id=rp.id + AND COALESCE(m.editor_hidden,FALSE)=FALSE + AND m.media_type IN ('photo','video') + AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = '' + ) + """ + ) + vk_queue = await conn.fetchval( + """ + SELECT COUNT(*) + FROM raw_posts rp + LEFT JOIN post_publications pp ON pp.raw_post_id=rp.id AND pp.platform='vk' + WHERE rp.status='storage_ready' + AND rp.rewrite_status='ready' + AND COALESCE(rp.editorial_status, 'review') IN ('accepted', 'published', 'publish_failed') + AND COALESCE(pp.status, 'pending') IN ('pending', 'publish_failed') + AND EXISTS ( + SELECT 1 FROM raw_post_media rpm + WHERE rpm.raw_post_id=rp.id + AND COALESCE(rpm.editor_hidden, FALSE)=FALSE + AND COALESCE(rpm.editor_added, FALSE)=FALSE + AND ( + (rpm.media_type='photo' AND rpm.original_attachment_id LIKE 'photo%') + OR (rpm.media_type='video' AND rpm.original_attachment_id LIKE 'video%') + ) + ) + """ + ) + publish_failed_now = await conn.fetchval( + """ + SELECT COUNT(*) + FROM raw_posts + WHERE rewrite_status='ready' + AND COALESCE(editorial_status, '')='publish_failed' + """ + ) + workers = await conn.fetch( + """ + SELECT wc.name, wc.enabled, wh.heartbeat_at, wh.status + FROM worker_controls wc + LEFT JOIN worker_heartbeats wh ON wh.name=wc.name + ORDER BY wc.name + """ + ) + top_errors = await conn.fetch( + """ + SELECT platform, error, COUNT(*) AS count + FROM post_publications + WHERE status='publish_failed' + AND updated_at >= $1 AND updated_at < $2 + AND COALESCE(error, '') <> '' + GROUP BY platform, error + ORDER BY COUNT(*) DESC + LIMIT 5 + """, + start_utc, + end_utc, + ) + return { + "date": report_date, + "raw_collected": raw_collected, + "storage_ready": storage_ready, + "media_failed": media_failed, + "qualification": {row["status"]: int(row["count"]) for row in qualification_rows}, + "writer_ready": writer_ready, + "review_now": review_now, + "accepted_today": accepted_today, + "rejected_today": rejected_today, + "publications": {(row["platform"], row["status"]): int(row["count"]) for row in publication_rows}, + "tg_queue": tg_queue, + "vk_queue": vk_queue, + "publish_failed_now": publish_failed_now, + "workers": [dict(row) for row in workers], + "top_errors": [dict(row) for row in top_errors], + "now_utc": now_utc, + } + + def render(self, data: dict[str, Any]) -> str: + q = data["qualification"] + pubs = data["publications"] + workers_lines: list[str] = [] + for row in data["workers"]: + heartbeat_at = row.get("heartbeat_at") + is_stale = True + if heartbeat_at: + is_stale = (data["now_utc"] - heartbeat_at).total_seconds() > 600 + marker = "OK" if row.get("enabled") and heartbeat_at and not is_stale else ("OFF" if not row.get("enabled") else "STALE") + workers_lines.append(f"{marker} {row['name']}: {row.get('status') or '-'}") + errors_lines = [ + f"- {row['platform']}: {row['count']} x {str(row['error'])[:120]}" + for row in data["top_errors"] + ] + if not errors_lines: + errors_lines = ["- нет"] + report_date = data["date"].strftime("%d.%m.%Y") + return "\n".join( + [ + f"Ежедневный отчет {settings.display_site_title} за {report_date}", + "", + "Сбор:", + f"raw собрано: {fmt_int(data['raw_collected'])}", + f"storage ready за день: {fmt_int(data['storage_ready'])}", + f"media failed/link_only: {fmt_int(data['media_failed'])}", + "", + "AI:", + f"принято категоризатором: {fmt_int(q.get('accepted', 0))}", + f"отклонено категоризатором: {fmt_int(q.get('rejected', 0))}", + f"ошибки категоризатора: {fmt_int(q.get('failed', 0))}", + f"рерайтов готово: {fmt_int(data['writer_ready'])}", + "", + "Редакторская:", + f"принято редактором за день: {fmt_int(data['accepted_today'])}", + f"отклонено редактором за день: {fmt_int(data['rejected_today'])}", + f"на проверке сейчас: {fmt_int(data['review_now'])}", + f"ошибка публикации сейчас: {fmt_int(data['publish_failed_now'])}", + "", + "Публикации за день:", + f"TG опубликовано: {fmt_int(pubs.get(('tg', 'published'), 0))}", + f"VK опубликовано: {fmt_int(pubs.get(('vk', 'published'), 0))}", + f"TG ошибки: {fmt_int(pubs.get(('tg', 'publish_failed'), 0))}", + f"VK ошибки: {fmt_int(pubs.get(('vk', 'publish_failed'), 0))}", + "", + "Очередь сейчас:", + f"TG кандидатов: {fmt_int(data['tg_queue'])}", + f"VK кандидатов: {fmt_int(data['vk_queue'])}", + "", + "Ошибки дня:", + *errors_lines, + "", + "Воркеры:", + *workers_lines, + ] + ) + + async def send_text(self, text: str) -> None: + if not self.bot: + raise RuntimeError("bot is not initialized") + for recipient_id in self.recipients: + for chunk in split_chunks(text): + while True: + try: + await self.bot.send_message(recipient_id, chunk, disable_web_page_preview=True) + break + except TelegramRetryAfter as exc: + await asyncio.sleep(float(exc.retry_after) + 1) + + async def send_report(self, report_date: date, save_last_sent: bool = False) -> str: + await self.reload_settings() + data = await self.collect(report_date) + text = self.render(data) + await self.send_text(text) + if save_last_sent: + await self.save_last_sent_date(report_date) + await self.heartbeat.beat( + self.pool, + status="sent", + meta={"date": report_date.isoformat(), "recipients": self.recipients}, + force=True, + ) + return text + + async def due_report_date(self) -> date | None: + now = datetime.now(LOCAL_TZ) + hour, minute = [int(part) for part in self.report_time.split(":", 1)] + slot_start = datetime.combine(now.date(), time(hour=hour, minute=minute), tzinfo=LOCAL_TZ) + slot_end = slot_start + timedelta(minutes=10) + if not (slot_start <= now < slot_end): + return None + report_date = now.date() - timedelta(days=1) + last_sent = str(await fetch_setting("daily_report_last_sent_date", "") or "").strip() + if last_sent == report_date.isoformat(): + return None + return report_date + + async def run_once(self) -> bool: + await self.reload_settings() + enabled = await is_worker_enabled(self.pool, WORKER_DAILY_REPORT) + app_enabled = await fetch_bool_setting("daily_report_enabled", True) + if not enabled or not app_enabled: + await self.heartbeat.beat(self.pool, status="disabled", force=True) + return False + report_date = await self.due_report_date() + if not report_date: + await self.heartbeat.beat(self.pool, status="idle") + return False + await self.send_report(report_date, save_last_sent=True) + return True + + async def run_loop(self) -> None: + await self.init() + logger.info("{} started pid={}", WORKER_DAILY_REPORT, os.getpid()) + while True: + try: + await self.run_once() + except Exception as exc: + logger.exception("Daily report loop error: {}", exc) + if self.pool: + await self.heartbeat.beat(self.pool, status="error", meta={"error": str(exc)}, force=True) + await asyncio.sleep(self.interval_sec) + + +async def run_send_now(report_date: date | None = None) -> str: + worker = DailyReportWorker() + await worker.init() + try: + return await worker.send_report(report_date or datetime.now(LOCAL_TZ).date(), save_last_sent=False) + finally: + await worker.close() + + +async def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--send-now", action="store_true") + parser.add_argument("--date", help="Report date YYYY-MM-DD, default today in EKB for --send-now") + args = parser.parse_args() + logger.remove() + logger.add(sys.stdout, level=settings.log_level) + if args.send_now: + report_date = date.fromisoformat(args.date) if args.date else None + text = await run_send_now(report_date) + print(text) + return + worker = DailyReportWorker() + await worker.run_loop() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/vk_parser_app/workers/max_poster.py b/src/vk_parser_app/workers/max_poster.py new file mode 100644 index 0000000..8882944 --- /dev/null +++ b/src/vk_parser_app/workers/max_poster.py @@ -0,0 +1,728 @@ +from __future__ import annotations + +import asyncio +import json +import mimetypes +import os +import re +import shutil +import sys +import tempfile +from datetime import datetime, time, timedelta +from pathlib import Path +from typing import Any +from urllib.parse import quote +from zoneinfo import ZoneInfo + +import aiohttp +from loguru import logger + +from ..config import settings +from ..constants import ( + PUBLICATION_STATUS_FAILED, + PUBLICATION_STATUS_PENDING, + PUBLICATION_STATUS_PUBLISHED, + WORKER_MAX_POSTER, +) +from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool +from ..heartbeat import HeartbeatReporter +from ..jobs import is_worker_enabled +from ..text_utils import build_publication_text + +LOCAL_TZ = ZoneInfo("Asia/Yekaterinburg") +MAX_MESSAGE_LIMIT = 4000 +MAX_MEDIA_ITEMS = 10 +MAX_VIDEO_BYTES = 250 * 1024 * 1024 +PROJECT_ROOT = Path(__file__).resolve().parents[3] +UPLOAD_ROOT = PROJECT_ROOT / "uploads" + + +class MAXAPIError(RuntimeError): + def __init__(self, message: str, status: int | None = None, code: str = "") -> None: + super().__init__(message) + self.status = status + self.code = code + + +def valid_time(value: str) -> bool: + return bool(re.fullmatch(r"(?:[01]\d|2[0-3]):[0-5]\d", str(value or "").strip())) + + +def normalize_schedule(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list): + value = [] + rows: list[dict[str, Any]] = [] + for idx, item in enumerate(value): + if not isinstance(item, dict): + continue + time_value = str(item.get("time") or "").strip() + if not valid_time(time_value): + continue + count = max(0, min(20, int(item.get("count") or 0))) + if count < 1: + continue + rows.append( + { + "id": str(item.get("id") or f"slot-{idx + 1}").strip()[:80], + "time": time_value, + "count": count, + "enabled": bool(item.get("enabled", True)), + } + ) + return sorted(rows, key=lambda row: row["time"]) + + +def split_message_chunks(text: str, limit: int) -> list[str]: + text = str(text or "").strip() + if not text: + return [] + chunks: list[str] = [] + while len(text) > limit: + split_at = text.rfind("\n", 0, limit) + if split_at < limit // 2: + split_at = text.rfind(" ", 0, limit) + if split_at < limit // 2: + split_at = limit + chunks.append(text[:split_at].strip()) + text = text[split_at:].strip() + if text: + chunks.append(text) + return chunks + + +def local_upload_path(url: str) -> Path | None: + raw = str(url or "").strip() + prefix = "/uploads/" + if not raw.startswith(prefix): + return None + path = (UPLOAD_ROOT / raw.removeprefix(prefix)).resolve() + try: + path.relative_to(UPLOAD_ROOT.resolve()) + except ValueError: + return None + return path if path.is_file() else None + + +class MAXAPIClient: + def __init__(self, token: str, base_url: str, timeout_sec: int, max_attempts: int, retry_backoff_max_sec: int) -> None: + self.token = token + self.base_url = base_url.rstrip("/") + self.timeout_sec = timeout_sec + self.max_attempts = max_attempts + self.retry_backoff_max_sec = retry_backoff_max_sec + self.session: aiohttp.ClientSession | None = None + + async def __aenter__(self) -> "MAXAPIClient": + timeout = aiohttp.ClientTimeout(total=self.timeout_sec) + self.session = aiohttp.ClientSession(timeout=timeout, headers={"Authorization": self.token}) + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + if self.session: + await self.session.close() + + async def request(self, method: str, path: str, **kwargs) -> dict[str, Any]: + if not self.session: + raise RuntimeError("MAX session is not initialized") + url = f"{self.base_url}{path}" + last_error = "" + for attempt in range(1, self.max_attempts + 1): + try: + async with self.session.request(method, url, **kwargs) as resp: + text = await resp.text() + try: + data = json.loads(text) if text else {} + except json.JSONDecodeError: + data = {"raw": text} + if 200 <= resp.status < 300: + return data + retry_after = resp.headers.get("Retry-After") + message = data.get("message") if isinstance(data, dict) else text + code = data.get("code") if isinstance(data, dict) else "" + last_error = f"MAX API {resp.status} {code}: {message or text}" + if resp.status == 429 and retry_after: + await asyncio.sleep(float(retry_after) + 0.5) + continue + if code == "attachment.not.ready" and attempt < self.max_attempts: + await asyncio.sleep(min(2 ** attempt, self.retry_backoff_max_sec)) + continue + if resp.status < 500: + raise MAXAPIError(last_error, resp.status, str(code or "")) + except Exception as exc: + last_error = str(exc) + if attempt >= self.max_attempts: + raise + await asyncio.sleep(min(2 ** attempt, self.retry_backoff_max_sec)) + raise RuntimeError(last_error or "MAX API request failed") + + async def send_message(self, chat_id: int, text: str, attachments: list[dict[str, Any]] | None = None) -> dict[str, Any]: + payload: dict[str, Any] = {"text": text, "notify": True} + if attachments: + payload["attachments"] = attachments + return await self.request("POST", f"/messages?chat_id={chat_id}", json=payload) + + async def get_message(self, message_id: str) -> dict[str, Any]: + return await self.request("GET", f"/messages/{quote(message_id, safe='')}") + + async def get_video_info(self, token: str) -> dict[str, Any]: + return await self.request("GET", f"/videos/{quote(token, safe='')}") + + async def get_upload_url(self, media_type: str) -> dict[str, Any]: + upload_type = "video" if media_type == "video" else "image" + return await self.request("POST", f"/uploads?type={upload_type}") + + async def upload_file(self, upload_url: str, path: Path) -> dict[str, Any]: + if not self.session: + raise RuntimeError("MAX session is not initialized") + form = aiohttp.FormData() + content_type = mimetypes.guess_type(str(path))[0] or "application/octet-stream" + with path.open("rb") as fh: + form.add_field("data", fh, filename=path.name, content_type=content_type) + async with self.session.post(upload_url, data=form) as resp: + text = await resp.text() + try: + data = json.loads(text) if text else {} + except json.JSONDecodeError: + data = {"raw": text} + if resp.status < 200 or resp.status >= 300: + raise RuntimeError(f"MAX upload {resp.status}: {data}") + return data + + async def react(self, path_template: str, message_id: str, reaction: str) -> None: + path = path_template.format(message_id=message_id) + payload = {"reaction": reaction} + await self.request("POST", path, json=payload) + + +class MAXPoster: + def __init__(self) -> None: + self.pool = None + self.heartbeat = HeartbeatReporter(WORKER_MAX_POSTER, 30) + self.token = "" + self.chat_id = 0 + self.api_base_url = "https://platform-api2.max.ru" + self.schedule: list[dict[str, Any]] = [] + self.interval_sec = 60 + self.message_limit = MAX_MESSAGE_LIMIT + self.media_max_items = MAX_MEDIA_ITEMS + self.media_process_delay_sec = 3.0 + self.video_ready_attempts = 5 + self.video_ready_delay_sec = 10.0 + self.max_attempts = 3 + self.retry_backoff_max_sec = 30 + self.send_delay_sec = 1.0 + self.recent_window = 20 + self.category_repeat_penalty = 3.0 + self.source_repeat_penalty = 4.0 + self.auto_reaction_enabled = True + self.auto_reaction = "👍" + self.reaction_path_template = "/messages/{message_id}/reactions" + self.dry_run = False + + async def init(self) -> None: + self.pool = await get_pool() + + async def reload_settings(self) -> None: + self.token = str(await fetch_setting("max_poster_bot_token", "") or "").strip() + self.chat_id = int(str(await fetch_setting("max_poster_chat_id", "0") or "0").strip()) + self.api_base_url = str(await fetch_setting("max_poster_api_base_url", self.api_base_url) or self.api_base_url).strip() + self.schedule = normalize_schedule(await fetch_setting("max_poster_schedule_json", [])) + self.interval_sec = max(10, await fetch_int_setting("max_poster_interval_sec", 60)) + self.message_limit = max(512, min(MAX_MESSAGE_LIMIT, await fetch_int_setting("max_poster_message_limit", MAX_MESSAGE_LIMIT))) + self.media_max_items = max(0, min(MAX_MEDIA_ITEMS, await fetch_int_setting("max_poster_media_max_items", MAX_MEDIA_ITEMS))) + self.media_process_delay_sec = max(0.0, await fetch_float_setting("max_poster_media_process_delay_sec", 3.0)) + self.video_ready_attempts = max(1, await fetch_int_setting("max_poster_video_ready_attempts", 5)) + self.video_ready_delay_sec = max(1.0, await fetch_float_setting("max_poster_video_ready_delay_sec", 10.0)) + self.max_attempts = max(1, await fetch_int_setting("max_poster_max_attempts", 3)) + self.retry_backoff_max_sec = max(1, await fetch_int_setting("max_poster_retry_backoff_max_sec", 30)) + self.send_delay_sec = max(0.0, await fetch_float_setting("max_poster_send_delay_sec", 1.0)) + self.recent_window = max(1, await fetch_int_setting("max_poster_recent_window", 20)) + self.category_repeat_penalty = max(0.0, await fetch_float_setting("max_poster_category_repeat_penalty", 3.0)) + self.source_repeat_penalty = max(0.0, await fetch_float_setting("max_poster_source_repeat_penalty", 4.0)) + self.auto_reaction_enabled = await fetch_bool_setting("max_poster_auto_reaction_enabled", True) + self.auto_reaction = str(await fetch_setting("max_poster_auto_reaction", "👍") or "👍").strip() or "👍" + self.reaction_path_template = str(await fetch_setting("max_poster_reaction_path_template", self.reaction_path_template) or "").strip() + self.dry_run = await fetch_bool_setting("max_poster_dry_run", False) + if not self.token: + raise RuntimeError("max_poster_bot_token is empty") + if not self.chat_id: + raise RuntimeError("max_poster_chat_id is empty") + logger.info("MAX poster config: chat={} schedule={} dry_run={}", self.chat_id, self.schedule, self.dry_run) + + async def due_slots(self) -> list[dict[str, Any]]: + now = datetime.now(LOCAL_TZ) + today = now.date() + due: list[dict[str, Any]] = [] + for row in self.schedule: + if not row.get("enabled"): + continue + hour, minute = [int(part) for part in str(row["time"]).split(":", 1)] + slot_start = datetime.combine(today, time(hour=hour, minute=minute), tzinfo=LOCAL_TZ) + slot_end = slot_start + timedelta(minutes=10) + if not (slot_start <= now < slot_end): + continue + inserted = await self.pool.fetchrow( + """ + INSERT INTO publication_runs(poster, schedule_id, scheduled_for, scheduled_time, planned_count, status) + VALUES($1, $2, $3, $4, $5, 'started') + ON CONFLICT (poster, schedule_id, scheduled_for) DO NOTHING + RETURNING id + """, + "max", + row["id"], + today, + row["time"], + int(row["count"]), + ) + if inserted: + due.append({**row, "run_id": int(inserted["id"]), "date": today}) + return due + + async def load_recent(self) -> list[dict[str, Any]]: + rows = await self.pool.fetch( + """ + SELECT rp.final_category_tag, rp.rewrite_category_tag, rp.final_source_tag, rp.rewrite_source_tag + FROM post_publications pp + JOIN raw_posts rp ON rp.id=pp.raw_post_id + WHERE pp.platform='max' + AND pp.status=$1 + AND pp.target_id=$2 + ORDER BY pp.published_at DESC NULLS LAST, pp.id DESC + LIMIT $3::int + """, + PUBLICATION_STATUS_PUBLISHED, + str(self.chat_id), + self.recent_window, + ) + return [dict(row) for row in rows] + + async def load_candidates(self, limit: int) -> list[dict[str, Any]]: + rows = await self.pool.fetch( + """ + SELECT rp.*, s.name AS source_name, s.tag AS source_tag + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + LEFT JOIN post_publications pp ON pp.raw_post_id=rp.id AND pp.platform='max' + WHERE rp.status='storage_ready' + AND rp.rewrite_status='ready' + AND COALESCE(rp.editorial_status, 'review') IN ('accepted', 'published', 'publish_failed') + AND COALESCE(pp.status, $1) = $1 + AND EXISTS ( + SELECT 1 + FROM raw_post_media rpm + WHERE rpm.raw_post_id=rp.id + AND COALESCE(rpm.editor_hidden, FALSE)=FALSE + AND rpm.media_type IN ('photo', 'video') + AND COALESCE(rpm.original_url, '') <> '' + ) + ORDER BY COALESCE(rp.reviewed_at, rp.edited_at, rp.rewritten_at, rp.created_at) ASC, + rp.id ASC + LIMIT $2::int + """, + PUBLICATION_STATUS_PENDING, + max(limit, 50), + ) + return [dict(row) for row in rows] + + def choose_posts(self, candidates: list[dict[str, Any]], recent: list[dict[str, Any]], count: int) -> list[dict[str, Any]]: + selected: list[dict[str, Any]] = [] + recent_categories = [str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "") for row in recent] + recent_sources = [str(row.get("final_source_tag") or row.get("rewrite_source_tag") or "") for row in recent] + pool = list(candidates) + while pool and len(selected) < count: + best_idx = 0 + selected_categories = [str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "") for row in selected] + selected_sources = [str(row.get("final_source_tag") or row.get("rewrite_source_tag") or row.get("source_tag") or "") for row in selected] + previous_category = selected_categories[-1] if selected_categories else (recent_categories[0] if recent_categories else "") + previous_source = selected_sources[-1] if selected_sources else (recent_sources[0] if recent_sources else "") + best_rank: tuple[int, float, int] | None = None + for idx, post in enumerate(pool): + category = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or "") + source = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "") + immediate_repeats = int(bool(category) and category == previous_category) + int(bool(source) and source == previous_source) + repeat_penalty = ( + (recent_categories.count(category) + selected_categories.count(category)) * self.category_repeat_penalty + + (recent_sources.count(source) + selected_sources.count(source)) * self.source_repeat_penalty + ) + rank = (immediate_repeats, repeat_penalty, idx) + if best_rank is None or rank < best_rank: + best_idx = idx + best_rank = rank + selected.append(pool.pop(best_idx)) + return selected + + async def load_media(self, raw_post_id: int) -> list[dict[str, Any]]: + rows = await self.pool.fetch( + """ + SELECT * + FROM raw_post_media + WHERE raw_post_id=$1 + AND COALESCE(editor_hidden, FALSE)=FALSE + AND media_type IN ('photo', 'video') + AND COALESCE(original_url, '') <> '' + ORDER BY sort_order ASC, id ASC + LIMIT $2::int + """, + raw_post_id, + self.media_max_items, + ) + return [dict(row) for row in rows] + + def build_text(self, post: dict[str, Any]) -> str: + category_tag = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or post.get("final_category") or post.get("rewrite_category") or "") + source_tag = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "") + return build_publication_text(post.get("final_text") or post.get("rewritten_text") or "", category_tag, source_tag) + + async def download_media_to_temp(self, session: aiohttp.ClientSession, item: dict[str, Any]) -> Path | None: + media_type = str(item.get("media_type") or "") + file_id = str(item.get("tg_file_id") or item.get("storage_attachment_id") or "").strip() + if media_type == "video" and file_id: + return await self.download_telegram_file_to_temp(session, file_id, ".mp4") + url = str(item.get("original_url") or "").strip() + local_path = local_upload_path(url) + if local_path: + return local_path + if not url.startswith(("http://", "https://")): + return None + suffix = Path(url.split("?", 1)[0]).suffix[:12] or (".mp4" if item.get("media_type") == "video" else ".jpg") + tmp = tempfile.NamedTemporaryFile(prefix="max-poster-", suffix=suffix, delete=False) + path = Path(tmp.name) + tmp.close() + total = 0 + try: + async with session.get(url) as resp: + resp.raise_for_status() + with path.open("wb") as fh: + async for chunk in resp.content.iter_chunked(1024 * 512): + total += len(chunk) + if total > MAX_VIDEO_BYTES: + raise RuntimeError("media file is larger than MAX video limit") + fh.write(chunk) + return path + except Exception: + path.unlink(missing_ok=True) + raise + + async def download_telegram_file_to_temp(self, session: aiohttp.ClientSession, file_id: str, suffix: str) -> Path: + token = settings.tg_bot_token + if not token: + raise RuntimeError("TG_BOT_TOKEN is empty; cannot download stored video") + api_base = str(await fetch_setting("local_bot_api_url", settings.local_bot_api_url) or "").strip().rstrip("/") + if not api_base: + api_base = "https://api.telegram.org" + get_file_url = f"{api_base}/bot{token}/getFile" + async with session.post(get_file_url, json={"file_id": file_id}) as resp: + data = await resp.json(content_type=None) + if not data.get("ok") or not isinstance(data.get("result"), dict): + raise RuntimeError(f"Telegram getFile failed: {data}") + file_path = str(data["result"].get("file_path") or "").strip() + if not file_path: + raise RuntimeError(f"Telegram getFile did not return file_path: {data}") + local_file = Path(file_path) + if local_file.is_absolute() and local_file.is_file(): + if local_file.stat().st_size > MAX_VIDEO_BYTES: + raise RuntimeError("telegram local video file is larger than MAX video limit") + tmp = tempfile.NamedTemporaryFile(prefix="max-poster-tg-", suffix=suffix, delete=False) + path = Path(tmp.name) + tmp.close() + try: + shutil.copyfile(local_file, path) + return path + except Exception: + path.unlink(missing_ok=True) + raise + download_url = f"{api_base}/file/bot{token}/{file_path}" + tmp = tempfile.NamedTemporaryFile(prefix="max-poster-tg-", suffix=suffix, delete=False) + path = Path(tmp.name) + tmp.close() + total = 0 + try: + async with session.get(download_url) as resp: + resp.raise_for_status() + with path.open("wb") as fh: + async for chunk in resp.content.iter_chunked(1024 * 1024): + total += len(chunk) + if total > MAX_VIDEO_BYTES: + raise RuntimeError("telegram video file is larger than MAX video limit") + fh.write(chunk) + return path + except Exception: + path.unlink(missing_ok=True) + raise + + async def upload_media(self, client: MAXAPIClient, item: dict[str, Any]) -> dict[str, Any] | None: + media_type = str(item.get("media_type") or "") + if media_type not in {"photo", "video"}: + return None + temp_path: Path | None = None + downloaded = False + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=180)) as session: + temp_path = await self.download_media_to_temp(session, item) + downloaded = bool(temp_path and not local_upload_path(str(item.get("original_url") or ""))) + if not temp_path: + return None + try: + upload_info = await client.get_upload_url("video" if media_type == "video" else "image") + upload_url = str(upload_info.get("url") or "").strip() + if not upload_url: + raise RuntimeError(f"MAX upload URL is empty: {upload_info}") + payload = await client.upload_file(upload_url, temp_path) + if media_type == "video" and upload_info.get("token") and not payload.get("token"): + payload["token"] = upload_info["token"] + has_image_token = bool(payload.get("photos")) if isinstance(payload.get("photos"), dict) else False + if not payload.get("token") and not has_image_token: + raise RuntimeError(f"MAX upload did not return token: {payload}") + return {"type": "video" if media_type == "video" else "image", "payload": payload} + finally: + if downloaded and temp_path: + temp_path.unlink(missing_ok=True) + + def message_id_from_response(self, data: dict[str, Any]) -> str: + candidates = [ + data.get("id"), + data.get("mid"), + (data.get("message") or {}).get("id") if isinstance(data.get("message"), dict) else None, + (data.get("message") or {}).get("mid") if isinstance(data.get("message"), dict) else None, + ] + message = data.get("message") if isinstance(data.get("message"), dict) else {} + body = message.get("body") if isinstance(message.get("body"), dict) else {} + candidates.extend([body.get("id"), body.get("mid"), body.get("message_id")]) + return next((str(value) for value in candidates if value), "") + + def message_url_from_response(self, data: dict[str, Any]) -> str | None: + candidates = [data.get("url")] + message = data.get("message") if isinstance(data.get("message"), dict) else {} + body = message.get("body") if isinstance(message.get("body"), dict) else {} + candidates.extend([message.get("url"), body.get("url")]) + return next((str(value).strip() for value in candidates if str(value or "").strip()), None) + + async def send_message_waiting_for_media( + self, + client: MAXAPIClient, + text: str, + attachments: list[dict[str, Any]], + ) -> dict[str, Any]: + has_video = any(item.get("type") == "video" for item in attachments) + attempts = self.video_ready_attempts if has_video else 1 + delay = self.video_ready_delay_sec + for attempt in range(1, attempts + 1): + try: + return await client.send_message(self.chat_id, text, attachments=attachments or None) + except MAXAPIError as exc: + if exc.code != "attachment.not.ready" or attempt >= attempts: + raise + logger.warning( + "MAX attachment is not ready, retry send in {}s attempt={}/{}", + delay, + attempt + 1, + attempts, + ) + await asyncio.sleep(delay) + delay = min(delay * 2, self.retry_backoff_max_sec * 4) + raise RuntimeError("MAX attachment readiness retry exhausted") + + def video_tokens(self, attachments: list[dict[str, Any]]) -> list[str]: + tokens: list[str] = [] + for item in attachments: + if item.get("type") != "video": + continue + payload = item.get("payload") if isinstance(item.get("payload"), dict) else {} + token = str(payload.get("token") or "").strip() + if token: + tokens.append(token) + return tokens + + async def wait_for_videos(self, client: MAXAPIClient, attachments: list[dict[str, Any]]) -> None: + tokens = self.video_tokens(attachments) + if not tokens: + return + delay = self.video_ready_delay_sec + for attempt in range(1, self.video_ready_attempts + 1): + pending: list[str] = [] + for token in tokens: + info = await client.get_video_info(token) + if not info.get("urls"): + pending.append(token) + if not pending: + return + if attempt >= self.video_ready_attempts: + raise RuntimeError(f"MAX video is not ready after {attempt} attempts") + logger.warning( + "MAX video is not ready, retry info in {}s attempt={}/{} pending={}", + delay, + attempt + 1, + self.video_ready_attempts, + len(pending), + ) + await asyncio.sleep(delay) + delay = min(delay * 2, self.retry_backoff_max_sec * 4) + + async def send_post(self, post: dict[str, Any]) -> tuple[list[str], str | None, list[dict[str, Any]], str | None]: + text = self.build_text(post) + chunks = split_message_chunks(text, self.message_limit) + media_rows = await self.load_media(int(post["id"])) if self.media_max_items else [] + message_ids: list[str] = [] + attachments: list[dict[str, Any]] = [] + first_url: str | None = None + reaction_error: str | None = None + async with MAXAPIClient( + self.token, + self.api_base_url, + timeout_sec=max(30, await fetch_int_setting("max_poster_timeout_sec", 120)), + max_attempts=self.max_attempts, + retry_backoff_max_sec=self.retry_backoff_max_sec, + ) as client: + for item in media_rows: + try: + attachment = await self.upload_media(client, item) + if attachment: + attachments.append(attachment) + except Exception as exc: + logger.warning("MAX media skipped raw_post={} media_id={}: {}", post["id"], item.get("id"), exc) + if len(attachments) >= self.media_max_items: + break + if self.media_process_delay_sec and attachments: + await asyncio.sleep(self.media_process_delay_sec) + if self.dry_run: + return ["dry-run"], None, attachments, None + await self.wait_for_videos(client, attachments) + first_text = chunks[0] if chunks else "" + first = await self.send_message_waiting_for_media(client, first_text, attachments) + message = first.get("message") if isinstance(first.get("message"), dict) else first + first_url = self.message_url_from_response(message) if isinstance(message, dict) else None + first_message_id = self.message_id_from_response(first) + if first_message_id: + message_ids.append(first_message_id) + if not first_url: + try: + first_url = self.message_url_from_response(await client.get_message(first_message_id)) + except Exception as exc: + logger.warning("MAX message URL lookup failed message={}: {}", first_message_id, exc) + if self.auto_reaction_enabled and self.reaction_path_template: + try: + await client.react(self.reaction_path_template, first_message_id, self.auto_reaction) + except Exception as exc: + reaction_error = str(exc) + logger.warning("MAX auto reaction failed message={}: {}", first_message_id, exc) + for chunk in chunks[1:]: + if self.send_delay_sec: + await asyncio.sleep(self.send_delay_sec) + result = await client.send_message(self.chat_id, chunk) + message_id = self.message_id_from_response(result) + if message_id: + message_ids.append(message_id) + return message_ids, first_url, attachments, reaction_error + + async def upsert_publication( + self, + post_id: int, + status: str, + external_id: str | None = None, + url: str | None = None, + error: str | None = None, + attachments: list[dict[str, Any]] | None = None, + ) -> None: + await self.pool.execute( + """ + INSERT INTO post_publications( + raw_post_id, platform, status, target_id, external_id, url, error, attachments_json, published_at, updated_at + ) + VALUES($1, 'max', $2, $3, $4, $5, $6, $7::jsonb, CASE WHEN $2=$8 THEN NOW() ELSE NULL END, NOW()) + ON CONFLICT (raw_post_id, platform) DO UPDATE + SET status=EXCLUDED.status, + target_id=EXCLUDED.target_id, + external_id=EXCLUDED.external_id, + url=EXCLUDED.url, + error=EXCLUDED.error, + attachments_json=EXCLUDED.attachments_json, + published_at=CASE WHEN EXCLUDED.status=$8 THEN COALESCE(EXCLUDED.published_at, NOW()) ELSE post_publications.published_at END, + updated_at=NOW() + """, + post_id, + status, + str(self.chat_id), + external_id, + url, + error[:1000] if error else None, + json.dumps(attachments or [], ensure_ascii=False), + PUBLICATION_STATUS_PUBLISHED, + ) + + async def process_slot(self, slot: dict[str, Any]) -> int: + count = int(slot["count"]) + recent = await self.load_recent() + candidates = await self.load_candidates(max(50, count * 10)) + posts = self.choose_posts(candidates, recent, count) + published = 0 + for post in posts: + await self.heartbeat.beat(self.pool, status="publishing", meta={"post_id": int(post["id"]), "slot": slot["id"]}, force=True) + try: + message_ids, url, attachments, reaction_error = await self.send_post(post) + await self.upsert_publication( + int(post["id"]), + PUBLICATION_STATUS_PUBLISHED, + ",".join(message_ids), + url, + error=None, + attachments=attachments, + ) + published += 1 + logger.info("MAX published raw_post={} messages={} attachments={}", post["id"], message_ids, len(attachments)) + except Exception as exc: + await self.upsert_publication(int(post["id"]), PUBLICATION_STATUS_FAILED, error=str(exc)) + logger.exception("MAX publish failed raw_post={}: {}", post["id"], exc) + if self.send_delay_sec: + await asyncio.sleep(self.send_delay_sec) + await self.pool.execute( + """ + UPDATE publication_runs + SET published_count=$2, + status=$3, + error=$4, + updated_at=NOW() + WHERE id=$1 + """, + int(slot["run_id"]), + published, + "done" if published == count else "partial", + None if published == count else f"published {published} of {count}", + ) + return published + + async def run_once(self) -> bool: + await self.reload_settings() + enabled = await is_worker_enabled(self.pool, WORKER_MAX_POSTER) + app_enabled = await fetch_bool_setting("max_poster_enabled", False) + if not enabled or not app_enabled: + await self.heartbeat.beat(self.pool, status="disabled", force=True) + return False + slots = await self.due_slots() + if not slots: + await self.heartbeat.beat(self.pool, status="idle") + return False + total = 0 + for slot in slots: + total += await self.process_slot(slot) + await self.heartbeat.beat(self.pool, status="idle", meta={"published": total}, force=True) + return bool(total) + + async def run_loop(self) -> None: + await self.init() + logger.info("{} started pid={}", WORKER_MAX_POSTER, os.getpid()) + while True: + try: + await self.run_once() + except Exception as exc: + logger.exception("MAX poster loop error: {}", exc) + await asyncio.sleep(self.interval_sec) + + +async def main() -> None: + logger.remove() + logger.add(sys.stdout, level=settings.log_level) + worker = MAXPoster() + await worker.run_loop() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/vk_parser_app/workers/parser.py b/src/vk_parser_app/workers/parser.py new file mode 100644 index 0000000..c505c13 --- /dev/null +++ b/src/vk_parser_app/workers/parser.py @@ -0,0 +1,449 @@ +from __future__ import annotations + +import asyncio +import hashlib +import json +from datetime import datetime, timedelta, timezone + +from loguru import logger + +from ..config import settings +from ..constants import ( + JOB_TYPE_VK_STORAGE_COPY, + PLATFORM_VK, + POST_STATUS_SKIPPED, + POST_STATUS_STORAGE_PENDING, + SOURCE_STATUS_ERROR, + SOURCE_STATUS_OK, + WORKER_PARSER, +) +from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, get_pool +from ..heartbeat import HeartbeatReporter +from ..jobs import is_worker_enabled +from ..vk_api import ( + VKAPIClient, + VKAPIError, + extract_media, + is_deleted_or_invalid, + is_fatal_source_error, + is_repost, + post_vk_url, +) + + +def utc_from_ts(value: int) -> datetime: + return datetime.fromtimestamp(int(value), tz=timezone.utc) + + +def make_hash(*parts: str) -> str: + h = hashlib.sha256() + for part in parts: + h.update((part or "").strip().lower().encode("utf-8")) + h.update(b"\0") + return h.hexdigest() + + +class VKParserWorker: + def __init__(self) -> None: + self.pool = None + self.heartbeat = HeartbeatReporter(WORKER_PARSER, 30) + + async def init(self) -> None: + self.pool = await get_pool() + + async def active_sources(self) -> list[dict]: + rows = await self.pool.fetch( + """ + SELECT * + FROM sources + WHERE platform=$1 + AND active=TRUE + AND archived_at IS NULL + ORDER BY last_checked_at NULLS FIRST, priority ASC, id ASC + """, + PLATFORM_VK, + ) + return [dict(r) for r in rows] + + async def mark_source_error(self, source_id: int, message: str) -> None: + await self.pool.execute( + """ + UPDATE sources + SET status=$2, + status_msg=$3, + last_checked_at=NOW(), + updated_at=NOW() + WHERE id=$1 + """, + source_id, + SOURCE_STATUS_ERROR, + message[:1000], + ) + + async def deactivate_source(self, source_id: int, message: str) -> None: + await self.pool.execute( + """ + UPDATE sources + SET active=FALSE, + status=$2, + status_msg=$3, + last_checked_at=NOW(), + updated_at=NOW() + WHERE id=$1 + """, + source_id, + SOURCE_STATUS_ERROR, + message[:1000], + ) + + async def mark_source_ok(self, source_id: int, last_parsed_at: datetime | None) -> None: + await self.pool.execute( + """ + UPDATE sources + SET status=$2, + status_msg=NULL, + last_checked_at=NOW(), + last_parsed_at=COALESCE($3, last_parsed_at), + updated_at=NOW() + WHERE id=$1 + """, + source_id, + SOURCE_STATUS_OK, + last_parsed_at, + ) + + async def resolve_source_if_needed(self, client: VKAPIClient, source: dict) -> dict: + if source.get("external_owner_id"): + return source + external_id, owner_id, resolved_name = await client.resolve_group(source["url"] or source.get("external_id") or "") + await self.pool.execute( + """ + UPDATE sources + SET external_id=$2, + external_owner_id=$3, + name=CASE WHEN name='' OR name=url THEN $4 ELSE name END, + updated_at=NOW() + WHERE id=$1 + """, + int(source["id"]), + external_id, + owner_id, + resolved_name, + ) + source["external_id"] = external_id + source["external_owner_id"] = owner_id + if not source.get("name") or source.get("name") == source.get("url"): + source["name"] = resolved_name + return source + + async def known_post_ids(self, source_id: int, external_post_ids: list[str]) -> set[str]: + if not external_post_ids: + return set() + rows = await self.pool.fetch( + """ + SELECT external_post_id + FROM raw_posts + WHERE source_id=$1 AND external_post_id=ANY($2::text[]) + """, + source_id, + external_post_ids, + ) + return {str(r["external_post_id"]) for r in rows} + + async def known_content_hashes(self, hashes: list[str]) -> set[str]: + if not hashes: + return set() + rows = await self.pool.fetch( + """ + SELECT content_hash + FROM raw_posts + WHERE content_hash=ANY($1::text[]) + """, + hashes, + ) + return {str(r["content_hash"]) for r in rows} + + async def save_post( + self, + source: dict, + post: dict, + *, + status: str, + skip_reason: str | None = None, + create_storage_job: bool = True, + ) -> int | None: + source_id = int(source["id"]) + owner_id = int(post.get("owner_id") or source["external_owner_id"]) + external_post_id = str(post["id"]) + raw_text = str(post.get("text") or "").strip() + posted_at = utc_from_ts(int(post["date"])) + media = extract_media(post) + media_ids = ",".join(m.attachment_id for m in media) + text_hash = make_hash(raw_text) + content_hash = make_hash(raw_text, media_ids) + original_url = post_vk_url(owner_id, external_post_id) + + async with self.pool.acquire() as conn: + async with conn.transaction(): + raw_post_id = await conn.fetchval( + """ + INSERT INTO raw_posts( + source_id, platform, external_post_id, external_owner_id, + original_url, raw_text, raw_json, text_hash, content_hash, + posted_at, status, skip_reason + ) + VALUES($1,$2,$3,$4,$5,$6,$7::jsonb,$8,$9,$10,$11,$12) + ON CONFLICT (source_id, external_post_id) DO NOTHING + RETURNING id + """, + source_id, + PLATFORM_VK, + external_post_id, + owner_id, + original_url, + raw_text, + json.dumps(post, ensure_ascii=False), + text_hash, + content_hash, + posted_at, + status, + skip_reason, + ) + if raw_post_id is None: + return None + + for m in media: + await conn.execute( + """ + INSERT INTO raw_post_media( + raw_post_id, platform, media_type, original_url, original_attachment_id, + width, height, duration_sec, sort_order + ) + VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9) + """, + raw_post_id, + PLATFORM_VK, + m.media_type, + m.original_url, + m.attachment_id, + m.width, + m.height, + m.duration_sec, + m.sort_order, + ) + + if create_storage_job: + await conn.execute( + """ + INSERT INTO jobs(type, entity_type, entity_id, payload_json, status) + VALUES($1, 'raw_post', $2, '{}'::jsonb, 'pending') + ON CONFLICT DO NOTHING + """, + JOB_TYPE_VK_STORAGE_COPY, + raw_post_id, + ) + return int(raw_post_id) + + async def parse_source(self, client: VKAPIClient, source: dict) -> int: + source = await self.resolve_source_if_needed(client, source) + source_id = int(source["id"]) + owner_id = int(source["external_owner_id"]) + page_size = max(1, min(100, await fetch_int_setting("vk_wall_page_size", 50))) + lookback_days = max(1, await fetch_int_setting("parser_new_source_lookback_days", 14)) + overlap_minutes = max(0, await fetch_int_setting("parser_reparse_overlap_minutes", 120)) + + last_parsed_at = source.get("last_parsed_at") + parse_from = source.get("parse_from") + if last_parsed_at: + since_dt = last_parsed_at - timedelta(minutes=overlap_minutes) + elif parse_from: + since_dt = parse_from + else: + since_dt = datetime.now(timezone.utc) - timedelta(days=lookback_days) + if since_dt.tzinfo is None: + since_dt = since_dt.replace(tzinfo=timezone.utc) + since_ts = int(since_dt.timestamp()) + + fetched: list[dict] = [] + offset = 0 + stop = False + while not stop: + response = await client.get_wall_posts(owner_id=owner_id, count=page_size, offset=offset) + items = response.get("items", []) or [] + if not items: + break + for post in items: + if is_deleted_or_invalid(post): + continue + if int(post["date"]) <= since_ts: + stop = True + break + fetched.append(post) + if len(items) < page_size: + break + offset += page_size + + known = await self.known_post_ids(source_id, [str(p["id"]) for p in fetched if p.get("id")]) + candidates = [p for p in fetched if str(p.get("id")) not in known] + saved = 0 + max_seen: datetime | None = None + min_text_length = max(0, await fetch_int_setting("parser_min_text_length", 0)) + skip_reposts = await fetch_bool_setting("parser_skip_reposts", True) + skip_empty_text = await fetch_bool_setting("parser_skip_empty_text", True) + skip_no_media = await fetch_bool_setting("parser_skip_no_media", True) + skip_short_text = await fetch_bool_setting("parser_skip_text_too_short", True) + store_skipped = await fetch_bool_setting("parser_store_skipped_posts", False) + dedupe_content_hash = await fetch_bool_setting("parser_dedupe_content_hash", True) + + hash_by_post_id: dict[str, str] = {} + if dedupe_content_hash: + candidate_hashes: list[str] = [] + for post in candidates: + if is_deleted_or_invalid(post) or (skip_reposts and is_repost(post)): + continue + media = extract_media(post) + media_ids = ",".join(m.attachment_id for m in media) + content_hash = make_hash(str(post.get("text") or "").strip(), media_ids) + hash_by_post_id[str(post["id"])] = content_hash + candidate_hashes.append(content_hash) + known_hashes = await self.known_content_hashes(candidate_hashes) + else: + known_hashes = set() + + for post in candidates: + if is_deleted_or_invalid(post): + continue + text = str(post.get("text") or "").strip() + posted_at = utc_from_ts(int(post["date"])) + if max_seen is None or posted_at > max_seen: + max_seen = posted_at + + if skip_reposts and is_repost(post): + continue + + media = extract_media(post) + content_hash = hash_by_post_id.get(str(post["id"])) + if dedupe_content_hash and content_hash in known_hashes: + continue + + skip_reason: str | None = None + if skip_empty_text and not text: + skip_reason = "empty_text" + elif skip_no_media and not media: + skip_reason = "no_media" + elif skip_short_text and len(text) < min_text_length: + skip_reason = "text_too_short" + + if skip_reason: + if not store_skipped: + continue + raw_id = await self.save_post( + source, + post, + status=POST_STATUS_SKIPPED, + skip_reason=skip_reason, + create_storage_job=False, + ) + else: + raw_id = await self.save_post( + source, + post, + status=POST_STATUS_STORAGE_PENDING, + skip_reason=None, + create_storage_job=True, + ) + if raw_id: + saved += 1 + if content_hash: + known_hashes.add(content_hash) + + await self.mark_source_ok(source_id, max_seen or last_parsed_at) + logger.info( + "Parsed source {}: fetched={} known={} candidates={} saved={}", + source.get("name"), + len(fetched), + len(known), + len(candidates), + saved, + ) + return saved + + async def run_once(self) -> None: + enabled = await is_worker_enabled(self.pool, WORKER_PARSER) + if not enabled: + await self.heartbeat.beat(self.pool, status="disabled", force=True) + return + + rps = await fetch_int_setting("vk_requests_per_second", 3) + timeout_total = await fetch_int_setting("vk_api_timeout_total_sec", 15) + timeout_connect = await fetch_int_setting("vk_api_timeout_connect_sec", 5) + rate_limit_sleep = await fetch_float_setting("vk_rate_limit_sleep_sec", 1.0) + retry_attempts = await fetch_int_setting("vk_api_retry_attempts", 3) + retry_min_delay = await fetch_float_setting("vk_api_retry_min_delay_sec", 2.0) + retry_max_delay = await fetch_float_setting("vk_api_retry_max_delay_sec", 10.0) + source_pause = max(0.0, await fetch_float_setting("parser_source_pause_sec", 0.0)) + sources = await self.active_sources() + await self.heartbeat.beat(self.pool, meta={"sources": len(sources)}) + if not sources: + logger.info("No active VK sources") + return + + logger.info( + "Parser cycle: sources={} rps={} timeout={}/{} retry={}/{}..{} rate_limit_sleep={} source_pause={}", + len(sources), + rps, + timeout_total, + timeout_connect, + retry_attempts, + retry_min_delay, + retry_max_delay, + rate_limit_sleep, + source_pause, + ) + + async with VKAPIClient( + rps=rps, + timeout_total_sec=timeout_total, + timeout_connect_sec=timeout_connect, + rate_limit_sleep_sec=rate_limit_sleep, + retry_attempts=retry_attempts, + retry_min_delay_sec=retry_min_delay, + retry_max_delay_sec=retry_max_delay, + ) as client: + for source in sources: + try: + await self.parse_source(client, source) + except VKAPIError as e: + if is_fatal_source_error(e): + await self.deactivate_source(int(source["id"]), str(e)) + logger.warning("VK source deactivated {}: {}", source.get("name"), e) + else: + await self.mark_source_error(int(source["id"]), str(e)) + logger.warning("VK source temporary error {}: {}", source.get("name"), e) + except Exception as e: + await self.mark_source_error(int(source["id"]), str(e)) + logger.exception("Unexpected source error {}: {}", source.get("name"), e) + if source_pause: + await asyncio.sleep(source_pause) + + async def run_loop(self) -> None: + await self.init() + logger.info("{} started", WORKER_PARSER) + while True: + try: + await self.run_once() + except Exception as e: + logger.exception("Parser loop error: {}", e) + interval = max(10, await fetch_int_setting("parser_interval_sec", 300)) + await asyncio.sleep(interval) + + +async def main() -> None: + logger.remove() + logger.add(lambda msg: print(msg, end=""), level=settings.log_level) + worker = VKParserWorker() + await worker.run_loop() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/vk_parser_app/workers/site_poster.py b/src/vk_parser_app/workers/site_poster.py new file mode 100644 index 0000000..47f8500 --- /dev/null +++ b/src/vk_parser_app/workers/site_poster.py @@ -0,0 +1,166 @@ +from __future__ import annotations + +import asyncio +import os +import sys +from pathlib import Path + +from loguru import logger + +from ..config import settings +from ..constants import WORKER_SITE_POSTER +from ..db import fetch_bool_setting, fetch_int_setting, fetch_setting, get_pool +from ..heartbeat import HeartbeatReporter +from ..jobs import is_worker_enabled + + +class SitePoster: + def __init__(self) -> None: + self.pool = None + self.heartbeat = HeartbeatReporter(WORKER_SITE_POSTER, 30) + self.project_root = Path(__file__).resolve().parents[3] + self.interval_sec = 300 + self.photo_batch_size = 10 + self.video_batch_size = 1 + self.provider = "" + self.base_url = "" + self.secret_file = "" + self.fallback_title = "Новость" + + async def init(self) -> None: + self.pool = await get_pool() + + async def reload_settings(self) -> None: + self.interval_sec = max(30, await fetch_int_setting("site_poster_interval_sec", 300)) + self.photo_batch_size = max(0, await fetch_int_setting("site_poster_photo_batch_size", 10)) + self.video_batch_size = max(0, await fetch_int_setting("site_poster_video_batch_size", 1)) + legacy_ghost_url = str(await fetch_setting("site_poster_ghost_url", "") or "").strip() + legacy_key_file = str(await fetch_setting("site_poster_key_file", "") or "").strip() + self.provider = str(await fetch_setting("site_poster_provider", "") or "").strip().lower() + if not self.provider and legacy_ghost_url: + self.provider = "ghost" + self.base_url = str( + await fetch_setting("site_poster_base_url", "") + or legacy_ghost_url + or "" + ).rstrip("/") + self.secret_file = str( + await fetch_setting("site_poster_secret_file", "") + or legacy_key_file + or "" + ).strip() + self.fallback_title = str(await fetch_setting("site_poster_fallback_title", "Новость") or "Новость").strip() + + def build_publish_command(self, flag: str, limit: int) -> list[str]: + if self.provider != "ghost": + raise RuntimeError(f"unsupported site_poster_provider: {self.provider or ''}") + if not self.base_url: + raise RuntimeError("site_poster_base_url is empty") + if not self.secret_file and not os.environ.get("SITE_POSTER_GHOST_ADMIN_KEY", "").strip(): + raise RuntimeError("site_poster_secret_file and SITE_POSTER_GHOST_ADMIN_KEY are empty") + return [ + sys.executable, + str(self.project_root / "scripts" / "publish_site_batch.py"), + flag, + "--limit", + str(limit), + "--ghost-url", + self.base_url, + "--key-file", + self.secret_file, + "--fallback-title", + self.fallback_title or "Новость", + ] + + async def publish_mode(self, flag: str, limit: int) -> tuple[int, int]: + if limit < 1: + return 0, 0 + command = self.build_publish_command(flag, limit) + env = os.environ.copy() + env["PYTHONPATH"] = str(self.project_root / "src") + env["PYTHONUNBUFFERED"] = "1" + process = await asyncio.create_subprocess_exec( + *command, + cwd=str(self.project_root), + env=env, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + ) + published = 0 + failed = 0 + assert process.stdout is not None + while True: + try: + raw_line = await asyncio.wait_for(process.stdout.readline(), timeout=20) + except TimeoutError: + await self.heartbeat.beat( + self.pool, + status="publishing", + meta={"mode": flag, "published": published, "failed": failed}, + force=True, + ) + continue + if not raw_line: + break + line = raw_line.decode("utf-8", errors="replace").strip() + if not line: + continue + if line.startswith("PUBLISHED "): + published += 1 + logger.info("Site poster: {}", line) + elif line.startswith("FAILED "): + failed += 1 + logger.error("Site poster: {}", line) + elif line not in {"NO_READY_POSTS", "ALREADY_RUNNING"}: + logger.debug("Site poster child: {}", line) + await self.heartbeat.beat( + self.pool, + status="publishing", + meta={"mode": flag, "published": published, "failed": failed}, + ) + return_code = await process.wait() + if return_code != 0: + raise RuntimeError(f"site publisher exited with code {return_code} for {flag}") + return published, failed + + async def run_once(self) -> bool: + await self.reload_settings() + worker_enabled = await is_worker_enabled(self.pool, WORKER_SITE_POSTER) + app_enabled = await fetch_bool_setting("site_poster_enabled", False) + if not worker_enabled or not app_enabled: + await self.heartbeat.beat(self.pool, status="disabled", force=True) + return False + await self.heartbeat.beat(self.pool, status="checking", force=True) + photo_published, photo_failed = await self.publish_mode("--all-ready", self.photo_batch_size) + video_published, video_failed = await self.publish_mode("--all-ready-videos", self.video_batch_size) + published = photo_published + video_published + failed = photo_failed + video_failed + await self.heartbeat.beat( + self.pool, + status="idle" if not failed else "error", + meta={"published": published, "failed": failed}, + force=True, + ) + return bool(published) + + async def run_loop(self) -> None: + await self.init() + logger.info("{} started pid={}", WORKER_SITE_POSTER, os.getpid()) + while True: + try: + await self.run_once() + except Exception as exc: + logger.exception("Site poster loop error: {}", exc) + await self.heartbeat.beat(self.pool, status="error", meta={"error": str(exc)[:500]}, force=True) + await asyncio.sleep(self.interval_sec) + + +async def main() -> None: + logger.remove() + logger.add(sys.stdout, level=settings.log_level) + worker = SitePoster() + await worker.run_loop() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/vk_parser_app/workers/tg_poster.py b/src/vk_parser_app/workers/tg_poster.py new file mode 100644 index 0000000..985c091 --- /dev/null +++ b/src/vk_parser_app/workers/tg_poster.py @@ -0,0 +1,637 @@ +from __future__ import annotations + +import asyncio +import aiohttp +import json +import os +import re +import sys +from datetime import datetime, time, timedelta +from pathlib import Path +from typing import Any +from zoneinfo import ZoneInfo + +from aiogram import Bot +from aiogram.exceptions import TelegramRetryAfter +from aiogram.types import FSInputFile, InputMediaPhoto, InputMediaVideo +from loguru import logger + +from ..config import settings +from ..constants import ( + PUBLICATION_STATUS_FAILED, + PUBLICATION_STATUS_PENDING, + PUBLICATION_STATUS_PUBLISHED, + WORKER_TG_POSTER, +) +from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool +from ..heartbeat import HeartbeatReporter +from ..jobs import is_worker_enabled +from ..text_utils import build_publication_text + +LOCAL_TZ = ZoneInfo("Asia/Yekaterinburg") +MAX_MEDIA_GROUP = 10 +MAX_RICH_MEDIA = 50 +MAX_RICH_TEXT = 32768 +PROJECT_ROOT = Path(__file__).resolve().parents[3] +UPLOAD_ROOT = PROJECT_ROOT / "uploads" + + +def parse_topic(value: str) -> tuple[int, int | None]: + raw = str(value or "").strip() + if not raw: + raise RuntimeError("tg_poster_chat_id is empty") + if ":" in raw: + chat_id, thread_id = raw.split(":", 1) + return int(chat_id), int(thread_id) + return int(raw), None + + +def valid_time(value: str) -> bool: + return bool(re.fullmatch(r"(?:[01]\d|2[0-3]):[0-5]\d", str(value or "").strip())) + + +def split_message_chunks(text: str, limit: int) -> list[str]: + text = str(text or "").strip() + if not text: + return [] + chunks: list[str] = [] + while len(text) > limit: + split_at = text.rfind("\n", 0, limit) + if split_at < limit // 2: + split_at = text.rfind(" ", 0, limit) + if split_at < limit // 2: + split_at = limit + chunks.append(text[:split_at].strip()) + text = text[split_at:].strip() + if text: + chunks.append(text) + return chunks + + +def tg_message_url(chat_id: int, message_id: int) -> str: + chat = str(abs(int(chat_id))) + if chat.startswith("100"): + chat = chat[3:] + return f"https://t.me/c/{chat}/{message_id}" + + +def local_upload_path(url: str) -> Path | None: + raw = str(url or "").strip() + prefix = "/uploads/" + if not raw.startswith(prefix): + return None + path = (UPLOAD_ROOT / raw.removeprefix(prefix)).resolve() + try: + path.relative_to(UPLOAD_ROOT.resolve()) + except ValueError: + return None + return path if path.is_file() else None + + +class RichMessageUnavailable(RuntimeError): + pass + + +class TelegramPoster: + def __init__(self) -> None: + self.pool = None + self.bot: Bot | None = None + self.bot_token = "" + self.chat_id: int | None = None + self.thread_id: int | None = None + self.heartbeat = HeartbeatReporter(WORKER_TG_POSTER, 30) + self.schedule: list[dict[str, Any]] = [] + self.media_group_max_items = MAX_MEDIA_GROUP + self.caption_limit = 1024 + self.message_limit = 4096 + self.max_attempts = 3 + self.retry_backoff_max_sec = 30 + self.send_delay_sec = 1.0 + self.overflow_caption = "⬇️ Описание" + self.recent_window = 20 + self.category_repeat_penalty = 3.0 + self.source_repeat_penalty = 4.0 + + async def init(self) -> None: + self.pool = await get_pool() + await self.reload_settings() + + async def reload_settings(self) -> None: + token = str(await fetch_setting("tg_poster_bot_token", "") or "").strip() or settings.tg_bot_token + if not token: + raise RuntimeError("tg_poster_bot_token and TG_BOT_TOKEN are empty") + if token != self.bot_token: + if self.bot: + await self.bot.session.close() + self.bot = Bot(token=token) + self.bot_token = token + self.chat_id, self.thread_id = parse_topic(str(await fetch_setting("tg_poster_chat_id", "") or "")) + self.schedule = self.normalize_schedule(await fetch_setting("tg_poster_schedule_json", [])) + self.media_group_max_items = max(1, min(MAX_MEDIA_GROUP, await fetch_int_setting("tg_poster_media_group_max_items", 10))) + self.caption_limit = max(128, min(1024, await fetch_int_setting("tg_poster_caption_limit", 1024))) + self.message_limit = max(512, min(4096, await fetch_int_setting("tg_poster_message_limit", 4096))) + self.max_attempts = max(1, await fetch_int_setting("tg_poster_max_attempts", 3)) + self.retry_backoff_max_sec = max(1, await fetch_int_setting("tg_poster_retry_backoff_max_sec", 30)) + self.send_delay_sec = max(0.0, await fetch_float_setting("tg_poster_send_delay_sec", 1.0)) + self.overflow_caption = str(await fetch_setting("tg_poster_text_overflow_caption", self.overflow_caption) or self.overflow_caption) + self.recent_window = max(1, await fetch_int_setting("tg_poster_recent_window", 20)) + self.category_repeat_penalty = max(0.0, await fetch_float_setting("tg_poster_category_repeat_penalty", 3.0)) + self.source_repeat_penalty = max(0.0, await fetch_float_setting("tg_poster_source_repeat_penalty", 4.0)) + logger.info("TG poster config: chat={} schedule={} caption_limit={}", self.chat_id, self.schedule, self.caption_limit) + + async def close(self) -> None: + if self.bot: + await self.bot.session.close() + + def normalize_schedule(self, value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list): + value = [] + rows: list[dict[str, Any]] = [] + for idx, item in enumerate(value): + if not isinstance(item, dict): + continue + time_value = str(item.get("time") or "").strip() + if not valid_time(time_value): + continue + count = max(0, min(20, int(item.get("count") or 0))) + if count < 1: + continue + rows.append( + { + "id": str(item.get("id") or f"slot-{idx + 1}").strip()[:80], + "time": time_value, + "count": count, + "enabled": bool(item.get("enabled", True)), + } + ) + return sorted(rows, key=lambda row: row["time"]) + + def chat_kwargs(self) -> dict[str, Any]: + kwargs: dict[str, Any] = {"chat_id": int(self.chat_id), "parse_mode": "HTML"} + if self.thread_id: + kwargs["message_thread_id"] = int(self.thread_id) + return kwargs + + async def tg_retry(self, fn): + for attempt in range(1, self.max_attempts + 1): + try: + return await fn() + except TelegramRetryAfter as exc: + delay = float(exc.retry_after) + 0.5 + logger.warning("Telegram flood control, sleep {}s", delay) + await asyncio.sleep(delay) + except Exception as exc: + logger.warning("Telegram post error without retry to avoid duplicate send: {}", exc) + raise + raise RuntimeError("telegram retry exhausted") + + async def due_slots(self) -> list[dict[str, Any]]: + now = datetime.now(LOCAL_TZ) + today = now.date() + due: list[dict[str, Any]] = [] + for row in self.schedule: + if not row.get("enabled"): + continue + hour, minute = [int(part) for part in str(row["time"]).split(":", 1)] + slot_start = datetime.combine(today, time(hour=hour, minute=minute), tzinfo=LOCAL_TZ) + slot_end = slot_start + timedelta(minutes=10) + if not (slot_start <= now < slot_end): + continue + inserted = await self.pool.fetchrow( + """ + INSERT INTO publication_runs(poster, schedule_id, scheduled_for, scheduled_time, planned_count, status) + VALUES($1, $2, $3, $4, $5, 'started') + ON CONFLICT (poster, schedule_id, scheduled_for) DO NOTHING + RETURNING id + """, + "tg", + row["id"], + today, + row["time"], + int(row["count"]), + ) + if inserted: + due.append({**row, "run_id": int(inserted["id"]), "date": today}) + return due + + async def load_recent(self) -> list[dict[str, Any]]: + rows = await self.pool.fetch( + """ + SELECT final_category_tag, rewrite_category_tag, final_source_tag, rewrite_source_tag + FROM raw_posts + WHERE publication_status=$1 + AND tg_publication_chat_id=$2 + ORDER BY published_at DESC NULLS LAST, id DESC + LIMIT $3::int + """, + PUBLICATION_STATUS_PUBLISHED, + self.chat_id, + self.recent_window, + ) + return [dict(row) for row in rows] + + async def load_candidates(self, limit: int) -> list[dict[str, Any]]: + rows = await self.pool.fetch( + """ + SELECT rp.*, s.name AS source_name, s.tag AS source_tag + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + WHERE rp.status='storage_ready' + AND rp.rewrite_status='ready' + AND rp.editorial_status='accepted' + AND COALESCE(rp.publication_status, 'pending') = $1 + AND EXISTS ( + SELECT 1 + FROM raw_post_media rpm + WHERE rpm.raw_post_id=rp.id + AND COALESCE(rpm.editor_hidden, FALSE)=FALSE + AND rpm.media_type IN ('photo', 'video') + AND COALESCE(rpm.tg_file_id, rpm.storage_attachment_id, '') <> '' + ) + AND NOT EXISTS ( + SELECT 1 + FROM raw_post_media rpm + WHERE rpm.raw_post_id=rp.id + AND COALESCE(rpm.editor_hidden, FALSE)=FALSE + AND rpm.media_type IN ('photo', 'video') + AND COALESCE(rpm.tg_file_id, rpm.storage_attachment_id, '') = '' + ) + ORDER BY COALESCE(rp.reviewed_at, rp.edited_at, rp.rewritten_at, rp.created_at) ASC, + rp.id ASC + LIMIT $2::int + """, + PUBLICATION_STATUS_PENDING, + max(limit, 50), + ) + return [dict(row) for row in rows] + + def choose_posts(self, candidates: list[dict[str, Any]], recent: list[dict[str, Any]], count: int) -> list[dict[str, Any]]: + selected: list[dict[str, Any]] = [] + recent_categories = [ + str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "") + for row in recent + ] + recent_sources = [ + str(row.get("final_source_tag") or row.get("rewrite_source_tag") or "") + for row in recent + ] + pool = list(candidates) + while pool and len(selected) < count: + best_idx = 0 + selected_categories = [str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "") for row in selected] + selected_sources = [str(row.get("final_source_tag") or row.get("rewrite_source_tag") or row.get("source_tag") or "") for row in selected] + previous_category = selected_categories[-1] if selected_categories else (recent_categories[0] if recent_categories else "") + previous_source = selected_sources[-1] if selected_sources else (recent_sources[0] if recent_sources else "") + best_rank: tuple[int, float, int] | None = None + for idx, post in enumerate(pool): + category = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or "") + source = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "") + immediate_repeats = int(bool(category) and category == previous_category) + int(bool(source) and source == previous_source) + repeat_penalty = ( + (recent_categories.count(category) + selected_categories.count(category)) * self.category_repeat_penalty + + (recent_sources.count(source) + selected_sources.count(source)) * self.source_repeat_penalty + ) + rank = (immediate_repeats, repeat_penalty, idx) + if best_rank is None or rank < best_rank: + best_idx = idx + best_rank = rank + selected.append(pool.pop(best_idx)) + return selected + + async def load_media(self, raw_post_id: int) -> list[dict[str, Any]]: + rows = await self.pool.fetch( + """ + SELECT * + FROM raw_post_media + WHERE raw_post_id=$1 + AND COALESCE(editor_hidden, FALSE)=FALSE + AND media_type IN ('photo', 'video') + ORDER BY sort_order ASC, id ASC + """, + raw_post_id, + ) + return [dict(row) for row in rows] + + def media_value(self, item: dict[str, Any]) -> Any | None: + stored = item.get("tg_file_id") or item.get("storage_attachment_id") + if stored: + return str(stored) + url = str(item.get("original_url") or "").strip() + path = local_upload_path(url) + if path: + return FSInputFile(path) + return None + + def build_text(self, post: dict[str, Any]) -> str: + category_tag = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or post.get("final_category") or post.get("rewrite_category") or "") + source_tag = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "") + return build_publication_text(post.get("final_text") or post.get("rewritten_text") or "", category_tag, source_tag, format_title=True, parse_mode="html") + + async def send_text(self, text: str) -> list[int]: + message_ids: list[int] = [] + for chunk in split_message_chunks(text, self.message_limit): + message = await self.tg_retry(lambda text=chunk: self.bot.send_message(text=text, **self.chat_kwargs())) + message_ids.append(int(message.message_id)) + if self.send_delay_sec: + await asyncio.sleep(self.send_delay_sec) + return message_ids + + async def send_media_items(self, items: list[dict[str, Any]], caption: str | None = None) -> list[int]: + if not items: + return [] + if len(items) == 1: + item = items[0] + if item["type"] == "photo": + message = await self.tg_retry( + lambda: self.bot.send_photo(photo=item["media"], caption=caption, **self.chat_kwargs()) + ) + else: + message = await self.tg_retry( + lambda: self.bot.send_video(video=item["media"], caption=caption, **self.chat_kwargs()) + ) + return [int(message.message_id)] + group = [] + for idx, item in enumerate(items): + media_caption = caption if idx == 0 else None + if item["type"] == "photo": + group.append(InputMediaPhoto(media=item["media"], caption=media_caption)) + else: + group.append(InputMediaVideo(media=item["media"], caption=media_caption)) + messages = await self.tg_retry(lambda: self.bot.send_media_group(media=group, **self.chat_kwargs())) + return [int(message.message_id) for message in messages] + + def rich_media_value(self, item: dict[str, Any]) -> str | None: + stored = str(item.get("tg_file_id") or item.get("storage_attachment_id") or "").strip() + return stored or None + + def rich_text_html(self, text: str) -> str: + paragraphs = [] + for paragraph in str(text or "").strip().split("\n\n"): + body = "
".join(line for line in paragraph.splitlines() if line.strip()) + if body: + paragraphs.append(f"

{body}

") + return "\n".join(paragraphs) + + def build_rich_message(self, text: str, media_rows: list[dict[str, Any]]) -> dict[str, Any] | None: + if len(media_rows) > MAX_RICH_MEDIA: + return None + + rich_media = [] + media_tags = [] + for idx, item in enumerate(media_rows[:MAX_RICH_MEDIA]): + media_type = str(item.get("media_type") or "") + if media_type not in ("photo", "video"): + continue + media = self.rich_media_value(item) + if not media: + return None + media_id = f"m{idx}" + input_type = "photo" if media_type == "photo" else "video" + rich_media.append({"id": media_id, "media": {"type": input_type, "media": media}}) + if media_type == "photo": + media_tags.append(f'') + else: + media_tags.append(f'') + if not rich_media: + return None + + rich_text = self.rich_text_html(text) + if len(rich_text) > MAX_RICH_TEXT: + return None + + media_html = media_tags[0] if len(media_tags) == 1 else f"{''.join(media_tags)}" + return { + "html": f"{media_html}\n{rich_text}" if rich_text else media_html, + "media": rich_media, + } + + async def send_rich_message(self, rich_message: dict[str, Any]) -> list[int]: + data: dict[str, Any] = { + "chat_id": int(self.chat_id), + "rich_message": rich_message, + } + if self.thread_id: + data["message_thread_id"] = int(self.thread_id) + + url = f"https://api.telegram.org/bot{self.bot_token}/sendRichMessage" + timeout = aiohttp.ClientTimeout(total=90) + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.post(url, json=data) as response: + payload = await response.json(content_type=None) + + if payload.get("ok"): + message_id = payload.get("result", {}).get("message_id") + if message_id: + return [int(message_id)] + raise RichMessageUnavailable("sendRichMessage returned no message_id") + + description = str(payload.get("description") or f"HTTP {response.status}") + if "Too Many Requests" in description and isinstance(payload.get("parameters"), dict): + retry_after = float(payload["parameters"].get("retry_after") or 0) + if retry_after > 0: + logger.warning("Telegram rich message flood control, sleep {}s", retry_after) + await asyncio.sleep(retry_after + 0.5) + raise RichMessageUnavailable(description) + + async def send_legacy_media_post(self, text: str, media_rows: list[dict[str, Any]]) -> list[int]: + media_items = [] + for item in media_rows: + media = self.media_value(item) + if media is None: + continue + media_type = str(item.get("media_type") or "") + if media_type == "photo": + media_items.append({"type": "photo", "media": media}) + elif media_type == "video": + media_items.append({"type": "video", "media": media}) + + if not media_items: + return await self.send_text(text) + + if len(text) <= self.caption_limit: + first_caption = text + text_chunks: list[str] = [] + else: + first_caption = self.overflow_caption[: self.caption_limit] + text_chunks = split_message_chunks(text, self.message_limit) + + first = media_items[: self.media_group_max_items] + message_ids = [] + message_ids.extend(await self.send_media_items(first, first_caption)) + if self.send_delay_sec: + await asyncio.sleep(self.send_delay_sec) + + rest = media_items[self.media_group_max_items :] + for start in range(0, len(rest), self.media_group_max_items): + chunk = rest[start : start + self.media_group_max_items] + message_ids.extend(await self.send_media_items(chunk)) + if self.send_delay_sec: + await asyncio.sleep(self.send_delay_sec) + + for chunk in text_chunks: + message_ids.extend(await self.send_text(chunk)) + + return message_ids + + async def send_post(self, post: dict[str, Any]) -> list[int]: + text = self.build_text(post) + media_rows = await self.load_media(int(post["id"])) + + if not media_rows: + return await self.send_text(text) + + rich_message = self.build_rich_message(text, media_rows) + if rich_message: + try: + return await self.send_rich_message(rich_message) + except RichMessageUnavailable as exc: + logger.warning("Telegram rich message failed, fallback to legacy media post: {}", exc) + + return await self.send_legacy_media_post(text, media_rows) + + async def mark_published(self, post_id: int, message_ids: list[int]) -> None: + async with self.pool.acquire() as conn: + async with conn.transaction(): + await conn.execute( + """ + UPDATE raw_posts + SET publication_status=$2, + editorial_status='published', + publication_error=NULL, + published_at=NOW(), + tg_publication_chat_id=$3, + tg_publication_thread_id=$4, + tg_publication_message_ids=$5, + tg_publication_url=$6, + updated_at=NOW() + WHERE id=$1 + """, + post_id, + PUBLICATION_STATUS_PUBLISHED, + self.chat_id, + self.thread_id, + message_ids, + tg_message_url(int(self.chat_id), message_ids[0]) if message_ids else None, + ) + await conn.execute( + """ + INSERT INTO post_publications(raw_post_id, platform, status, target_id, external_id, url, attachments_json, published_at, updated_at) + VALUES($1, 'tg', $2, $3, $4, $5, $6::jsonb, NOW(), NOW()) + ON CONFLICT (raw_post_id, platform) DO UPDATE + SET status=EXCLUDED.status, + target_id=EXCLUDED.target_id, + external_id=EXCLUDED.external_id, + url=EXCLUDED.url, + error=NULL, + attachments_json=EXCLUDED.attachments_json, + published_at=NOW(), + updated_at=NOW() + """, + post_id, + PUBLICATION_STATUS_PUBLISHED, + str(self.chat_id), + ",".join(str(mid) for mid in message_ids), + tg_message_url(int(self.chat_id), message_ids[0]) if message_ids else None, + json.dumps(message_ids, ensure_ascii=False), + ) + + async def mark_failed(self, post_id: int, error: str) -> None: + async with self.pool.acquire() as conn: + async with conn.transaction(): + await conn.execute( + """ + UPDATE raw_posts + SET publication_status=$2, + editorial_status='publish_failed', + publication_error=$3, + updated_at=NOW() + WHERE id=$1 + """, + post_id, + PUBLICATION_STATUS_FAILED, + error[:1000], + ) + await conn.execute( + """ + INSERT INTO post_publications(raw_post_id, platform, status, target_id, error, updated_at) + VALUES($1, 'tg', $2, $3, $4, NOW()) + ON CONFLICT (raw_post_id, platform) DO UPDATE + SET status=EXCLUDED.status, + target_id=EXCLUDED.target_id, + error=EXCLUDED.error, + updated_at=NOW() + """, + post_id, + PUBLICATION_STATUS_FAILED, + str(self.chat_id), + error[:1000], + ) + async def process_slot(self, slot: dict[str, Any]) -> int: + count = int(slot["count"]) + recent = await self.load_recent() + candidates = await self.load_candidates(max(50, count * 10)) + posts = self.choose_posts(candidates, recent, count) + published = 0 + for post in posts: + await self.heartbeat.beat(self.pool, status="publishing", meta={"post_id": int(post["id"]), "slot": slot["id"]}, force=True) + try: + message_ids = await self.send_post(post) + await self.mark_published(int(post["id"]), message_ids) + published += 1 + logger.info("TG published raw_post={} messages={}", post["id"], message_ids) + except Exception as exc: + await self.mark_failed(int(post["id"]), str(exc)) + logger.exception("TG publish failed raw_post={}: {}", post["id"], exc) + await self.pool.execute( + """ + UPDATE publication_runs + SET published_count=$2, + status=$3, + error=$4, + updated_at=NOW() + WHERE id=$1 + """, + int(slot["run_id"]), + published, + "done" if published == count else "partial", + None if published == count else f"published {published} of {count}", + ) + return published + + async def run_once(self) -> bool: + await self.reload_settings() + enabled = await is_worker_enabled(self.pool, WORKER_TG_POSTER) + app_enabled = await fetch_bool_setting("tg_poster_enabled", False) + if not enabled or not app_enabled: + await self.heartbeat.beat(self.pool, status="disabled", force=True) + return False + slots = await self.due_slots() + if not slots: + await self.heartbeat.beat(self.pool, status="idle") + return False + total = 0 + for slot in slots: + total += await self.process_slot(slot) + await self.heartbeat.beat(self.pool, status="idle", meta={"published": total}, force=True) + return bool(total) + + async def run_loop(self) -> None: + await self.init() + logger.info("{} started pid={}", WORKER_TG_POSTER, os.getpid()) + try: + while True: + await self.run_once() + await asyncio.sleep(max(10, await fetch_int_setting("tg_poster_interval_sec", 60))) + finally: + await self.close() + + +async def main() -> None: + logger.remove() + logger.add(sys.stdout, level=settings.log_level) + worker = TelegramPoster() + await worker.run_loop() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/vk_parser_app/workers/tg_reactor.py b/src/vk_parser_app/workers/tg_reactor.py new file mode 100644 index 0000000..7ff094e --- /dev/null +++ b/src/vk_parser_app/workers/tg_reactor.py @@ -0,0 +1,326 @@ +from __future__ import annotations + +import asyncio +import json +import os +import re +import sys +from typing import Any + +import aiohttp +from loguru import logger + +from ..config import settings +from ..constants import PUBLICATION_STATUS_PUBLISHED, WORKER_TG_REACTOR +from ..db import fetch_bool_setting, fetch_int_setting, fetch_setting, get_pool +from ..heartbeat import HeartbeatReporter +from ..jobs import is_worker_enabled + +PLATFORM_TG = "tg" +REACTION_STATUS_PENDING = "pending" +REACTION_STATUS_IN_PROGRESS = "in_progress" +REACTION_STATUS_REACTED = "reacted" +REACTION_STATUS_FAILED = "failed" + + +def split_tokens(value: Any) -> list[str]: + if isinstance(value, list): + candidates = [str(item or "") for item in value] + else: + candidates = re.split(r"[\s,;]+", str(value or "")) + return [token.strip() for token in candidates if token.strip()] + + +def normalize_reactions(value: Any) -> list[str]: + if not isinstance(value, list): + value = [] + reactions = [] + for item in value: + emoji = str(item or "").strip() + if emoji: + reactions.append(emoji) + return reactions or ["👍"] + + +def parse_message_ids(row: dict[str, Any]) -> list[str]: + raw = row.get("attachments_json") + values: list[Any] = [] + if isinstance(raw, list): + values = raw + elif isinstance(raw, str): + try: + parsed = json.loads(raw) + if isinstance(parsed, list): + values = parsed + except json.JSONDecodeError: + pass + if not values and row.get("external_id"): + values = str(row["external_id"]).split(",") + message_ids = [] + for value in values: + text = str(value or "").strip() + if text: + message_ids.append(text) + return message_ids + + +class TelegramReactor: + def __init__(self) -> None: + self.pool = None + self.heartbeat = HeartbeatReporter(WORKER_TG_REACTOR, 30) + self.bot_tokens: list[str] = [] + self.reactions: list[str] = ["👍"] + self.delay_sec = 60 + self.interval_sec = 60 + self.reaction_pause_sec = 2 + self.react_all_messages = False + self.max_attempts = 3 + self.retry_backoff_max_sec = 30 + self.since = "" + + async def init(self) -> None: + self.pool = await get_pool() + await self.reload_settings() + + async def reload_settings(self) -> None: + tokens = split_tokens(await fetch_setting("tg_reactor_bot_tokens", "")) + if not tokens: + fallback = str(await fetch_setting("tg_poster_bot_token", "") or "").strip() or settings.tg_bot_token + tokens = split_tokens(fallback) + if not tokens: + raise RuntimeError("tg_reactor_bot_tokens, tg_poster_bot_token and TG_BOT_TOKEN are empty") + self.bot_tokens = tokens + self.reactions = normalize_reactions(await fetch_setting("tg_reactor_reactions_json", ["👍"])) + self.since = str(await fetch_setting("tg_reactor_since", "") or "").strip() + self.delay_sec = max(0, await fetch_int_setting("tg_reactor_delay_sec", 60)) + self.interval_sec = max(10, await fetch_int_setting("tg_reactor_interval_sec", 60)) + self.reaction_pause_sec = max(0, await fetch_int_setting("tg_reactor_reaction_pause_sec", 2)) + self.react_all_messages = await fetch_bool_setting("tg_reactor_react_all_messages", False) + self.max_attempts = max(1, await fetch_int_setting("tg_reactor_max_attempts", 3)) + self.retry_backoff_max_sec = max(1, await fetch_int_setting("tg_reactor_retry_backoff_max_sec", 30)) + logger.info( + "TG reactor config: bots={} reactions={} since={} delay={}s pause={}s all_messages={}", + len(self.bot_tokens), + self.reactions, + self.since or "-", + self.delay_sec, + self.reaction_pause_sec, + self.react_all_messages, + ) + + async def load_publications(self, limit: int = 50) -> list[dict[str, Any]]: + rows = await self.pool.fetch( + """ + SELECT pp.* + FROM post_publications pp + WHERE pp.platform=$1 + AND pp.status=$2 + AND pp.published_at IS NOT NULL + AND pp.published_at <= NOW() - ($3::int * INTERVAL '1 second') + AND ($4::text = '' OR pp.published_at >= $4::timestamptz) + ORDER BY pp.published_at DESC, pp.id DESC + LIMIT $5::int + """, + PLATFORM_TG, + PUBLICATION_STATUS_PUBLISHED, + self.delay_sec, + self.since, + limit, + ) + return [dict(row) for row in rows] + + async def ensure_reaction_row( + self, + publication: dict[str, Any], + reactor_key: str, + reaction: str, + message_id: str, + ) -> dict[str, Any] | None: + row = await self.pool.fetchrow( + """ + INSERT INTO post_reactions( + publication_id, raw_post_id, platform, reactor_key, reaction, target_id, message_id, status, updated_at + ) + VALUES($1, $2, $3, $4, $5, $6, $7, $8, NOW()) + ON CONFLICT (publication_id, reactor_key, message_id) DO NOTHING + RETURNING * + """, + int(publication["id"]), + int(publication["raw_post_id"]), + PLATFORM_TG, + reactor_key, + reaction, + str(publication["target_id"]), + message_id, + REACTION_STATUS_PENDING, + ) + if row: + return dict(row) + row = await self.pool.fetchrow( + """ + SELECT * + FROM post_reactions + WHERE publication_id=$1 AND reactor_key=$2 AND message_id=$3 + """, + int(publication["id"]), + reactor_key, + message_id, + ) + if not row or row["status"] == REACTION_STATUS_REACTED or int(row["attempts"] or 0) >= self.max_attempts: + return None + return dict(row) + + async def claim_reaction(self, reaction_id: int) -> dict[str, Any] | None: + row = await self.pool.fetchrow( + """ + UPDATE post_reactions + SET status=$2, + attempts=attempts + 1, + error=NULL, + updated_at=NOW() + WHERE id=$1 + AND status IN ($3, $4) + AND attempts < $5 + RETURNING * + """, + reaction_id, + REACTION_STATUS_IN_PROGRESS, + REACTION_STATUS_PENDING, + REACTION_STATUS_FAILED, + self.max_attempts, + ) + return dict(row) if row else None + + async def set_reaction(self, token: str, chat_id: str, message_id: str, reaction: str) -> None: + url = f"https://api.telegram.org/bot{token}/setMessageReaction" + payload = { + "chat_id": int(chat_id), + "message_id": int(message_id), + "reaction": [{"type": "emoji", "emoji": reaction}], + "is_big": False, + } + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=30)) as session: + async with session.post(url, json=payload) as resp: + data = await resp.json(content_type=None) + if not data.get("ok"): + description = data.get("description") or data + retry_after = ((data.get("parameters") or {}).get("retry_after")) if isinstance(data, dict) else None + if retry_after: + raise RuntimeError(f"Telegram retry_after={retry_after}: {description}") + raise RuntimeError(str(description)) + + async def mark_reacted(self, reaction_id: int) -> None: + await self.pool.execute( + """ + UPDATE post_reactions + SET status=$2, + error=NULL, + reacted_at=NOW(), + updated_at=NOW() + WHERE id=$1 + """, + reaction_id, + REACTION_STATUS_REACTED, + ) + + async def mark_failed(self, reaction_id: int, error: str) -> None: + await self.pool.execute( + """ + UPDATE post_reactions + SET status=$2, + error=$3, + updated_at=NOW() + WHERE id=$1 + """, + reaction_id, + REACTION_STATUS_FAILED, + error[:1000], + ) + + async def process_publication(self, publication: dict[str, Any]) -> int: + message_ids = parse_message_ids(publication) + if not message_ids: + return 0 + if not self.react_all_messages: + message_ids = message_ids[:1] + + done = 0 + for message_id in message_ids: + for idx, token in enumerate(self.bot_tokens): + reaction = self.reactions[idx % len(self.reactions)] + reactor_key = f"bot-{idx + 1}:{reaction}" + row = await self.ensure_reaction_row(publication, reactor_key, reaction, message_id) + if not row: + continue + claimed = await self.claim_reaction(int(row["id"])) + if not claimed: + continue + try: + await self.set_reaction(token, str(publication["target_id"]), message_id, reaction) + await self.mark_reacted(int(claimed["id"])) + done += 1 + logger.info( + "TG reacted raw_post={} publication={} message={} reactor={}", + publication["raw_post_id"], + publication["id"], + message_id, + reactor_key, + ) + if self.reaction_pause_sec: + await asyncio.sleep(self.reaction_pause_sec) + except Exception as exc: + await self.mark_failed(int(claimed["id"]), str(exc)) + logger.warning( + "TG reaction failed raw_post={} publication={} message={} reactor={}: {}", + publication["raw_post_id"], + publication["id"], + message_id, + reactor_key, + exc, + ) + await asyncio.sleep(min(2 ** int(claimed["attempts"]), self.retry_backoff_max_sec)) + return done + + async def run_once(self) -> bool: + await self.reload_settings() + enabled = await is_worker_enabled(self.pool, WORKER_TG_REACTOR) + app_enabled = await fetch_bool_setting("tg_reactor_enabled", False) + if not enabled or not app_enabled: + await self.heartbeat.beat(self.pool, status="disabled", force=True) + return False + publications = await self.load_publications() + if not publications: + await self.heartbeat.beat(self.pool, status="idle") + return False + total = 0 + for publication in publications: + await self.heartbeat.beat( + self.pool, + status="reacting", + meta={"publication_id": int(publication["id"]), "raw_post_id": int(publication["raw_post_id"])}, + force=True, + ) + total += await self.process_publication(publication) + await self.heartbeat.beat(self.pool, status="idle", meta={"reactions": total}, force=True) + return bool(total) + + async def run_loop(self) -> None: + await self.init() + logger.info("{} started pid={}", WORKER_TG_REACTOR, os.getpid()) + while True: + try: + await self.run_once() + except Exception as exc: + logger.exception("TG reactor loop error: {}", exc) + await asyncio.sleep(self.interval_sec) + + +async def main() -> None: + logger.remove() + logger.add(sys.stdout, level=settings.log_level) + worker = TelegramReactor() + await worker.run_loop() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/vk_parser_app/workers/vk_poster.py b/src/vk_parser_app/workers/vk_poster.py new file mode 100644 index 0000000..9ffa960 --- /dev/null +++ b/src/vk_parser_app/workers/vk_poster.py @@ -0,0 +1,442 @@ +from __future__ import annotations + +import asyncio +import json +import os +import re +import sys +from datetime import datetime, time, timedelta, timezone +from typing import Any +from zoneinfo import ZoneInfo + +import aiohttp +from loguru import logger + +from ..config import settings +from ..constants import ( + PUBLICATION_STATUS_FAILED, + PUBLICATION_STATUS_PENDING, + PUBLICATION_STATUS_PUBLISHED, + WORKER_VK_POSTER, +) +from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool +from ..heartbeat import HeartbeatReporter +from ..jobs import is_worker_enabled +from ..text_utils import build_publication_text +from ..vk_api import VKAPIClient, post_vk_url + +LOCAL_TZ = ZoneInfo("Asia/Yekaterinburg") +VK_MESSAGE_LIMIT = 16384 +VK_MAX_ATTACHMENTS = 10 + + +def valid_time(value: str) -> bool: + return bool(re.fullmatch(r"(?:[01]\d|2[0-3]):[0-5]\d", str(value or "").strip())) + + +def normalize_schedule(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list): + value = [] + rows: list[dict[str, Any]] = [] + for idx, item in enumerate(value): + if not isinstance(item, dict): + continue + time_value = str(item.get("time") or "").strip() + if not valid_time(time_value): + continue + count = max(0, min(20, int(item.get("count") or 0))) + if count < 1: + continue + rows.append( + { + "id": str(item.get("id") or f"slot-{idx + 1}").strip()[:80], + "time": time_value, + "count": count, + "enabled": bool(item.get("enabled", True)), + } + ) + return sorted(rows, key=lambda row: row["time"]) + + +class VKPoster: + def __init__(self) -> None: + self.pool = None + self.heartbeat = HeartbeatReporter(WORKER_VK_POSTER, 30) + self.token = "" + self.owner_id = 0 + self.from_group = True + self.schedule: list[dict[str, Any]] = [] + self.interval_sec = 60 + self.send_delay_sec = 1.0 + self.recent_window = 20 + self.category_repeat_penalty = 3.0 + self.source_repeat_penalty = 4.0 + self.dry_run = False + + async def init(self) -> None: + self.pool = await get_pool() + + def token_expires_soon(self, value: Any) -> bool: + raw = str(value or "").strip() + if not raw: + return False + try: + expires_at = datetime.fromisoformat(raw) + if expires_at.tzinfo is None: + expires_at = expires_at.replace(tzinfo=timezone.utc) + except ValueError: + return False + return expires_at <= datetime.now(timezone.utc) + timedelta(minutes=15) + + async def save_setting(self, key: str, value: Any) -> None: + await self.pool.execute( + """ + UPDATE app_settings + SET value_json=$2::jsonb, + updated_at=NOW() + WHERE key=$1 + """, + key, + json.dumps(value, ensure_ascii=False), + ) + + async def refresh_access_token_if_needed(self) -> None: + expires_at = await fetch_setting("vk_poster_token_expires_at", "") + if not self.token_expires_soon(expires_at): + return + refresh_token = str(await fetch_setting("vk_poster_refresh_token", "") or "").strip() + client_id = str(await fetch_setting("vk_poster_app_id", "") or "").strip() + client_secret = str(await fetch_setting("vk_poster_client_secret", "") or "").strip() + device_id = str(await fetch_setting("vk_poster_token_device_id", "") or "").strip() + if not refresh_token or not client_id: + return + data = { + "grant_type": "refresh_token", + "refresh_token": refresh_token, + "client_id": client_id, + "device_id": device_id, + "state": "vk-poster-refresh", + } + if client_secret: + data["client_secret"] = client_secret + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=30)) as session: + async with session.post("https://id.vk.ru/oauth2/auth", data=data) as resp: + payload = await resp.json(content_type=None) + if payload.get("error"): + raise RuntimeError(f"VK token refresh failed: {payload.get('error')}: {payload.get('error_description')}") + access_token = str(payload.get("access_token") or "").strip() + if not access_token: + raise RuntimeError(f"VK token refresh did not return access_token: {payload}") + new_refresh = str(payload.get("refresh_token") or "").strip() or refresh_token + new_device_id = str(payload.get("device_id") or "").strip() or device_id + expires_in = payload.get("expires_in") + expires_at_value = "" + try: + expires_at_value = (datetime.now(timezone.utc) + timedelta(seconds=int(expires_in))).isoformat() + except Exception: + pass + await self.save_setting("vk_poster_access_token", access_token) + await self.save_setting("vk_poster_refresh_token", new_refresh) + await self.save_setting("vk_poster_token_device_id", new_device_id) + await self.save_setting("vk_poster_token_expires_at", expires_at_value) + self.token = access_token + logger.info("VK access token refreshed; expires_at={}", expires_at_value) + + async def reload_settings(self) -> None: + self.token = ( + str(await fetch_setting("vk_poster_access_token", "") or "").strip() + or settings.vk_group_access_token + or settings.vk_access_token + ) + await self.refresh_access_token_if_needed() + self.owner_id = int(await fetch_int_setting("vk_poster_owner_id", settings.vk_storage_owner_id if settings.vk_storage_group_id else 0)) + self.from_group = await fetch_bool_setting("vk_poster_from_group", True) + self.schedule = normalize_schedule(await fetch_setting("vk_poster_schedule_json", [])) + self.interval_sec = max(10, await fetch_int_setting("vk_poster_interval_sec", 60)) + self.send_delay_sec = max(0.0, await fetch_float_setting("vk_poster_send_delay_sec", 1.0)) + self.recent_window = max(1, await fetch_int_setting("vk_poster_recent_window", 20)) + self.category_repeat_penalty = max(0.0, await fetch_float_setting("vk_poster_category_repeat_penalty", 3.0)) + self.source_repeat_penalty = max(0.0, await fetch_float_setting("vk_poster_source_repeat_penalty", 4.0)) + self.dry_run = await fetch_bool_setting("vk_poster_dry_run", False) + if not self.token: + raise RuntimeError("vk_poster_access_token, VK_GROUP_ACCESS_TOKEN and VK_ACCESS_TOKEN are empty") + if not self.owner_id: + raise RuntimeError("vk_poster_owner_id is empty") + logger.info("VK poster config: owner={} schedule={} dry_run={}", self.owner_id, self.schedule, self.dry_run) + + async def due_slots(self) -> list[dict[str, Any]]: + now = datetime.now(LOCAL_TZ) + today = now.date() + due: list[dict[str, Any]] = [] + for row in self.schedule: + if not row.get("enabled"): + continue + hour, minute = [int(part) for part in str(row["time"]).split(":", 1)] + slot_start = datetime.combine(today, time(hour=hour, minute=minute), tzinfo=LOCAL_TZ) + slot_end = slot_start + timedelta(minutes=10) + if not (slot_start <= now < slot_end): + continue + inserted = await self.pool.fetchrow( + """ + INSERT INTO publication_runs(poster, schedule_id, scheduled_for, scheduled_time, planned_count, status) + VALUES($1, $2, $3, $4, $5, 'started') + ON CONFLICT (poster, schedule_id, scheduled_for) DO NOTHING + RETURNING id + """, + "vk", + row["id"], + today, + row["time"], + int(row["count"]), + ) + if inserted: + due.append({**row, "run_id": int(inserted["id"]), "date": today}) + return due + + async def load_recent(self) -> list[dict[str, Any]]: + rows = await self.pool.fetch( + """ + SELECT rp.final_category_tag, rp.rewrite_category_tag, rp.final_source_tag, rp.rewrite_source_tag + FROM post_publications pp + JOIN raw_posts rp ON rp.id=pp.raw_post_id + WHERE pp.platform='vk' + AND pp.status=$1 + AND pp.target_id=$2 + ORDER BY pp.published_at DESC NULLS LAST, pp.id DESC + LIMIT $3::int + """, + PUBLICATION_STATUS_PUBLISHED, + str(self.owner_id), + self.recent_window, + ) + return [dict(row) for row in rows] + + async def load_candidates(self, limit: int) -> list[dict[str, Any]]: + rows = await self.pool.fetch( + """ + SELECT rp.*, s.name AS source_name, s.tag AS source_tag + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + LEFT JOIN post_publications pp ON pp.raw_post_id=rp.id AND pp.platform='vk' + WHERE rp.status='storage_ready' + AND rp.rewrite_status='ready' + AND COALESCE(rp.editorial_status, 'review') IN ('accepted', 'published', 'publish_failed') + AND COALESCE(pp.status, $1) = $1 + AND EXISTS ( + SELECT 1 + FROM raw_post_media rpm + WHERE rpm.raw_post_id=rp.id + AND COALESCE(rpm.editor_hidden, FALSE)=FALSE + AND COALESCE(rpm.editor_added, FALSE)=FALSE + AND ( + (rpm.media_type='photo' AND rpm.original_attachment_id LIKE 'photo%') + OR ( + rpm.media_type='video' + AND rpm.original_attachment_id LIKE 'video%' + AND COALESCE(rpm.status, '') <> 'link_only' + ) + ) + ) + ORDER BY COALESCE(rp.reviewed_at, rp.edited_at, rp.rewritten_at, rp.created_at) ASC, + rp.id ASC + LIMIT $2::int + """, + PUBLICATION_STATUS_PENDING, + max(limit, 50), + ) + return [dict(row) for row in rows] + + def choose_posts(self, candidates: list[dict[str, Any]], recent: list[dict[str, Any]], count: int) -> list[dict[str, Any]]: + selected: list[dict[str, Any]] = [] + recent_categories = [str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "") for row in recent] + recent_sources = [str(row.get("final_source_tag") or row.get("rewrite_source_tag") or "") for row in recent] + pool = list(candidates) + while pool and len(selected) < count: + best_idx = 0 + selected_categories = [str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "") for row in selected] + selected_sources = [str(row.get("final_source_tag") or row.get("rewrite_source_tag") or row.get("source_tag") or "") for row in selected] + previous_category = selected_categories[-1] if selected_categories else (recent_categories[0] if recent_categories else "") + previous_source = selected_sources[-1] if selected_sources else (recent_sources[0] if recent_sources else "") + best_rank: tuple[int, float, int] | None = None + for idx, post in enumerate(pool): + category = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or "") + source = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "") + immediate_repeats = int(bool(category) and category == previous_category) + int(bool(source) and source == previous_source) + repeat_penalty = ( + (recent_categories.count(category) + selected_categories.count(category)) * self.category_repeat_penalty + + (recent_sources.count(source) + selected_sources.count(source)) * self.source_repeat_penalty + ) + rank = (immediate_repeats, repeat_penalty, idx) + if best_rank is None or rank < best_rank: + best_idx = idx + best_rank = rank + selected.append(pool.pop(best_idx)) + return selected + + async def load_media(self, raw_post_id: int) -> list[dict[str, Any]]: + rows = await self.pool.fetch( + """ + SELECT * + FROM raw_post_media + WHERE raw_post_id=$1 + AND COALESCE(editor_hidden, FALSE)=FALSE + AND media_type IN ('photo', 'video') + ORDER BY sort_order ASC, id ASC + LIMIT $2::int + """, + raw_post_id, + VK_MAX_ATTACHMENTS, + ) + return [dict(row) for row in rows] + + def build_text(self, post: dict[str, Any]) -> str: + category_tag = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or post.get("final_category") or post.get("rewrite_category") or "") + source_tag = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "") + return build_publication_text(post.get("final_text") or post.get("rewritten_text") or "", category_tag, source_tag)[:VK_MESSAGE_LIMIT] + + async def media_attachment(self, client: VKAPIClient, item: dict[str, Any]) -> str | None: + attachment_id = str(item.get("original_attachment_id") or "").strip() + media_type = str(item.get("media_type") or "").strip() + if media_type == "video": + if str(item.get("status") or "").strip() == "link_only": + return None + return attachment_id if attachment_id.startswith("video") else None + if media_type != "photo": + return None + if attachment_id.startswith("photo") and not item.get("editor_added"): + return attachment_id + return None + + async def send_post(self, post: dict[str, Any]) -> tuple[int, str, list[str]]: + text = self.build_text(post) + media_rows = await self.load_media(int(post["id"])) + async with VKAPIClient( + token=self.token, + rps=2, + timeout_total_sec=max(30, await fetch_int_setting("vk_poster_timeout_sec", 90)), + retry_attempts=max(1, await fetch_int_setting("vk_poster_max_attempts", 3)), + retry_min_delay_sec=2, + retry_max_delay_sec=max(2, await fetch_int_setting("vk_poster_retry_backoff_max_sec", 30)), + ) as client: + attachments: list[str] = [] + for item in media_rows: + try: + attachment = await self.media_attachment(client, item) + except Exception as exc: + logger.warning("VK media skipped raw_post={} media_id={}: {}", post["id"], item.get("id"), exc) + continue + if attachment: + attachments.append(attachment) + if len(attachments) >= VK_MAX_ATTACHMENTS: + break + if self.dry_run: + return 0, "", attachments + post_id = await client.create_wall_post(self.owner_id, text, attachments, from_group=self.from_group) + return post_id, post_vk_url(self.owner_id, post_id), attachments + + async def upsert_publication( + self, + post_id: int, + status: str, + external_id: str | None = None, + url: str | None = None, + error: str | None = None, + attachments: list[str] | None = None, + ) -> None: + await self.pool.execute( + """ + INSERT INTO post_publications( + raw_post_id, platform, status, target_id, external_id, url, error, attachments_json, published_at, updated_at + ) + VALUES($1, 'vk', $2, $3, $4, $5, $6, $7::jsonb, CASE WHEN $2=$8 THEN NOW() ELSE NULL END, NOW()) + ON CONFLICT (raw_post_id, platform) DO UPDATE + SET status=EXCLUDED.status, + target_id=EXCLUDED.target_id, + external_id=EXCLUDED.external_id, + url=EXCLUDED.url, + error=EXCLUDED.error, + attachments_json=EXCLUDED.attachments_json, + published_at=CASE WHEN EXCLUDED.status=$8 THEN COALESCE(EXCLUDED.published_at, NOW()) ELSE post_publications.published_at END, + updated_at=NOW() + """, + post_id, + status, + str(self.owner_id), + external_id, + url, + error[:1000] if error else None, + json.dumps(attachments or [], ensure_ascii=False), + PUBLICATION_STATUS_PUBLISHED, + ) + + async def process_slot(self, slot: dict[str, Any]) -> int: + count = int(slot["count"]) + recent = await self.load_recent() + candidates = await self.load_candidates(max(50, count * 10)) + posts = self.choose_posts(candidates, recent, count) + published = 0 + for post in posts: + await self.heartbeat.beat(self.pool, status="publishing", meta={"post_id": int(post["id"]), "slot": slot["id"]}, force=True) + try: + vk_post_id, url, attachments = await self.send_post(post) + await self.upsert_publication(int(post["id"]), PUBLICATION_STATUS_PUBLISHED, str(vk_post_id), url, attachments=attachments) + published += 1 + logger.info("VK published raw_post={} post_id={} attachments={}", post["id"], vk_post_id, len(attachments)) + except Exception as exc: + await self.upsert_publication(int(post["id"]), PUBLICATION_STATUS_FAILED, error=str(exc)) + logger.exception("VK publish failed raw_post={}: {}", post["id"], exc) + if self.send_delay_sec: + await asyncio.sleep(self.send_delay_sec) + await self.pool.execute( + """ + UPDATE publication_runs + SET published_count=$2, + status=$3, + error=$4, + updated_at=NOW() + WHERE id=$1 + """, + int(slot["run_id"]), + published, + "done" if published == count else "partial", + None if published == count else f"published {published} of {count}", + ) + return published + + async def run_once(self) -> bool: + await self.reload_settings() + enabled = await is_worker_enabled(self.pool, WORKER_VK_POSTER) + app_enabled = await fetch_bool_setting("vk_poster_enabled", False) + if not enabled or not app_enabled: + await self.heartbeat.beat(self.pool, status="disabled", force=True) + return False + slots = await self.due_slots() + if not slots: + await self.heartbeat.beat(self.pool, status="idle") + return False + total = 0 + for slot in slots: + total += await self.process_slot(slot) + await self.heartbeat.beat(self.pool, status="idle", meta={"published": total}, force=True) + return bool(total) + + async def run_loop(self) -> None: + await self.init() + logger.info("{} started pid={}", WORKER_VK_POSTER, os.getpid()) + while True: + try: + await self.run_once() + except Exception as exc: + logger.exception("VK poster loop error: {}", exc) + await asyncio.sleep(self.interval_sec) + + +async def main() -> None: + logger.remove() + logger.add(sys.stdout, level=settings.log_level) + worker = VKPoster() + await worker.run_loop() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/vk_parser_app/workers/vk_storage_uploader.py b/src/vk_parser_app/workers/vk_storage_uploader.py new file mode 100644 index 0000000..5237e6a --- /dev/null +++ b/src/vk_parser_app/workers/vk_storage_uploader.py @@ -0,0 +1,662 @@ +from __future__ import annotations + +import asyncio +import html +import os +import re +import sys +import time +from pathlib import Path + +import aiohttp +from aiogram import Bot +from aiogram.client.session.aiohttp import AiohttpSession +from aiogram.client.telegram import TelegramAPIServer +from aiogram.exceptions import TelegramRetryAfter +from aiogram.types import BufferedInputFile, FSInputFile, InputMediaPhoto, InputMediaVideo +from loguru import logger + +from ..config import settings +from ..constants import ( + JOB_TYPE_VK_STORAGE_COPY, + MEDIA_STATUS_FAILED, + MEDIA_STATUS_LINK_ONLY, + MEDIA_STATUS_UPLOADED, + POST_STATUS_FAILED, + POST_STATUS_STORAGE_READY, + WORKER_STORAGE_UPLOADER, +) +from ..db import fetch_float_setting, fetch_int_setting, fetch_setting, get_pool +from ..heartbeat import HeartbeatReporter +from ..jobs import ack_done, ack_retry, claim_job, is_worker_enabled, recover_stale_jobs + +TMP_DIR = Path("/tmp") +TMP_PREFIX = "vkparser_tg_media_" +MAX_MEDIA_GROUP = 10 + + +def parse_topic(value: str) -> tuple[int, int | None]: + raw = str(value or "").strip() + if not raw: + raise RuntimeError("TG_MEDIA_CHANNEL_ID is empty") + if ":" in raw: + chat_id, thread_id = raw.split(":", 1) + return int(chat_id), int(thread_id) + return int(raw), None + + +def parse_vk_video_url(vk_url: str) -> tuple[int, int] | None: + match = re.search(r"video(-?\d+)_(\d+)", vk_url or "") + if not match: + return None + return int(match.group(1)), int(match.group(2)) + + +def split_message_chunks(text: str, limit: int) -> list[str]: + text = str(text or "").strip() + if not text: + return [] + chunks: list[str] = [] + while len(text) > limit: + split_at = text.rfind("\n", 0, limit) + if split_at < limit // 2: + split_at = text.rfind(" ", 0, limit) + if split_at < limit // 2: + split_at = limit + chunks.append(text[:split_at].strip()) + text = text[split_at:].strip() + if text: + chunks.append(text) + return chunks + + +def original_link(post: dict) -> str: + url = str(post.get("original_url") or "").strip() + return f'#{int(post["id"])}' if url else f'#{int(post["id"])}' + + +class TelegramStorageUploader: + def __init__(self) -> None: + self.pool = None + self.bot: Bot | None = None + self.storage_chat_id: int | None = None + self.storage_thread_id: int | None = None + self.heartbeat = HeartbeatReporter(WORKER_STORAGE_UPLOADER, 30) + self.media_group_max_items = MAX_MEDIA_GROUP + self.media_upload_delay_sec = 1.0 + self.tg_retry_max_attempts = 4 + self.tg_retry_backoff_max_sec = 15 + self.caption_limit = 1024 + self.message_limit = 4096 + self.text_overflow_marker = "Продолжение следующим сообщением." + self.download_timeout_sec = 45 + self.video_max_size_mb = 49 + self.video_max_height = 720 + self.video_max_duration_sec = 300 + self.yt_dlp_timeout_sec = 300 + self.max_media_attempts = 3 + self.post_job_pause_sec = 0.2 + self.using_local_bot_api = False + + async def init(self) -> None: + self.pool = await get_pool() + recovered = await recover_stale_jobs(self.pool, JOB_TYPE_VK_STORAGE_COPY, stale_minutes=20) + if recovered: + logger.warning("Recovered stale storage jobs: {}", recovered) + + if not settings.tg_bot_token: + raise RuntimeError("TG_BOT_TOKEN is empty") + media_channel = str(await fetch_setting("tg_media_channel_id", settings.tg_media_channel_id) or "").strip() + self.storage_chat_id, self.storage_thread_id = parse_topic(media_channel) + + local_bot_api_url = str(await fetch_setting("local_bot_api_url", settings.local_bot_api_url) or "").strip() + if local_bot_api_url: + session = AiohttpSession( + api=TelegramAPIServer.from_base(local_bot_api_url.rstrip("/"), is_local=True) + ) + self.bot = Bot(token=settings.tg_bot_token, session=session) + try: + await self.bot.get_me() + self.using_local_bot_api = True + logger.info("Using local Telegram Bot API: {}", local_bot_api_url) + except Exception as exc: + logger.warning("Local Telegram Bot API unavailable at {}: {}. Falling back to cloud Bot API.", local_bot_api_url, exc) + await self.bot.session.close() + self.bot = Bot(token=settings.tg_bot_token) + else: + self.bot = Bot(token=settings.tg_bot_token) + + self.media_group_max_items = max(1, min(MAX_MEDIA_GROUP, await fetch_int_setting("telegram_media_group_max_items", 10))) + self.media_upload_delay_sec = max(0.0, await fetch_float_setting("telegram_media_upload_delay_sec", 1.0)) + self.tg_retry_max_attempts = max(1, await fetch_int_setting("telegram_retry_max_attempts", 4)) + self.tg_retry_backoff_max_sec = max(1, await fetch_int_setting("telegram_retry_backoff_max_sec", 15)) + self.caption_limit = max(128, await fetch_int_setting("telegram_caption_limit", 1024)) + self.message_limit = max(512, await fetch_int_setting("telegram_message_limit", 4096)) + self.text_overflow_marker = str(await fetch_setting("telegram_text_overflow_marker", self.text_overflow_marker)) + self.download_timeout_sec = max(5, await fetch_int_setting("uploader_download_timeout_sec", 45)) + self.video_max_size_mb = max(1, await fetch_int_setting("video_max_size_mb", 49)) + if not self.using_local_bot_api: + self.video_max_size_mb = min(self.video_max_size_mb, 49) + self.video_max_height = max(144, await fetch_int_setting("video_max_height", 720)) + self.video_max_duration_sec = max(1, await fetch_int_setting("video_max_duration_sec", 300)) + self.yt_dlp_timeout_sec = max(30, await fetch_int_setting("uploader_yt_dlp_timeout_sec", 300)) + self.max_media_attempts = max(1, await fetch_int_setting("uploader_max_media_attempts", 3)) + self.post_job_pause_sec = max(0.0, await fetch_float_setting("media_post_job_pause_sec", 0.2)) + + logger.info( + "Telegram storage config: media_delay={}s, media_group_max_items={}, tg_retry={}, tg_backoff_max={}s, dl_timeout={}s, ytdlp_timeout={}s, post_pause={}s", + self.media_upload_delay_sec, + self.media_group_max_items, + self.tg_retry_max_attempts, + self.tg_retry_backoff_max_sec, + self.download_timeout_sec, + self.yt_dlp_timeout_sec, + self.post_job_pause_sec, + ) + + async def close(self) -> None: + if self.bot: + await self.bot.session.close() + + def chat_kwargs(self) -> dict: + kwargs = {"chat_id": self.storage_chat_id} + if self.storage_thread_id: + kwargs["message_thread_id"] = self.storage_thread_id + return kwargs + + async def tg_retry(self, fn): + for attempt in range(1, self.tg_retry_max_attempts + 1): + try: + return await fn() + except TelegramRetryAfter as exc: + delay = float(exc.retry_after) + 0.5 + logger.warning("Telegram flood control, sleep {}s", delay) + await asyncio.sleep(delay) + except Exception as exc: + if attempt >= self.tg_retry_max_attempts: + raise + delay = min(2**attempt, self.tg_retry_backoff_max_sec) + logger.warning("Telegram send error: {}. retry in {}s", exc, delay) + await asyncio.sleep(delay) + raise RuntimeError("telegram retry exhausted") + + async def load_raw_post(self, raw_post_id: int) -> dict | None: + row = await self.pool.fetchrow( + """ + SELECT rp.*, s.name AS source_name + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + WHERE rp.id=$1 + """, + raw_post_id, + ) + return dict(row) if row else None + + async def load_media(self, raw_post_id: int) -> list[dict]: + rows = await self.pool.fetch( + """ + SELECT * + FROM raw_post_media + WHERE raw_post_id=$1 + ORDER BY sort_order ASC, id ASC + """, + raw_post_id, + ) + return [dict(row) for row in rows] + + def media_blocking_error(self, media: list[dict], prepared_media_ids: set[int] | None = None) -> str | None: + prepared_media_ids = prepared_media_ids or set() + visible_media = [ + item + for item in media + if str(item.get("media_type") or "") in ("photo", "video") + and not bool(item.get("editor_hidden")) + ] + if not visible_media: + return "no publishable media" + pending = [ + item + for item in visible_media + if str(item.get("status") or "") == "pending" + and int(item["id"]) not in prepared_media_ids + ] + if pending: + details = ", ".join( + f"{item.get('media_type')}#{item.get('id')}: {item.get('error') or 'pending'}" + for item in pending[:5] + ) + return f"media still pending: {details}" + failed = [ + item + for item in visible_media + if str(item.get("status") or "") == MEDIA_STATUS_FAILED + ] + if failed: + details = ", ".join( + f"{item.get('media_type')}#{item.get('id')}: {item.get('error') or 'failed'}" + for item in failed[:5] + ) + return f"media failed: {details}" + link_only = [ + item + for item in visible_media + if str(item.get("status") or "") == MEDIA_STATUS_LINK_ONLY + ] + if link_only: + details = ", ".join( + f"{item.get('media_type')}#{item.get('id')}: {item.get('error') or 'link only'}" + for item in link_only[:5] + ) + return f"media link only: {details}" + missing_upload = [ + item + for item in visible_media + if not (item.get("tg_file_id") or item.get("storage_attachment_id")) + and int(item["id"]) not in prepared_media_ids + ] + if missing_upload: + details = ", ".join( + f"{item.get('media_type')}#{item.get('id')}: missing tg_file_id" + for item in missing_upload[:5] + ) + return f"media not uploaded: {details}" + return None + + async def download_bytes(self, session: aiohttp.ClientSession, url: str) -> bytes | None: + try: + async with session.get(url, timeout=aiohttp.ClientTimeout(total=self.download_timeout_sec)) as response: + if response.status != 200: + return None + return await response.read() + except Exception: + return None + + async def mark_media_uploaded(self, media_id: int, file_id: str, unique_id: str | None) -> None: + await self.pool.execute( + """ + UPDATE raw_post_media + SET status=$2, + storage_attachment_id=$3, + storage_url=$3, + tg_file_id=$3, + tg_file_unique_id=$4, + error=NULL, + updated_at=NOW() + WHERE id=$1 + """, + media_id, + MEDIA_STATUS_UPLOADED, + file_id, + unique_id, + ) + + async def mark_media_link_only(self, media_id: int, error: str | None = None) -> None: + await self.pool.execute( + """ + UPDATE raw_post_media + SET status=$2, + error=COALESCE($3, error), + updated_at=NOW() + WHERE id=$1 + """, + media_id, + MEDIA_STATUS_LINK_ONLY, + error, + ) + + async def mark_media_failed_attempt(self, media_id: int, error: str) -> None: + await self.pool.execute( + """ + UPDATE raw_post_media + SET attempts=attempts+1, + status=CASE WHEN attempts + 1 >= $3 THEN $2 ELSE status END, + error=$4, + updated_at=NOW() + WHERE id=$1 + """, + media_id, + MEDIA_STATUS_FAILED, + self.max_media_attempts, + error[:1000], + ) + + async def download_video(self, vk_url: str, output_path: str) -> dict | None: + parsed = parse_vk_video_url(vk_url) + if not parsed: + return None + owner_id, video_id = parsed + max_size = self.video_max_size_mb * 1024 * 1024 + netrc_path = f"{output_path}.netrc" + fd = os.open(netrc_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as fh: + fh.write(f"machine vk.com login vk_token password {settings.vk_access_token}\n") + cmd = [ + sys.executable, + "-m", + "yt_dlp", + "--netrc-location", + netrc_path, + f"https://vk.com/video{owner_id}_{video_id}", + "-o", + output_path, + "--no-playlist", + "-f", + ( + f"best[height<={self.video_max_height}][filesize<{max_size}]" + f"/best[height<={self.video_max_height}]" + f"/bestvideo[height<={self.video_max_height}][filesize<{max_size}]+bestaudio/best" + f"/bestvideo[height<={self.video_max_height}]+bestaudio/best" + f"/best[filesize<{max_size}]" + ), + "--quiet", + "--no-warnings", + ] + try: + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + _, stderr = await asyncio.wait_for(proc.communicate(), timeout=self.yt_dlp_timeout_sec) + except asyncio.TimeoutError: + proc.kill() + await proc.communicate() + return {"error": "video download timeout", "permanent": False} + finally: + try: + os.unlink(netrc_path) + except FileNotFoundError: + pass + if proc.returncode != 0 or not os.path.exists(output_path): + err = (stderr or b"").decode("utf-8", errors="ignore").lower() + permanent = any(marker in err for marker in ("removed", "unavailable", "private", "access denied")) + return {"error": "video unavailable or download failed", "permanent": permanent} + size = os.path.getsize(output_path) + if size > max_size: + return {"error": "video too large", "permanent": True} + return {"path": output_path, "size_bytes": size} + + async def prepare_media(self, raw_post_id: int, media: list[dict]) -> list[dict]: + prepared: list[dict] = [] + async with aiohttp.ClientSession() as session: + for item in media: + media_id = int(item["id"]) + media_type = str(item["media_type"]) + url = str(item.get("original_url") or "") + + if item.get("tg_file_id"): + prepared.append({"media_id": media_id, "media_type": media_type, "media": item["tg_file_id"]}) + continue + if int(item.get("attempts") or 0) >= self.max_media_attempts: + continue + + if media_type == "photo": + data = await self.download_bytes(session, url) + if not data: + await self.mark_media_failed_attempt(media_id, "photo download failed") + continue + prepared.append( + { + "media_id": media_id, + "media_type": "photo", + "media": BufferedInputFile(data, filename=f"photo_{media_id}.jpg"), + } + ) + continue + + if media_type == "video": + duration = item.get("duration_sec") + if duration and int(duration) > self.video_max_duration_sec: + await self.mark_media_link_only(media_id, "video too long") + continue + temp_path = str(TMP_DIR / f"{TMP_PREFIX}{raw_post_id}_{media_id}.mp4") + info = await self.download_video(url, temp_path) + if not info: + await self.mark_media_failed_attempt(media_id, "video download failed") + continue + if info.get("error"): + if info.get("permanent"): + await self.mark_media_link_only(media_id, str(info["error"])) + else: + await self.mark_media_failed_attempt(media_id, str(info["error"])) + continue + prepared.append( + { + "media_id": media_id, + "media_type": "video", + "media": FSInputFile(temp_path, filename="video.mp4"), + "tmp_path": temp_path, + } + ) + continue + + await self.mark_media_link_only(media_id, "unsupported media type") + return prepared + + async def send_text_chunk(self, text: str) -> int: + message = await self.tg_retry( + lambda: self.bot.send_message( + text=text, + parse_mode="HTML", + disable_web_page_preview=True, + **self.chat_kwargs(), + ) + ) + return int(message.message_id) + + def build_text_parts(self, post: dict, has_media: bool, link_only_media: list[dict] | None = None) -> tuple[str | None, list[str]]: + link = original_link(post) + raw_text = html.escape(str(post.get("raw_text") or "").strip()) + link_only_media = link_only_media or [] + media_lines = [] + for item in link_only_media: + url = str(item.get("original_url") or "").strip() + if not url: + continue + media_type = html.escape(str(item.get("media_type") or "media")) + reason = html.escape(str(item.get("error") or "link only")) + media_lines.append(f"- {media_type}: ссылка ({reason})") + media_note = "\n\nМедиа по ссылке:\n" + "\n".join(media_lines) if media_lines else "" + suffix = f"\n\n{link}" + body = raw_text + media_note + if has_media: + if len(body + suffix) <= self.caption_limit: + return body + suffix if body else link, [] + chunks = split_message_chunks(body + suffix, self.message_limit) + return self.text_overflow_marker[: self.caption_limit], chunks + chunks = split_message_chunks(body + suffix if body else link, self.message_limit) + return None, chunks + + async def save_sent_media_ids(self, prepared: list[dict], messages: list) -> None: + for item, message in zip(prepared, messages): + if item["media_type"] == "photo" and message.photo: + photo = message.photo[-1] + await self.mark_media_uploaded(int(item["media_id"]), photo.file_id, photo.file_unique_id) + elif item["media_type"] == "video" and message.video: + video = message.video + await self.mark_media_uploaded(int(item["media_id"]), video.file_id, video.file_unique_id) + + async def send_storage_post(self, post: dict, prepared: list[dict], media: list[dict]) -> tuple[list[int], int | None]: + uploaded_media_ids = {int(item["media_id"]) for item in prepared} + link_only_media = [ + item + for item in media + if int(item["id"]) not in uploaded_media_ids and str(item.get("status") or "") == MEDIA_STATUS_LINK_ONLY + ] + caption, text_chunks = self.build_text_parts(post, bool(prepared), link_only_media) + message_ids: list[int] = [] + + if prepared: + first = prepared[: self.media_group_max_items] + group = [] + for idx, item in enumerate(first): + cap = caption if idx == 0 else None + if item["media_type"] == "photo": + group.append(InputMediaPhoto(media=item["media"], caption=cap, parse_mode="HTML")) + elif item["media_type"] == "video": + group.append(InputMediaVideo(media=item["media"], caption=cap, parse_mode="HTML")) + if group: + messages = await self.tg_retry(lambda: self.bot.send_media_group(media=group, **self.chat_kwargs())) + await self.save_sent_media_ids(first, messages) + message_ids.extend(int(msg.message_id) for msg in messages) + await asyncio.sleep(self.media_upload_delay_sec * len(first)) + + rest = prepared[self.media_group_max_items :] + for start in range(0, len(rest), self.media_group_max_items): + chunk = rest[start : start + self.media_group_max_items] + group = [ + InputMediaPhoto(media=item["media"]) + if item["media_type"] == "photo" + else InputMediaVideo(media=item["media"]) + for item in chunk + if item["media_type"] in ("photo", "video") + ] + if group: + messages = await self.tg_retry(lambda g=group: self.bot.send_media_group(media=g, **self.chat_kwargs())) + await self.save_sent_media_ids(chunk, messages) + message_ids.extend(int(msg.message_id) for msg in messages) + await asyncio.sleep(self.media_upload_delay_sec * len(chunk)) + for chunk in text_chunks: + message_ids.append(await self.send_text_chunk(chunk)) + return message_ids, None + + async def mark_post_ready(self, raw_post_id: int, message_ids: list[int], meta_message_id: int | None) -> None: + await self.pool.execute( + """ + UPDATE raw_posts + SET status=$2, + tg_storage_chat_id=$3, + tg_storage_thread_id=$4, + tg_storage_message_ids=$5, + tg_storage_meta_message_id=$6, + storage_post_url=$7, + copied_at=NOW(), + error_reason=NULL, + updated_at=NOW() + WHERE id=$1 + """, + raw_post_id, + POST_STATUS_STORAGE_READY, + self.storage_chat_id, + self.storage_thread_id, + message_ids, + meta_message_id, + f"tg://resolve?domain=c/{str(abs(int(self.storage_chat_id or 0))).removeprefix('100')}/{message_ids[0]}" if message_ids else None, + ) + + async def mark_post_failed(self, raw_post_id: int, error: str) -> None: + await self.pool.execute( + """ + UPDATE raw_posts + SET status=$2, + error_reason=$3, + updated_at=NOW() + WHERE id=$1 + """, + raw_post_id, + POST_STATUS_FAILED, + error[:1000], + ) + + async def process_job(self, job: dict, worker_id: str) -> None: + job_id = int(job["id"]) + raw_post_id = int(job["entity_id"]) + await self.heartbeat.beat(self.pool, current_job_id=job_id, force=True) + + post = await self.load_raw_post(raw_post_id) + if not post: + await ack_retry(self.pool, job, f"raw_post {raw_post_id} not found", delay_sec=60) + return + if post.get("tg_storage_message_ids"): + await ack_done(self.pool, job_id) + return + + media = await self.load_media(raw_post_id) + prepared = await self.prepare_media(raw_post_id, media) + media = await self.load_media(raw_post_id) + prepared_media_ids = {int(item["media_id"]) for item in prepared} + blocking_error = self.media_blocking_error(media, prepared_media_ids) + if blocking_error: + for item in prepared: + tmp_path = item.get("tmp_path") + if tmp_path: + try: + if os.path.exists(tmp_path): + os.remove(tmp_path) + except Exception: + pass + if not blocking_error.startswith("media still pending:"): + await self.mark_post_failed(raw_post_id, blocking_error) + await ack_done(self.pool, job_id) + logger.warning("Telegram storage blocked raw_post={}: {}", raw_post_id, blocking_error) + return + await ack_retry(self.pool, job, blocking_error, delay_sec=120) + logger.warning("Telegram storage retry raw_post={}: {}", raw_post_id, blocking_error) + return + try: + message_ids, meta_message_id = await self.send_storage_post(post, prepared, media) + finally: + for item in prepared: + tmp_path = item.get("tmp_path") + if tmp_path: + try: + if os.path.exists(tmp_path): + os.remove(tmp_path) + except Exception: + pass + await self.mark_post_ready(raw_post_id, message_ids, meta_message_id) + await ack_done(self.pool, job_id) + logger.info("Telegram storage done: raw_post={} messages={}", raw_post_id, message_ids) + + async def run_once(self, worker_id: str) -> bool: + enabled = await is_worker_enabled(self.pool, WORKER_STORAGE_UPLOADER) + if not enabled: + await self.heartbeat.beat(self.pool, status="disabled", force=True) + return False + + job = await claim_job(self.pool, JOB_TYPE_VK_STORAGE_COPY, worker_id) + if not job: + await self.heartbeat.beat(self.pool, status="idle") + return False + try: + await self.process_job(job, worker_id) + except Exception as exc: + await self.mark_post_failed(int(job["entity_id"]), str(exc)) + await ack_retry(self.pool, job, str(exc), delay_sec=120) + logger.exception("Telegram storage job {} failed: {}", job["id"], exc) + return True + + async def cleanup_tmp_files(self) -> None: + now = time.time() + for path in TMP_DIR.glob(f"{TMP_PREFIX}*"): + try: + if path.is_file() and path.stat().st_mtime < now - 3600: + path.unlink(missing_ok=True) + except Exception: + pass + + async def run_loop(self) -> None: + await self.init() + worker_id = f"{WORKER_STORAGE_UPLOADER}:{os.getpid()}" + logger.info("{} started", worker_id) + try: + while True: + await self.cleanup_tmp_files() + had_job = await self.run_once(worker_id) + if not had_job: + await asyncio.sleep(max(1, await fetch_int_setting("uploader_interval_sec", 5))) + else: + await asyncio.sleep(self.post_job_pause_sec) + finally: + await self.close() + + +async def main() -> None: + logger.remove() + logger.add(sys.stdout, level=settings.log_level) + worker = TelegramStorageUploader() + await worker.run_loop() + + +if __name__ == "__main__": + asyncio.run(main())