diff --git a/PROJECT.md b/PROJECT.md new file mode 100644 index 0000000..3eab660 --- /dev/null +++ b/PROJECT.md @@ -0,0 +1,1714 @@ +# N8 Parser: проектная документация + +Этот документ описывает текущую архитектуру проекта `n8_parser`: зачем он нужен, как устроены БД, воркеры, админка, AI-контур, медиа-хранилище, деплой и типовые операции. Документ написан как рабочая карта проекта, чтобы к нему можно было вернуться через месяц и быстро понять, где что лежит и почему сделано именно так. + +## 1. Цель проекта + +Проект автоматизирует сбор и редакционную обработку постов из VK-сообществ производителей, магазинов и площадок по тактическому, милитари и airsoft-снаряжению. + +Основная цепочка: + +```text +Источники VK + -> VK-парсер + -> raw_posts / raw_post_media в PostgreSQL + -> Telegram media storage + -> AI-квалификатор + -> AI-райтер + -> редакторская + -> принятие / отклонение / ручное редактирование +``` + +Проект не должен превращаться в набор разрозненных скриптов. Главная идея текущей реализации: все долгие операции живут в отдельных воркерах, все настройки управляются из админки, а секреты остаются в `.env`. + +## 2. Текущий стек + +- Backend: Python 3.12+, FastAPI. +- Шаблоны: Jinja2. +- БД: PostgreSQL. +- Асинхронный доступ к БД: `asyncpg`. +- VK API: собственный клиент в `src/vk_parser_app/vk_api.py`. +- Telegram: `aiogram`. +- AI: `litellm`, сейчас используется Anthropic через LiteLLM. +- Логи: `loguru`. +- Системный запуск на сервере: `systemd`. +- Reverse proxy / TLS: nginx + сертификат для `https://sw.exostring.xyz`. +- Репозиторий: GitHub `exostring/n8_parser`. + +## 3. Структура проекта + +```text +. +├── db/ +│ └── migrations/ # SQL-миграции +├── scripts/ +│ ├── apply_migrations.py # применение миграций +│ └── vk_storage_spike.py # старый тест VK storage +├── src/ +│ └── vk_parser_app/ +│ ├── admin.py # FastAPI админка и маршруты +│ ├── config.py # env-настройки +│ ├── constants.py # статусы, имена воркеров, типы jobs +│ ├── db.py # pool, settings, миграции +│ ├── heartbeat.py # heartbeat воркеров +│ ├── jobs.py # очередь jobs +│ ├── main.py # ASGI entrypoint +│ ├── security.py # пароли, сессии, CSRF +│ ├── text_utils.py # тэги, категории +│ ├── vk_api.py # VK API client и извлечение media +│ ├── templates/ # HTML-шаблоны +│ └── workers/ +│ ├── parser.py # VK-парсер +│ ├── vk_storage_uploader.py# Telegram media storage uploader +│ ├── ai_qualifier.py # AI-квалификатор +│ └── ai_writer.py # AI-райтер +├── original_project/ # старый проект как справочник +├── requirements.txt +├── README.md # короткий early quick start, частично устарел +└── PROJECT.md # этот документ +``` + +## 4. Важное архитектурное решение + +### 4.0. Важные изменения фиксируются в документации + +Если меняется поведение системы, эксплуатационная схема, публикационный формат, БД, деплой или админка, это нужно отражать в `PROJECT.md` или другом релевантном `.md` рядом с кодом. + +Текущие зафиксированные изменения: + +- Публичный сайт вынесен в самостоятельное приложение `forma_n8_site/` и разворачивается на отдельном VPS. Сайт хранит собственные статьи и медиа, отдаёт серверный HTML, RSS, sitemap, robots.txt, canonical и JSON-LD `NewsArticle`. +- Интеграция с сайтом строится через idempotent `POST /api/v1/articles`: `external_id` уникален, а slug и дата первой публикации сохраняются при обновлениях. Дизайн можно менять без изменения URL и повторной загрузки постов. Медиа передаются через защищённый `POST /api/v1/media` и хранятся на стороне сайта. +- Публичные названия категорий сайта отделены от описательных AI-категорий парсера: таблица `categories` связывает стабильный латинский tag с коротким SEO-названием. API нормализует входной tag и использует справочник для H1, breadcrumbs и category URL. +- Пока у публичного сайта нет домена, `SITE_INDEXING_ENABLED=false`: HTML получает `noindex`, а robots.txt закрывает IP от обхода. Индексация включается только после настройки домена, HTTPS, корректного `SITE_BASE_URL` и редиректа с IP. + +- `favi.png` лежит в корне проекта и отдаётся FastAPI по `/favi.png` и `/favicon.ico`. +- Финальный текст публикации собирается через общий helper: к тексту добавляется строка `#category #source`. +- Категорийный хэштег берётся из `content_categories.tag`, а не из человекочитаемого названия категории. +- В админке добавлена вкладка `/logs` для просмотра `audit_log`; записи старше 30 дней удаляются автоматически. +- Добавлен TG-постер: расписание по Екб, отдельный токен/чат, ранжирование, Telegram-лимиты и статусы `published` / `publish_failed`. +- Добавлен TG-реактор: отдельный воркер ставит реакции на опубликованные Telegram-посты из `post_publications`; один bot token ставит одну реакцию, несколько bot tokens могут поставить несколько реакций. +- Добавлен ежедневный Telegram-отчет `daily-report`: в `00:04` по Екатеринбургу отправляет получателям из `daily_report_recipient_ids` операционный отчет за прошедший день: raw-сбор, AI-принято/отклонено, редакторская очередь, TG/VK публикации, ошибки и состояние воркеров. По умолчанию получатель `442509142`, bot token берется из `daily_report_bot_token`, затем `tg_poster_bot_token`, затем `TG_BOT_TOKEN`. +- Текущая схема TG-реактора: основной бот публикации + два отдельных reaction-бота; реакции распределены по порядку токенов как `👍`, `🔥`, `❤`. +- Для догонки реакций TG-реактор использует паузу `tg_reactor_reaction_pause_sec` между успешными реакциями, чтобы не отправлять пачку запросов в Telegram одним всплеском. +- Разделительная полоса в AI-рерайтах должна быть строго `━━━━━━━━━━━━━━━━━` (17 символов). +- TG-постер перечитывает настройки перед циклом и не догоняет прошедшие часы; слот срабатывает только в 10-минутном окне после своего времени. +- TG-постер публикует медиа только через Telegram `file_id`/storage attachment или локальный upload; сырые `http(s)` media URL не отправляются как Telegram media, чтобы не ловить `WEBPAGE_MEDIA_EMPTY` / `wrong type of the web page content`. +- TG-постер повторяет отправку только при `TelegramRetryAfter` flood-control. Обычные timeout/network ошибки после `send_message`/`send_media` не ретраятся, потому что Telegram мог уже принять сообщение, а повтор создаёт дубли. +- На сервере собран и запущен локальный Telegram Bot API (`telegram-bot-api.service`) на `127.0.0.1:8081`. Uploader использует его через `local_bot_api_url`, чтобы загружать видео больше облачного лимита Bot API. +- Telegram storage uploader скачивает VK-видео через `yt-dlp` и только после успешной загрузки всех pending фото/видео переводит raw-пост в `storage_ready`. Если видео временно не скачалось, job ретраится; если media окончательно failed, raw-пост получает `failed`, чтобы публикация не ушла с одной обложкой вместо ролика. Текущие лимиты после включения local Bot API: `video_max_size_mb=512`, `video_max_height=1080`, `video_max_duration_sec=1200`, `uploader_yt_dlp_timeout_sec=1800`. +- Вся цепочка после storage теперь требует минимум одно видимое загруженное `photo`/`video`: uploader не переводит пост без media в `storage_ready`, AI-квалификатор и AI-райтер не забирают посты без загруженного media, TG-постер не публикует старые готовые записи без загруженного media. Если видео ушло в `failed` или `link_only`, пост считается `failed` и не попадает даже на квалификацию. +- При финальной проверке media uploader не считает `pending` те файлы, которые уже успешно скачаны и подготовлены в текущем проходе: их Telegram `file_id` появляется только после отправки. Иначе uploader зацикливается между подготовкой файла и повтором job, не отправляя медиа в storage. +- TG-постер дополнительно не берёт кандидата в публикацию, если у видимого `photo`/`video` нет `tg_file_id`/`storage_attachment_id`; это защищает от старых `storage_ready` записей, которые успели стать готовыми до ужесточения uploader. +- Для разовой дозагрузки старых timeout-видео добавлен `scripts/backfill_pending_videos.py`: он последовательно скачивает pending VK-видео, отправляет в Telegram storage через local Bot API и проставляет `tg_file_id`. +- На странице Raw можно выделить несколько постов и вручную отправить их в AI-рерайт; обработчик переводит только `storage_ready` посты в `qualification_status='accepted'` и `rewrite_status='pending'`. +- Категория AI-райтера `18` / `Не целевой контент` / `НЦК` считается автоматическим AI-отклонением: такие посты получают `qualification_status='rejected'`, `editorial_status='rejected'` и не попадают в редакторскую `На проверке`. +- Добавлена общая таблица `post_publications`: в редакторской у каждой новости видны платформенные статусы TG/VK/MAX/Сайт. TG-постер зеркалит результат в эту таблицу, VK-постер пишет туда напрямую. +- Запрос редакторской агрегирует строки `post_publications` для каждой новости; статусы TG/VK берутся из фактических платформенных публикаций, поэтому опубликованный пост не отображается как `ожидает`. +- Добавлен VK-постер `src/vk_parser_app/workers/vk_poster.py`: публикует принятые/уже опубликованные в TG посты в целевое VK-сообщество по отдельному расписанию Екб. На текущем этапе VK-постер не загружает медиа через `photos.*`, а берёт только исходные VK attachments (`photo...` / `video...`) из `raw_post_media.original_attachment_id`; посты без такого VK attachment не попадают в кандидаты VK-публикации. +- Добавлен MAX-постер `src/vk_parser_app/workers/max_poster.py`: публикует принятые/уже опубликованные материалы в MAX-канал через Bot API от имени бота-администратора. Расписание по умолчанию повторяет TG hourly schedule. Текст режется по лимиту MAX `4000` символов, медиа загружается в MAX отдельно через `/uploads`, а результат пишется в `post_publications(platform='max')`. +- MAX-постер для видео использует не `original_url` VK-страницы, а готовый файл из Telegram storage по `tg_file_id` / `storage_attachment_id`. Если локальный Telegram Bot API возвращает абсолютный `file_path` в `/var/lib/telegram-bot-api/...`, воркер читает файл напрямую с диска. После upload видео воркер poll-ит `GET /videos/{videoToken}` и отправляет пост только когда MAX вернул `urls`. +- В MAX Bot API на момент внедрения нет публичного метода для реакции на пост: попытка `/messages/{message_id}/reactions` возвращает `404 method.not.found`. Auto-reaction оставлен как non-fatal hook и не должен помечать успешную публикацию ошибкой. +- В исходных VK-постах могут встречаться отдельные пустые/белые фото рядом с видео. Пример: `raw_post_id=398`, `photo-190867868_457284718` полностью белая (`1433x2000`, RGB `255/255/255`). MAX-постер пока прикрепляет такие фото как обычные исходные медиа; следующий безопасный шаг - добавить фильтр почти полностью белых изображений перед upload. +- Ранжирование TG/VK-постеров не использует `qualification_score`: квалификатор только допускает пост в дальнейшую цепочку. Постер сначала выбирает вариант без совпадения категории/источника с предыдущей публикацией, затем минимизирует повторы категории и источника в окне последних 20 публикаций, а при равенстве берёт самый старый ожидающий пост. +- Для VK-постера user token нужно получать через VK ID Authorization Code Flow с PKCE: `/vk-oauth/start` -> `id.vk.ru/authorize` -> `/vk-oauth/callback`. Callback обменивает `code` через `id.vk.ru/oauth2/auth` на `access_token` + `refresh_token`, сохраняет `vk_poster_access_token`, `vk_poster_refresh_token`, `vk_poster_token_device_id`, `vk_poster_token_expires_at` и проверяет `wall.get` / `photos.getWallUploadServer`. Воркер обновляет access token через refresh token до истечения. Refresh token живёт 180 дней по документации VK ID; Redirect URI в VK ID-приложении: `https://sw.exostring.xyz/vk-oauth/callback`. +- По документации VK API метод `photos.getWallUploadServer` требует пользовательское право `photos`, которое VK выдаёт в исключительных случаях через запрос в поддержку `devsupport@corp.vk.com`. Если token без `photos`, VK-постер пропускает upload редакторских/URL-фото, но всё равно публикует текст и существующие VK `photo...`/`video...` attachments. +- VK-постер исключает из очереди video-only записи с `raw_post_media.status='link_only'`: VK API не разрешает прикреплять часть чужих видео без предварительной загрузки в целевое сообщество. Медиа `link_only` также не добавляются к смешанным постам, а публикации со статусом `publish_failed` не возвращаются автоматически в каждый следующий часовой слот и требуют явного ручного повтора. + +### 4.1. Секреты в `.env`, операционные настройки в БД + +В `.env` должны лежать только секреты и базовые параметры окружения: + +- `APP_SECRET_KEY` +- `ADMIN_BOOTSTRAP_LOGIN` +- `ADMIN_BOOTSTRAP_PASSWORD` +- параметры подключения к PostgreSQL +- `VK_ACCESS_TOKEN` +- `TG_BOT_TOKEN` +- и другие токены/пароли + +В БД, в таблице `app_settings`, лежат операционные настройки: + +- интервалы воркеров; +- batch size; +- лимиты текста; +- включение/выключение AI; +- выбор provider/model; +- prompt/contract; +- Telegram storage channel id; +- лимиты Telegram uploader; +- параметры парсинга. + +Причина: секреты опасно показывать в админке и хранить в обычных настройках, а операционные параметры удобно менять без деплоя. + +### 4.2. Worker control состоит из двух флагов + +Для AI-воркеров используется двойная защита: + +1. `worker_controls.enabled` +2. `app_settings.ai_*_enabled` + +Воркер начнёт работу только если включены оба. Это сделано специально, чтобы случайное включение одного тумблера не запускало расход денег на LLM. + +## 5. БД: ключевые таблицы + +### 5.1. `admin_users` + +Пользователи админки. + +Основные поля: + +- `id` +- `login` +- `password_hash` +- `is_active` +- `created_at` + +Первый пользователь создаётся из `ADMIN_BOOTSTRAP_LOGIN` и `ADMIN_BOOTSTRAP_PASSWORD`. + +### 5.2. `admin_sessions` + +Сессии админки. + +Хранит: + +- `token_hash` +- `user_id` +- `csrf_token` +- `expires_at` +- `created_at` + +Сырой session token в БД не хранится. + +### 5.3. `sources` + +Источники для парсинга. + +Сейчас реально используется только `platform='vk'`, но схема заложена под другие площадки. + +Важные поля: + +- `id` +- `platform` +- `name` +- `tag` +- `url` +- `external_id` +- `external_owner_id` +- `active` +- `status` +- `status_msg` +- `last_checked_at` +- `last_parsed_at` +- `parse_from` +- `priority` +- `archived_at` + +`tag` нужен для будущих хэштегов и финального оформления. + +### 5.4. `raw_posts` + +Главная таблица жизненного цикла поста. + +Основные поля raw: + +- `id` +- `source_id` +- `platform` +- `external_post_id` +- `external_owner_id` +- `original_url` +- `raw_text` +- `raw_json` +- `text_hash` +- `content_hash` +- `posted_at` +- `created_at` +- `updated_at` +- `status` +- `skip_reason` + +Storage-поля: + +- `storage_post_url` +- `tg_storage_chat_id` +- `tg_storage_thread_id` +- `tg_storage_message_ids` + +AI-квалификация: + +- `qualification_status` +- `qualification_score` +- `qualification_decision` +- `qualification_model_decision` +- `qualification_reason` +- `qualification_reject_tag` +- `qualification_model` +- `qualification_prompt_hash` +- `qualification_batch_id` +- `qualified_at` + +AI-райтер: + +- `rewrite_status` +- `rewritten_text` +- `rewrite_notes` +- `rewrite_model` +- `rewrite_prompt_hash` +- `rewrite_batch_id` +- `rewrite_category_id` +- `rewrite_category` +- `rewrite_category_tag` +- `rewrite_source_tag` +- `rewritten_at` + +Редакторская: + +- `editorial_status` +- `final_text` +- `final_category_id` +- `final_category` +- `final_category_tag` +- `final_source_tag` +- `editor_notes` +- `reviewed_by` +- `reviewed_at` +- `edited_at` + +### 5.5. `raw_post_media` + +Медиа исходного поста. + +Основные поля: + +- `id` +- `raw_post_id` +- `platform` +- `media_type` +- `original_url` +- `original_attachment_id` +- `preview_url` +- `storage_url` +- `storage_attachment_id` +- `tg_file_id` +- `tg_file_unique_id` +- `width` +- `height` +- `duration_sec` +- `sort_order` +- `status` +- `attempts` +- `error` +- `editor_hidden` +- `editor_added` +- `local_path` + +Удаление медиа в редакторской не обязано физически удалять старое медиа из Telegram. В редакторском контуре оно помечается как скрытое через `editor_hidden`. + +### 5.6. `jobs` + +Очередь фоновых задач. + +Основные поля: + +- `id` +- `type` +- `entity_type` +- `entity_id` +- `payload_json` +- `status` +- `attempts` +- `max_attempts` +- `next_run_at` +- `locked_by` +- `locked_at` +- `last_error` + +Сейчас основной тип: + +- `vk.storage.copy` + +Название историческое: сначала планировался VK storage, потом медиа-сторедж вернулся в Telegram. Тип задачи пока не переименован. + +### 5.7. `worker_controls` + +Включение/выключение воркеров. + +Поля: + +- `name` +- `enabled` +- `settings_json` +- `updated_by` +- `updated_at` + +Имена воркеров: + +- `vk-parser` +- `vk-storage-uploader` +- `ai-qualifier` +- `ai-writer` +- `tg-poster` +- `max-poster` + +### 5.8. `worker_heartbeats` + +Состояние воркеров для админки. + +Поля: + +- `name` +- `heartbeat_at` +- `status` +- `current_job_id` +- `meta_json` + +### 5.9. `app_settings` + +Настройки приложения. + +Поля: + +- `key` +- `value_json` +- `value_type` +- `title` +- `description` +- `category` +- `updated_at` + +Важно: `value_json` всегда JSONB, даже если настройка выглядит как строка. + +### 5.10. `ai_qualification_batches` + +История запросов AI-квалификатора. + +Хранит: + +- provider/model; +- prompt hash; +- полный prompt text; +- request JSON; +- response JSON; +- количество токенов; +- примерную стоимость; +- счётчики accepted/rejected/maybe; +- ошибку, если batch упал. + +### 5.11. `ai_writer_batches` + +История запросов AI-райтера. + +Хранит: + +- provider/model; +- prompt hash; +- полный prompt text; +- request JSON; +- response JSON; +- количество токенов; +- примерную стоимость; +- ready/failed count; +- ошибку. + +Важно: после фикса batch с ошибкой также должен сохранять `response_json`, если модель вернула валидный JSON неправильной структуры. + +### 5.12. `content_categories` + +Категории публикаций для AI-райтера и редакторской. + +Поля: + +- `id` - внутренний технический PK; +- `sort_order` - публичный стабильный ID категории для AI; +- `name` - человекочитаемое название/описание; +- `tag` - тэг, который будет подставляться системой; +- `is_active`; +- `created_at`; +- `updated_at`. + +Решение: `sort_order` используется как `category_id` для AI-райтера. В UI он отображается как `ID`, readonly. Модель возвращает только `category_id`, а код сам подставляет `name` и `tag` из БД. + +### 5.13. `audit_log` + +Журнал действий админки. + +Хранит: + +- `id`; +- `actor_id`; +- `action`; +- `entity_type`; +- `entity_id`; +- `before_json`; +- `after_json`; +- `created_at`. + +В `after_json` дополнительно пишется request-контекст, если он доступен: + +- `ip`; +- `method`; +- `path`; +- `query`; +- `user_agent`; +- `status_code` для request-level записей. + +Retention: записи старше 30 дней удаляются при старте приложения, при применении миграции `022_audit_log_retention_and_indexes.sql` и при открытии вкладки `/logs`. + +### 5.14. `publication_runs` + +Защита расписания постеров от повторной публикации одного и того же слота. + +Поля: + +- `poster` - тип постера, сейчас `tg`; +- `schedule_id` - ID строки расписания; +- `scheduled_for` - дата по Екб; +- `scheduled_time` - время строки расписания; +- `planned_count`; +- `published_count`; +- `status`; +- `error`. + +Уникальность `poster + schedule_id + scheduled_for` гарантирует, что слот расписания за день будет обработан один раз. + +Важно: TG-постер не должен публиковать все прошедшие за день слоты при включении. Он обрабатывает только текущий слот расписания в коротком окне после заданного времени и перед проверкой перечитывает настройки, включая `tg_poster_chat_id`. + +## 6. Статусы + +### 6.1. `raw_posts.status` + +Основные значения: + +- `storage_pending` - пост сохранён, ждёт загрузки в Telegram storage. +- `storage_ready` - пост и медиа загружены/сохранены в storage. +- `skipped` - пост сохранён как мусорный, если включено `parser_store_skipped_posts`. +- `failed` - ошибка на этапе storage. + +### 6.2. `raw_posts.qualification_status` + +Основные значения: + +- `pending` / `NULL` - ещё не квалифицирован. +- `processing` - взят AI-квалификатором. +- `accepted` - подходит для рерайта. +- `rejected` - мусор/не подходит. +- `maybe` - спорный по модели. Сейчас финальная логика может сводить `maybe` к accepted/rejected в зависимости от score и min_score. +- `failed` - ошибка AI-квалификации. + +### 6.3. `raw_posts.rewrite_status` + +Основные значения: + +- `pending` / `NULL` - ещё не переписан. +- `processing` - взят AI-райтером. +- `ready` - рерайт готов. +- `failed` - ошибка рерайта. + +### 6.4. `raw_posts.editorial_status` + +Основные значения: + +- `review` - на проверке редактора. +- `edited` - сохранены ручные правки. +- `accepted` - принято к публикации. +- `rejected` - отклонено. + +### 6.5. `raw_posts.publication_status` + +Статус финальной публикации. Сейчас используется TG-постером, позже может расшириться под VK/MAX/site. + +Основные значения: + +- `pending` - пост ещё не опубликован. +- `published` - пост опубликован в Telegram. +- `publish_failed` - публикация не удалась, причина лежит в `publication_error`. + +## 7. VK-парсер + +Файл: `src/vk_parser_app/workers/parser.py`. + +### 7.1. Что делает + +1. Берёт активные источники из `sources`. +2. Если источник ещё не resolved, вызывает VK API и получает: + - `external_id`; + - `external_owner_id`; + - нормальное имя. +3. Загружает посты со стены через VK API. +4. Определяет временное окно парсинга: + - если есть `last_parsed_at`, берёт его минус overlap; + - иначе если есть `parse_from`, берёт его; + - иначе берёт `now - parser_new_source_lookback_days`. +5. Убирает уже известные посты по `(source_id, external_post_id)`. +6. При включенном `parser_dedupe_content_hash` также убирает дубли по `content_hash`. +7. Применяет политику мусорных постов. +8. Сохраняет raw post и media. +9. Создаёт job `vk.storage.copy`. + +### 7.2. Политика мусорных постов + +Настройки: + +- `parser_skip_empty_text` +- `parser_skip_no_media` +- `parser_skip_text_too_short` +- `parser_min_text_length` +- `parser_skip_reposts` +- `parser_store_skipped_posts` + +Если пост не проходит фильтр: + +- при `parser_store_skipped_posts=false` он вообще не сохраняется; +- при `true` сохраняется со статусом `skipped` и `skip_reason`. + +Возможные `skip_reason`: + +- `empty_text` +- `no_media` +- `text_too_short` + +### 7.3. Дедупликация + +Есть два уровня: + +1. Точный дубль по `source_id + external_post_id`. +2. Контентный дубль по `content_hash`, если включен `parser_dedupe_content_hash`. + +`content_hash` считается по тексту и media attachment ids. + +### 7.4. Настройки VK-парсера + +Ключевые: + +- `parser_interval_sec` +- `parser_new_source_lookback_days` +- `parser_reparse_overlap_minutes` +- `parser_min_text_length` +- `parser_skip_empty_text` +- `parser_skip_no_media` +- `parser_skip_text_too_short` +- `parser_skip_reposts` +- `parser_store_skipped_posts` +- `parser_dedupe_content_hash` +- `parser_source_pause_sec` +- `vk_requests_per_second` +- `vk_wall_page_size` +- `vk_api_timeout_total_sec` +- `vk_api_timeout_connect_sec` +- `vk_rate_limit_sleep_sec` +- `vk_api_retry_attempts` +- `vk_api_retry_min_delay_sec` +- `vk_api_retry_max_delay_sec` + +## 8. Media storage uploader + +Файл: `src/vk_parser_app/workers/vk_storage_uploader.py`. + +Название файла историческое: сейчас фактическое хранилище медиа - Telegram, а не VK. + +### 8.1. Что делает + +1. Забирает job типа `vk.storage.copy`. +2. Загружает `raw_post` и media. +3. Скачивает изображения/видео. +4. Отправляет в Telegram storage channel. +5. Сохраняет Telegram file ids и message ids. +6. Помечает post как `storage_ready`. + +### 8.2. Почему Telegram storage + +Изначально рассматривался VK storage через приватную группу VK. От идеи отказались, потому что: + +- права VK API и редактирование постов создают лишнюю сложность; +- stable raw проще держать через Telegram bot API; +- старый проект уже имел рабочую логику Telegram uploader; +- Telegram file ids удобны для повторного отображения и хранения. + +### 8.3. Ограничения Telegram + +Uploader учитывает: + +- media group limit до 10 элементов; +- caption limit; +- message limit; +- flood control через `TelegramRetryAfter`; +- retry/backoff; +- размер видео; +- длительность видео; +- отдельные задержки между media upload и job. + +### 8.4. Текст и ссылка на оригинал + +Логика должна стремиться к такому формату: + +- медиа и текст отправляются вместе, если текст влезает в caption; +- если текст не влезает, текст отправляется отдельным сообщением/частями; +- в конец добавляется ссылка на оригинал; +- `#id` должен быть ссылкой на оригинальный VK-пост. + +### 8.5. Временные файлы + +Временные файлы uploader должны жить в `/tmp` с префиксом `vkparser_tg_media_`. Для видео используется `yt-dlp` и временные файлы/netrc. Нужно регулярно проверять, что временные файлы удаляются после upload/failure. Это важный пункт техдолга. + +### 8.6. Настройки uploader + +Исторически часть ключей в коде называется `telegram_*`, а часть миграций/админки - `media_*`, `tg_*`, `uploader_*`. + +Это стоит привести к единому виду. Сейчас важно проверять конкретный ключ в коде. + +Ключевые настройки: + +- `tg_media_channel_id` +- `local_bot_api_url` +- `uploader_interval_sec` +- `uploader_download_timeout_sec` +- `media_group_max_items` +- `media_upload_delay_sec` +- `media_post_job_pause_sec` +- `max_media_attempts` +- `tg_retry_attempts` +- `tg_retry_backoff_max_sec` +- `video_max_size_mb` +- `video_max_duration_sec` +- `uploader_yt_dlp_timeout_sec` + +Потенциальное расхождение: в коде встречаются `telegram_media_group_max_items`, `telegram_media_upload_delay_sec`, `telegram_retry_max_attempts`, `telegram_retry_backoff_max_sec`, `telegram_caption_limit`, `telegram_message_limit`, `telegram_text_overflow_marker`, `uploader_max_media_attempts`. Это надо унифицировать. + +## 9. AI-квалификатор + +Файл: `src/vk_parser_app/workers/ai_qualifier.py`. + +### 9.1. Задача + +Отсеять мусорные или неподходящие raw-посты до рерайта. + +Примеры мусора: + +- мемы; +- политика; +- вакансии; +- низкоконтентные посты; +- скидка без полезной новости; +- оффтоп; +- неправильный язык; +- injection/попытка управлять моделью; +- оружие, если оно не подходит под редакционную политику. + +### 9.2. Вход в модель + +AI получает JSON-массив: + +```json +[ + { + "id": 123, + "source": "Название источника", + "original_url": "https://vk.com/wall-...", + "media_count": 3, + "media_types": ["photo"], + "text": "исходный текст" + } +] +``` + +Текст обрезается по `ai_qualifier_max_text_chars`. + +### 9.3. Prompt + contract + +Промпт разделён на две части: + +1. `ai_qualifier_prompt` - свободная редакционная инструкция. +2. `ai_qualifier_contract` - технический контракт JSON. + +Контракт требует: + +```json +{ + "results": [ + { + "id": 123, + "score": 8, + "decision": "accepted", + "reason": "до 10 слов на русском", + "reject_tag": null + } + ] +} +``` + +### 9.4. Настройки + +- `ai_qualifier_enabled` +- `ai_qualifier_provider` +- `ai_qualifier_model` +- `ai_qualifier_api_key` +- `ai_qualifier_api_base` +- `ai_qualifier_prompt` +- `ai_qualifier_contract` +- `ai_qualifier_batch_size` +- `ai_qualifier_min_score` +- `ai_qualifier_max_text_chars` +- `ai_qualifier_temperature` +- `ai_qualifier_timeout_sec` +- `ai_qualifier_interval_sec` + +### 9.5. Деньги и токены + +`ai_qualification_batches` хранит: + +- `prompt_tokens` +- `completion_tokens` +- `total_tokens` +- `estimated_cost_usd` + +Эта информация нужна, чтобы контролировать стоимость экспериментов с prompt/model. + +## 10. AI-райтер + +Файл: `src/vk_parser_app/workers/ai_writer.py`. + +### 10.1. Задача + +Взять посты: + +- `status='storage_ready'` +- `qualification_status='accepted'` +- `rewrite_status is NULL/pending/failed` + +И переписать их в короткую публикацию для Telegram/VK-канала. + +### 10.2. Вход в модель + +AI получает JSON-объект: + +```json +{ + "task": "rewrite_accepted_posts", + "categories": [ + { + "id": 1, + "name": "Одежда", + "tag": "одежда" + }, + { + "id": 2, + "name": "Снаряжение - подсумки, чехол для плит, нагрудники, боевые пояса и тд", + "tag": "снаряжение" + } + ], + "posts": [ + { + "id": 123, + "producer_name": "academy_gear", + "producer_tag": "academy_gear", + "original_url": "https://vk.com/wall-...", + "qualification_score": 8, + "qualification_reason": "...", + "media_count": 4, + "media_types": ["photo"], + "text": "исходный текст" + } + ] +} +``` + +Текст обрезается по `ai_writer_max_text_chars`. + +### 10.3. Prompt + contract + +Промпт разделён на: + +1. `ai_writer_prompt` - редакционная инструкция. +2. `ai_writer_contract` - технический JSON-контракт. + +Свободная часть описывает: + +- роль редактора; +- знания в тактическом/милитари/airsoft-снаряжении; +- структуру поста; +- стиль; +- запрет на выдумывание фактов; +- обязательное явное упоминание `producer_name` в тексте каждого рерайта; +- выбор категории. + +Технический контракт требует: + +```json +{ + "rewrites": [ + { + "id": 123, + "category_id": 2, + "text": "готовый текст без ссылок и хэштегов", + "notes": "короткая заметка для редактора или null" + } + ] +} +``` + +Важно: модель возвращает `category_id`, а не тэг и не название. Код сам берёт категорию из `content_categories` и сохраняет: + +- `rewrite_category_id` +- `rewrite_category` +- `rewrite_category_tag` + +### 10.4. Почему category_id, а не category + +Название категории может быть длинным и описательным: + +```text +Снаряжение - подсумки, чехол для плит, нагрудники, боевые пояса и тд +``` + +Модели легче вернуть число `2`, чем точно повторить длинную строку. Это снижает ошибки валидации и позволяет менять описание категории без переписывания логики. + +### 10.5. Категории + +Категории редактируются в `/workers`, блок "Категории публикаций". + +Поля: + +- `Название` +- `Тэг` +- `ID` +- `Статус` + +`ID` отображается как readonly. В БД это поле `content_categories.sort_order`. + +### 10.6. Настройки + +- `ai_writer_enabled` +- `ai_writer_provider` +- `ai_writer_model` +- `ai_writer_api_key` +- `ai_writer_api_base` +- `ai_writer_prompt` +- `ai_writer_contract` +- `ai_writer_categories` - legacy, сейчас основной источник `content_categories` +- `ai_writer_batch_size` +- `ai_writer_max_text_chars` +- `ai_writer_temperature` +- `ai_writer_timeout_sec` +- `ai_writer_interval_sec` + +Текущая рекомендация: держать `ai_writer_batch_size=1`, потому что редактировать и отлаживать результат по одному посту проще и безопаснее. + +### 10.7. Последний проверенный тест + +Один контролируемый прогон `ai-writer`: + +- batch `#6` +- `posts_count=1` +- `status=done` +- model `anthropic/claude-haiku-4-5-20251001` +- результат для post `#26` +- `category_id=1` +- `rewrite_category=Одежда` +- `rewrite_category_tag=одежда` +- `rewrite_status=ready` +- `editorial_status=review` +- стоимость около `$0.007296` + +После теста writer был возвращён в безопасное состояние: + +- `ai_writer_enabled=false` +- `ai_writer_batch_size=1` +- `worker_controls.ai-writer.enabled=false` +- `ai-writer.service=active`, но работу не берёт. + +## 11. Prompt-test + +Страница: `/prompt-test`. + +Назначение: песочница для проверки prompt райтера на существующем raw-посте. + +Важно: + +- сейчас тестируется именно `ai_writer_prompt`; +- `ai_writer_contract` намеренно не добавляется в тест, чтобы можно было свободно смотреть стиль; +- входной payload должен совпадать с боевым форматом райтера; +- можно выбрать пост по фильтру квалификации; +- ответ отображается как человеческий текст с нормальными переносами, плюс raw JSON. + +Риск: если тестировать prompt без contract, модель может не вернуть JSON. Это нормально для песочницы, но боевой worker всегда склеивает prompt + contract. + +## 11.5. TG poster + +Файл: `src/vk_parser_app/workers/tg_poster.py`. + +Назначение: брать посты, принятые редактором, и публиковать их в Telegram по расписанию. Это первый финальный poster; позже рядом должны появиться VK/MAX/site poster-воркеры. + +### 11.5.1. Условия публикации + +Пост берётся в очередь TG-постера, если: + +- `raw_posts.status='storage_ready'`; +- `rewrite_status='ready'`; +- `editorial_status='accepted'`; +- `publication_status='pending'`. + +После успеха: + +- `publication_status='published'`; +- `editorial_status='published'`, поэтому пост уходит из вкладки "Принято"; +- заполняются `published_at`, `tg_publication_chat_id`, `tg_publication_thread_id`, `tg_publication_message_ids`, `tg_publication_url`. + +После ошибки: + +- `publication_status='publish_failed'`; +- `editorial_status='publish_failed'`; +- причина пишется в `publication_error`; +- автоматический повтор не делается, чтобы ошибка не перетиралась на каждом слоте расписания. + +### 11.5.2. Настройки + +Категория настроек: `TG Poster`. + +Ключевые: + +- `tg_poster_enabled` - второй защитный флаг, синхронизируется с кнопкой worker toggle; +- `tg_poster_bot_token` - отдельный токен бота для финальной публикации; если пустой, worker берёт `TG_BOT_TOKEN`; +- `tg_poster_chat_id` - куда публиковать, для теста `-1003712724327`; +- `tg_poster_schedule_json` - строки расписания; +- `tg_poster_interval_sec`; +- `tg_poster_caption_limit`; +- `tg_poster_message_limit`; +- `tg_poster_media_group_max_items`; +- `tg_poster_max_attempts`; +- `tg_poster_retry_backoff_max_sec`; +- `tg_poster_send_delay_sec`; +- `tg_poster_text_overflow_caption`; +- `tg_poster_recent_window`; +- `tg_poster_category_repeat_penalty`; +- `tg_poster_source_repeat_penalty`. + +Расписание редактируется на `/workers` отдельным блоком "Расписание TG-постера". Время указывается по Екатеринбургу (`Asia/Yekaterinburg`). Сейчас по умолчанию стоит 24 строки: каждый час `HH:00`, по 4 поста. + + +### 11.5.3. Telegram-лимиты + +Worker учитывает: + +- caption limit до 1024 символов; +- message limit до 4096 символов с разбиением длинного текста на части; +- media group limit до 10 элементов; +- одиночное медиа отправляется через `send_photo`/`send_video`, а не через album API; +- flood control через `TelegramRetryAfter`; +- retry/backoff; +- задержку между отправками. + +Если текст вместе с хэштегами влезает в caption, он публикуется под первым медиа. Если не влезает, caption становится `⬇️ Описание`, а полный текст публикуется следующим сообщением. + +### 11.5.4. Формат текста + +Финальный текст собирается системно: + +```text +пост + +#category #source +``` + +Категорийный хэштег берётся из `content_categories.tag`, source-тэг - из `final_source_tag`, `rewrite_source_tag` или `sources.tag`. + +### 11.5.5. Ранжирование + +Логика intentionally простая: + +1. Берём кандидатов из принятых неопубликованных постов. +2. Базовый порядок: выше `qualification_score`, затем более старые принятые посты. +3. Смотрим последние `tg_poster_recent_window` опубликованных постов. +4. Штрафуем кандидата за повторы категории и источника среди недавних публикаций. +5. При выборе пачки дополнительно штрафуем повторы внутри текущего слота. + +Так категории и источники реже идут подряд, но алгоритм остаётся понятным и предсказуемым. + +## 12. Админка + +Основной файл: `src/vk_parser_app/admin.py`. + +### 12.1. Основные страницы + +- `/login` - вход. +- `/sources` - источники парсинга. +- `/raw` - raw-посты. +- `/raw/{id}` - подробная карточка raw-поста, AI-ответы, batch info. +- `/editor` - редакторская лента. +- `/workers` - воркеры, настройки, категории публикаций, расписание TG-постера. +- `/logs` - журнал действий, запросов и служебных событий админки. +- `/users` - пользователи. +- `/prompt-test` - тест промпта райтера. +- `/vk/oauth/callback` - callback для VK OAuth, исторически нужен для экспериментов с VK токенами. + +### 12.2. Sources + +На странице источников можно: + +- добавлять источник; +- добавлять пачкой; +- редактировать; +- включать/выключать; +- архивировать; +- видеть статус и ошибки. + +Для VK важны: + +- platform `vk` +- name +- tag +- url +- active +- parse_from + +### 12.3. Raw + +Raw-страница показывает: + +- источник; +- этап; +- пост; +- AI-оценку; +- медиа-превью; +- дату; +- фильтры справа. + +Фильтры построены как sidebar в стиле интернет-магазина: + +- score min/max; +- дата; +- источник; +- категория; +- raw status; +- qualification status; +- rewrite status. + +### 12.4. Editor + +Редакторская нужна для постов после AI-райтера. + +Возможности: + +- посмотреть посты на проверке; +- принять; +- отклонить; +- открыть редактирование; +- изменить текст; +- поменять категорию; +- поменять source tag; +- скрыть/добавить медиа; +- сохранить правки; +- принять к публикации. + +Текущая логика: + +- `Сохранить правки` сохраняет изменения, но не принимает пост. +- `Принять к публикации` сохраняет и переводит в `accepted`. + +### 12.5. Workers + +Страница `/workers` содержит: + +- список воркеров и heartbeat; +- категории публикаций; +- расписание TG-постера; +- настройки по разделам. + +Prompt-настройки имеют подсказки: + +- что можно менять безопасно; +- что приходит на вход; +- базовая JSON-схема. + +### 12.6. Logs + +Страница `/logs` показывает `audit_log` за последние 30 дней. + +Что логируется: + +- успешные, неуспешные и заблокированные входы; +- выходы; +- GET/POST-запросы авторизованной админки, кроме `/health`, favicon и `/uploads`; +- существующие предметные действия: источники, редакторская, категории, воркеры, настройки, пользователи, prompt-test. + +Фильтры: + +- поиск по action/entity/JSON; +- action; +- пользователь; +- entity type; +- диапазон дат. + +Старые записи чистятся автоматически, чтобы журнал не рос бесконечно. + +## 13. Безопасность админки + +Сейчас реализовано: + +- password hash; +- session token hash; +- CSRF token; +- таблица `admin_login_attempts`; +- ограничение попыток входа; +- users management; +- fail2ban/ufw на сервере были добавлены в рамках настройки сервера. + +Что важно помнить: + +- API keys сейчас могут отображаться в админке целиком для удобства. Нельзя открывать страницу настроек при демонстрации экрана. +- Нужно держать `.env` вне git. +- Нужно регулярно менять временные пароли. +- Root SSH по паролю лучше заменить на ключи и отключить password login. + +## 14. Миграции + +Миграции лежат в `db/migrations`. + +Применение: + +```bash +cd /opt/vk-parser +PYTHONPATH=src .venv/bin/python scripts/apply_migrations.py +``` + +Механика: + +- `schema_migrations` хранит имена применённых файлов; +- файлы выполняются по алфавитному порядку; +- каждая миграция выполняется в transaction; +- повторно применённая миграция пропускается. + +Последние важные миграции: + +- `017_content_categories_and_media_cleanup.sql` - категории публикаций. +- `018_writer_prompt_contract_cleanup.sql` - JSON-only вынесен из prompt в contract. +- `019_writer_prompt_profile_style.sql` - новый подробный prompt райтера. +- `020_writer_category_ids.sql` - `category_id` для AI-райтера, новые поля в raw_posts. +- `021_writer_contract_no_empty_response.sql` - усиленный writer contract и сохранение неправильного ответа. +- `024_tg_poster.sql` - TG-постер, статусы публикации, расписание, лимиты и prompt-лимит райтера под caption. +- `026_writer_separator_17_chars.sql` - строгая разделительная полоса `━━━━━━━━━━━━━━━━━` в prompt/contract и существующих рерайтах. +- `031_writer_require_producer_name.sql` - AI-райтер обязан явно упоминать `producer_name` в тексте каждого рерайта. +- `032_tg_reactor.sql` - TG-реактор, таблица `post_reactions`, настройки реакций и worker flag. +- `033_tg_reactor_since.sql` - настройка `tg_reactor_since`, чтобы реактор не проходил по старым TG-публикациям при включении. +- `034_tg_reactor_reaction_pause.sql` - пауза между успешными реакциями TG-реактора, чтобы безопасно догонять архив. +- `035_uploader_video_format_limits.sql` - ограничение высоты VK-видео для `yt-dlp` и увеличение timeout скачивания, чтобы ролики укладывались в Telegram Bot API. +- `039_max_poster.sql` - MAX-постер, расписание как у TG, настройки MAX Bot API, лимиты сообщения/медиа, ожидание обработки видео и double safety flags. + +## 15. Локальный запуск + +Установка: + +```bash +python -m venv .venv +.venv\Scripts\activate +pip install -r requirements.txt +``` + +Для Windows PowerShell: + +```powershell +$env:PYTHONPATH="src" +python scripts/apply_migrations.py +uvicorn vk_parser_app.admin:app --host 0.0.0.0 --port 8080 +``` + +Воркеры локально: + +```powershell +$env:PYTHONPATH="src" +python -m vk_parser_app.workers.parser +python -m vk_parser_app.workers.vk_storage_uploader +python -m vk_parser_app.workers.ai_qualifier +python -m vk_parser_app.workers.ai_writer +python -m vk_parser_app.workers.tg_poster +python -m vk_parser_app.workers.max_poster +``` + +На Windows при `py_compile` может быть ошибка доступа к `__pycache__`. Тогда: + +```powershell +$env:PYTHONPYCACHEPREFIX="$env:TEMP\n8_pycache" +python -m py_compile src\vk_parser_app\admin.py +``` + +## 16. Сервер + +Рабочий путь: + +```bash +/opt/vk-parser +``` + +Основные команды: + +```bash +cd /opt/vk-parser +git pull --ff-only +PYTHONPATH=src .venv/bin/python scripts/apply_migrations.py +systemctl restart vk-parser-admin.service +``` + +Проверка сервисов: + +```bash +systemctl is-active vk-parser-admin.service +systemctl is-active vk-parser.service +systemctl is-active vk-storage-uploader.service +systemctl is-active ai-qualifier.service +systemctl is-active ai-writer.service +systemctl is-active tg-poster.service +systemctl is-active max-poster.service +systemctl is-active telegram-bot-api.service +``` + +Логи: + +```bash +journalctl -u vk-parser-admin.service -n 100 --no-pager +journalctl -u vk-parser.service -n 100 --no-pager +journalctl -u vk-storage-uploader.service -n 100 --no-pager +journalctl -u ai-qualifier.service -n 100 --no-pager +journalctl -u ai-writer.service -n 100 --no-pager +journalctl -u tg-poster.service -n 100 --no-pager +journalctl -u max-poster.service -n 100 --no-pager +journalctl -u telegram-bot-api.service -n 100 --no-pager +``` + +## 17. Git workflow + +Локальный репозиторий: + +```bash +git status +git add ... +git commit -m "Message" +git push +``` + +На сервере origin настроен через deploy key. + +Обычный деплой: + +```bash +cd /opt/vk-parser +git pull --ff-only +PYTHONPATH=src .venv/bin/python scripts/apply_migrations.py +systemctl restart vk-parser-admin.service +``` + +Если менялся конкретный worker, перезапустить его: + +```bash +systemctl restart ai-writer.service +systemctl restart ai-qualifier.service +systemctl restart vk-parser.service +systemctl restart vk-storage-uploader.service +``` + +## 18. Контролируемый тест AI-райтера + +Важно: не включать AI writer обычным способом, если нужно проверить один пост. Иначе живой systemd service может начать брать очередь самостоятельно. + +Правильная схема: + +1. Остановить сервис: + +```bash +systemctl stop ai-writer.service +``` + +2. Временно включить: + +- `worker_controls.ai-writer.enabled=true` +- `app_settings.ai_writer_enabled=true` +- `app_settings.ai_writer_batch_size=1` + +3. Выполнить один `AIWriterWorker.run_once()`. + +4. Вернуть: + +- `worker_controls.ai-writer.enabled=false` +- `app_settings.ai_writer_enabled=false` +- `app_settings.ai_writer_batch_size=1` + +5. Запустить сервис обратно: + +```bash +systemctl start ai-writer.service +``` + +Так service остаётся active, но в disabled-режиме. + +## 19. Контролируемый тест AI-квалификатора + +Схема такая же: + +1. Остановить `ai-qualifier.service`. +2. Включить оба флага. +3. Поставить нужный `ai_qualifier_batch_size`. +4. Выполнить `AIQualifierWorker.run_once()`. +5. Вернуть настройки. +6. Запустить service обратно. + +Важно: при batch size 25 и активном service можно случайно получить несколько batch подряд. Так уже происходило. Поэтому сначала останавливать service. + +## 20. Известные хвосты и техдолг + +### 20.1. Старые AI writer batches + +В истории есть старые тестовые batches: + +- batch `#4` со статусом `processing`; +- batch `#5` со статусом `failed`, созданный во время теста на 5 постов. + +Они не влияют на текущую очередь, но могут путать в админке. Их стоит аккуратно пометить как failed/cancelled или скрывать служебные тесты. + +### 20.2. Название `vk_storage_uploader.py` + +Файл исторически называется VK storage uploader, но фактически загружает в Telegram. Можно позже переименовать: + +```text +vk_storage_uploader.py -> tg_storage_uploader.py +JOB_TYPE_VK_STORAGE_COPY -> tg.storage.copy +``` + +Делать только миграцией и аккуратно, чтобы не потерять jobs. + +### 20.3. Настройки Telegram uploader + +Часть ключей в UI/миграциях и в коде отличается по именам. Нужно унифицировать, чтобы админка реально управляла всеми параметрами uploader. + +### 20.4. README устарел + +`README.md` был написан на раннем этапе, когда ещё рассматривался VK storage. Сейчас он полезен как quick start, но его надо обновить или заменить ссылкой на `PROJECT.md`. + +### 20.5. Медиа cleanup + +Нужно отдельно проверить: + +- удаляются ли временные файлы из `/tmp`; +- что происходит с локально загруженными медиа редактора; +- как чистить orphan files; +- как чистить Telegram storage, если редактор удалил медиа. + +### 20.6. Категории и старые посты + +Новые посты получают `rewrite_category_id`. Старые посты могут иметь только текстовую категорию. Это нормально, но фильтры/отображение должны учитывать оба варианта. + +### 20.7. Prompt-test без contract + +Это осознанное решение. Но если хочется тестировать боевой формат, стоит добавить переключатель: + +- "тестировать только стиль" +- "тестировать style + contract" + +## 21. Что делать дальше + +Ближайшие полезные задачи: + +1. Обновить `README.md`, чтобы он не противоречил реальности. +2. Унифицировать ключи настроек Telegram uploader. +3. Добавить отдельную страницу batch history для AI. +4. Аккуратно закрыть/пометить старые зависшие AI writer batches. +5. Улучшить отображение категорий в editor/raw: + - показывать `category_id`; + - показывать `tag`; + - показывать source tag. +6. Добавить режим "боевой prompt-test" с contract. +7. Добавить будущие постеры рядом с TG-постером: + - VK; + - MAX; + - сайт. +8. Добавить backup strategy: + - PostgreSQL dump; + - `.env` отдельно; + - uploads/editor_media. +9. Навести порядок в UI, если админка кажется слишком самописной: + - либо продолжить текущий server-rendered UI; + - либо вынести frontend в React/shadcn; + - либо подключить готовую admin-систему. + +## 22. Короткий mental model + +Если нужно быстро вспомнить проект: + +```text +sources + -> parser + -> raw_posts + raw_post_media + -> jobs(vk.storage.copy) + -> Telegram storage uploader + -> status=storage_ready + -> AI qualifier + -> qualification_status=accepted/rejected + -> AI writer + -> rewrite_status=ready, editorial_status=review + -> editor + -> accepted/rejected/edited + -> tg-poster + -> publication_status=published/publish_failed +``` + +Главные таблицы: + +```text +sources +raw_posts +raw_post_media +jobs +worker_controls +worker_heartbeats +app_settings +publication_runs +ai_qualification_batches +ai_writer_batches +content_categories +admin_users +admin_sessions +``` + +Главные файлы: + +```text +src/vk_parser_app/admin.py +src/vk_parser_app/workers/parser.py +src/vk_parser_app/workers/vk_storage_uploader.py +src/vk_parser_app/workers/ai_qualifier.py +src/vk_parser_app/workers/ai_writer.py +src/vk_parser_app/workers/tg_poster.py +src/vk_parser_app/vk_api.py +db/migrations/*.sql +``` + +Главное правило эксплуатации: + +```text +Перед тестом AI-воркера останавливай systemd service, +иначе живой worker может параллельно забрать очередь. +``` + +## 23. Сайт Forma N8 на Ghost + +21 июня 2026 года лента сайта перенесена на Ghost CMS 6.46.0. Это основа +для дальнейшей работы над сайтом, SEO и отдельным site-poster worker. + +### 23.1. Сервер и домен + +- VPS: `139.100.234.51`; +- SSH: `root@139.100.234.51:22`, пароль `7LaoHsQ1JxnK`; +- подключение к этому серверу выполнять из Python через `paramiko`; +- домен: `f-n8.ru`; +- Ghost: `/var/www/f-n8.ru`; +- systemd: `ghost_f-n8-ru.service`; +- nginx vhost: `/etc/nginx/sites-available/f-n8.ru`; +- MySQL database: `ghost_prod`; +- публичный Ghost порт слушает только `127.0.0.1:2368`. + +Сервер основного парсера доступен по SSH: `root@109.120.156.205:22`, +пароль `OU8gUSCtQYcf`. Подключение к обоим серверам выполняется через +Python-библиотеку `paramiko`. + +23 июня 2026 года устранён 502 после автоматического обновления MySQL: +`unattended-upgrade` остановил MySQL, а жёсткая зависимость +`Requires=mysql.service` вслед за ним остановила Ghost без последующего +запуска. В `ghost_f-n8-ru.service` зависимость заменена на +`Wants=mysql.service` при сохранённом `After=mysql.service`; unit включён и +использует `Restart=always`. Резервная копия исходного unit: +`/etc/systemd/system/ghost_f-n8-ru.service.bak-20260623-1115`. + +25 июня 2026 года из основного CSS Ghost Admin удалены 21 внешних +`@import` с `fonts.bunny.net`: CDN сбрасывал соединения у части клиентов, +из-за чего `/ghost/` оставался на бесконечной загрузке. В `index.html` к +административному CSS добавлен cache-busting query +`?v=localfonts-20260625`; резервная копия находится в +`/root/forma-n8-backups/ghost-admin-fonts-20260624-190120/`. Обновление +Ghost может перезаписать этот патч, поэтому после обновлений нужно проверить +наличие `fonts.bunny.net` в `current/core/built/admin/assets/*.css`. + +В тот же день отключена проверка нового устройства для staff-входов: +`security.staffDeviceVerification=false` в +`/var/www/f-n8.ru/config.production.json`. Без рабочего SMTP эта проверка +завершалась ошибкой `Failed to send email` и не позволяла новым +администраторам войти. Резервная копия конфигурации: +`/var/www/f-n8.ru/config.production.json.bak-device-verification-20260624-190851`. + +22 июня 2026 года DNS начал разрешаться публично: `f-n8.ru` и +`www.f-n8.ru` указывают на `139.100.234.51`. Выпущен сертификат Let's +Encrypt для обоих имён, включено автоматическое обновление через +`certbot.timer`, HSTS и принудительный HTTPS. Основной адрес Ghost и +canonical URL: `https://f-n8.ru`; `www` перенаправляется на основной домен. + +Для подтверждения сайта в Яндекс Вебмастере nginx отдаёт статический файл +`https://f-n8.ru/yandex_9c5e1348b34e9b34.html`; исходный файл хранится в +`/var/www/f-n8.ru/yandex_9c5e1348b34e9b34.html`, exact-location настроен в +`/etc/nginx/sites-available/f-n8.ru`. + +### 23.2. Тема и контент + +- исходники темы в репозитории: `forma_n8_site/ghost-theme/forma-n8`; +- установленная тема: `/var/www/f-n8.ru/content/themes/forma-n8`; +- тема проходит Ghost gscan без ошибок; +- favicon и кириллические Roboto Condensed лежат внутри темы; +- постоянный URL публикаций: `/news/{slug}/`; +- категории и производители переносятся в Ghost tags; +- RSS: `/rss/`; +- sitemap: `/sitemap.xml`; +- админка: `/ghost/`. + +В Ghost перенесены 12 опубликованных материалов старой ленты с исходными +slug, датами, SEO-полями и локальными изображениями. Стартовые публикации +Ghost удалены. Старый FastAPI-сайт и PostgreSQL пока сохранены как резерв +на сервере; предмиграционный backup находится в +`/root/forma-n8-backups/20260621-172603/`. + +### 23.3. Доступы и будущий site-poster + +Секреты не хранятся в git: + +- MySQL credentials: `/root/ghost-db.env`; +- initial owner credentials: `/root/ghost-owner-credentials`; +- Admin API key интеграции `Forma N8 Site Poster`: + `/root/ghost-admin-api-key`. + +Site-poster использует Ghost Admin API key, создаёт публикации через Admin +API, загружает медиа в Ghost и записывает внешний Ghost post id/url в общую +таблицу публикаций. Прямые INSERT в Ghost posts для рабочего постинга +запрещены. + +### 23.4. Категории сайта + +Миграция `037_site_category_fields.sql` разделяет три разных назначения +категории: + +- `name` — подробное описание категории для AI; +- `tag` — неизменяемый тэг для хэштега в соцсетях; +- `site_name` — тэг с заменой `_` на пробел и нормальным регистром; +- `site_slug` — стабильный латинский хвост URL; +- `site_enabled` — разрешение публикации категории на сайт. + +Категория `18` / `НЦК` имеет `site_enabled=FALSE`. Архивы Ghost используют +URL `/category/{site_slug}/`; URL публикации не содержит категорию и +остаётся `/news/{slug}/`, чтобы смена категории не ломала canonical. +Производители хранятся во внутренних Ghost tags и не попадают в sitemap +как категории. + +### 23.5. Изображения и аналитика + +В ленте показывается только `feature_image`. Если у публикации есть +дополнительные изображения, site-poster должен добавить их в тело статьи +нативной Ghost Gallery; тема поддерживает плитку и полноэкранный просмотр. + +Перед загрузкой изображения нужно: + +- применить EXIF orientation и удалить метаданные; +- ограничить размер до 2000x2000 без увеличения; +- сохранить в WebP quality 82; +- первое изображение назначить обложкой, остальные поместить в галерею. + +Контрольный замер на первых 12 публикациях: исходные изображения занимали +12,09 МБ, после такой обработки — 3,18 МБ, в среднем 272 КБ на файл. + +В тему добавлен счётчик Яндекс Метрики `110060335` с clickmap, +trackLinks, accurateTrackBounce и webvisor. +Google Tag Manager подключён в общем Ghost-шаблоне с контейнером +`GTM-MM4FR8LJ`: script находится в начале ``, noscript iframe — сразу +после открывающего ``. + +### 23.6. Ручной batch site-poster + +Скрипт `scripts/publish_site_batch.py` публикует явно выбранные raw-посты +через Ghost Admin API: + +```bash +PYTHONPATH=/opt/vk-parser/src .venv/bin/python \ + scripts/publish_site_batch.py --ids 26,455 +``` + +Скрипт: + +- не публикует `site_enabled=FALSE` и уже опубликованные на сайт записи; +- скачивает сохранённые фото через локальный Telegram Bot API; +- применяет EXIF orientation, 2000x2000, WebP quality 82; +- назначает первое фото `feature_image`; +- собирает остальные фото в Ghost Gallery; +- создаёт статью через Admin API; +- записывает результат или ошибку в `post_publications` с platform `site`. + +Первый боевой batch 22 июня 2026 года: 10 статей, 46 фотографий, +все публикации успешны. Upload payload после нормализации занял около +10,9 МБ. Ghost штатно хранит рабочую версию и исходную WebP-копию `_o` +для последующей регенерации размеров: суммарно 92 файла / 21,12 МБ на диске. + +В тот же день выполнен полный photo-ready backfill. В `post_publications` +создан backfill для 12 ранних импортов, после чего опубликован весь +оставшийся хвост без дублей. Итог: + +- 310 Ghost posts / 310 `site · published`; +- 0 `site · publish_failed`; +- 0 оставшихся photo-ready постов; +- 16 video-only постов опубликованы через нативные Ghost video cards; +- 5 длинных VK-видео остаются `link_only / video too long` без Telegram + file_id и не публикуются, чтобы на сайте не было карточек без медиа; +- video pipeline загружает MP4 через Ghost `/media/upload/`, извлекает + WebP-обложку ffmpeg и хранит ролик в нативном Lexical `video` node; +- ролики больше 40 МБ или выше 720p перекодируются в H.264/AAC, + максимум 720p, target около 38 МБ, preset `fast`, `faststart`; +- после photo backfill каталог Ghost images занимал около 453 МБ; + актуальный общий расход проверяется отдельно с учётом MP4. + +Режим полного backfill: + +```bash +PYTHONPATH=/opt/vk-parser/src .venv/bin/python \ + scripts/publish_site_batch.py --all-ready +``` + +Он исключает уже опубликованные записи и ранжирует очередь так, чтобы +категории и источники реже повторялись подряд. + +### 23.6.1. Автоматический site-poster + +- worker: `src/vk_parser_app/workers/site_poster.py`; +- systemd: `site-poster.service`; +- двойное включение: `worker_controls.site-poster.enabled` и + `app_settings.site_poster_enabled`; +- интервал по умолчанию: 300 секунд; +- за проход: до 10 photo-ready и 1 video-only материала; +- worker вызывает общий проверенный pipeline `scripts/publish_site_batch.py`, + который защищён lock-файлом `/tmp/forma-n8-site-poster.lock` от параллельных + ручных и автоматических запусков; +- обычный автоматический запуск не повторяет `publish_failed`; для явного + ручного повтора используется флаг `--retry-failed`. + +### 23.7. Навигация Ghost + +- лента выводит 25 постов на страницу; +- H1 главной: `Новости тактического снаряжения и экипировки`; +- только на первой странице `/`, после ленты и пагинации, выводится + статический SEO-блок `О новостной ленте Forma N8` с описанием тематики, + категорий и производителей; на `/page/2/` и далее блок не дублируется; +- пагинация полностью русская; +- шапка: `Лента`, выпадающие `Категории`, `О проекте`, поиск; +- ссылки на каналы Forma N8 (`https://vk.com/forma_n8` и `https://t.me/forma_n8`) с SVG-иконками присутствуют в шапке, каждом превью, открытой статье, разделе `О проекте` и футере; +- описание проекта начинается с текста: «Ежедневно мы собираем новости снаряжения с сотен источников, отбираем лучшее и публикуем в одном месте!»; +- футер: `RSS`, `Карта сайта`; +- старые `/tag/{slug}/` перенаправляются на `/category/{slug}/`; +- `/about/` содержит описание проекта; +- ссылка `Первоисточник` удалена из существующих статей и не добавляется + новым site-poster batch. + +### 23.8. SEO Ghost + +- название сайта: `Forma N8`; +- слоган главной: `Forma N8 — все новости снаряжения в одном месте`; +- description: `Forma N8 — все новости снаряжения в одном месте. Новинки, обзоры и события производителей тактического, туристического и специального снаряжения.`; +- title статьи: `Forma N8 — Название статьи`; +- title раздела: `Forma N8 — Название раздела`; +- meta keywords и «SEO-хештеги» не используются: категории оформляются + обычными Ghost tags, формирующими структуру сайта и посадочные страницы; +- тема должна сохранять один H1, canonical, Open Graph, Twitter Cards, + JSON-LD, RSS и sitemap, которые формирует Ghost через `ghost_head`; +- SSH-доступ и изменения на серверах `139.100.234.51` и + `109.120.156.205` выполняются через Python-библиотеку `paramiko`. + +доступ к серверу по ssh root@109.120.156.205 -p 22 +пароль OU8gUSCtQYcf diff --git a/STATE.md b/STATE.md new file mode 100644 index 0000000..adc9aa9 --- /dev/null +++ b/STATE.md @@ -0,0 +1,110 @@ +# N8 Parser: current state + +Last updated: 2026-06-17 + +This file is the short handoff state for future Codex threads. Read this first before touching the project, so the whole chat history does not need to be carried forward. + +## Server + +- Production host: `109.120.156.205` +- Domain: `https://sw.exostring.xyz` +- App directory on server: `/opt/vk-parser` +- Timezone: `Asia/Yekaterinburg` +- Main services: + - `vk-parser-admin.service` + - `vk-parser.service` + - `vk-storage-uploader.service` + - `ai-qualifier.service` + - `ai-writer.service` + - `site-poster.service` +- Useful commands: + - `cd /opt/vk-parser` + - `systemctl status vk-parser-admin.service ai-qualifier.service ai-writer.service` + - `journalctl -u ai-writer.service -n 100 --no-pager` + - `journalctl -u ai-qualifier.service -n 100 --no-pager` + - `PYTHONPATH=src .venv/bin/python -m py_compile src/vk_parser_app/admin.py src/vk_parser_app/workers/ai_writer.py` + +## Local Project + +- Local workspace: `D:\DEVELOPMENT\new_vk_parser` +- Main app package: `src/vk_parser_app` +- Admin routes: `src/vk_parser_app/admin.py` +- Templates: `src/vk_parser_app/templates` +- Workers: + - VK parser: `src/vk_parser_app/workers/parser.py` + - Telegram media/storage uploader: `src/vk_parser_app/workers/vk_storage_uploader.py` + - AI qualifier: `src/vk_parser_app/workers/ai_qualifier.py` + - AI writer: `src/vk_parser_app/workers/ai_writer.py` +- DB migrations: `db/migrations` +- Extended project documentation: `PROJECT.md` + +## Pipeline + +1. Sources are configured in admin. +2. VK parser reads enabled sources and stores raw posts. +3. Media/storage uploader sends raw post copies to Telegram storage and stores media metadata. +4. AI qualifier scores raw posts and marks accepted/rejected/maybe. +5. AI writer rewrites accepted posts into editor-ready drafts. +6. Editor page allows review, editing, category/source tag selection, media upload/removal, accept/reject. +7. Site poster automatically publishes accepted media-ready posts to Ghost. + +## Important Text Rules + +- AI writer output text must be clean: no physical hashtags at the end. +- `raw_posts.final_text` must also stay clean. +- Category and source hashtags are separate fields: + - category tag: `final_category_tag` / `rewrite_category_tag` + - source tag: `final_source_tag` / `rewrite_source_tag` / `sources.tag` +- Publication preview may show hashtags by composing: + - clean text + - blank line + - `#category #source` +- The editor textarea must show only clean editable text. +- The list preview in `/editor` should show the publication preview, including composed hashtags. +- `build_publication_text()` in `src/vk_parser_app/text_utils.py` is display/composition helper only. Do not use it before writing `final_text` to DB. + +## Recent Hotfix: Hashtags + +On 2026-06-17 the hashtag flow was fixed: + +- `ai_writer.py` now stores clean `final_text=$2`, not text with hashtags. +- `admin.py` prepares: + - `review_text`: clean text for textarea + - `publication_preview_text`: composed display text with hashtags +- editor accept/save store clean text. +- existing DB rows with trailing `#category #source` in `final_text` were cleaned. +- server check after fix: `remaining_hashtag_tail: 0`. + +## Admin Pages + +- `/sources`: source management. +- `/raw`: raw post feed. +- `/editor`: editor queue. +- `/workers`: worker controls and settings. +- `/prompt-test`: AI writer prompt test page. +- `/users`: admin users. +- `/logs`: audit/log page if enabled. + +## Prompt Architecture + +- Prompts are split into editable human part and technical contract part. +- User should be able to edit human prompt safely without breaking JSON schema. +- Contracts should contain JSON-only/output-schema requirements. +- Writer should return category ID, not free-form tag, where possible. +- Categories are managed in worker/admin settings and have stable IDs. + +## Security Notes + +- Secrets should not be committed. +- `.env` should contain only secrets and deploy-specific values. +- Runtime settings that are not secrets should live in DB/admin settings. +- Admin has login protection and server has firewall/fail2ban hardening from previous iterations. + +## Working Rules For Future Codex Runs + +- Prefer small targeted file reads with `rg` and narrow ranges. +- Avoid dumping long post texts from DB unless explicitly needed. +- For server checks, return counts/statuses first. +- Be careful: both local and server worktrees may be dirty. +- Do not `git reset --hard` or revert user/server hotfixes. +- When fixing production, patch server and then mirror the same change locally. diff --git a/db/migrations/016_remove_regeneration_and_writer_prompt.sql b/db/migrations/016_remove_regeneration_and_writer_prompt.sql index e60d43d..b22a050 100644 --- a/db/migrations/016_remove_regeneration_and_writer_prompt.sql +++ b/db/migrations/016_remove_regeneration_and_writer_prompt.sql @@ -21,8 +21,6 @@ SET value_json = to_jsonb($prompt$ Формат текста: - 3-6 коротких строк или 2-4 компактных абзаца; -- можно использовать 1-3 тематических эмоджи как маркеры структуры, например: 🧵 материал/конструкция, 🎒 переноска, 🛡 защита, ⚙️ детали, 📦 комплект/наличие, 🎯 назначение; -- эмоджи должны помогать читать текст, а не украшать каждую строку; - не добавляй ссылки и хэштеги в text. Важное правило про источник: diff --git a/db/migrations/019_writer_prompt_profile_style.sql b/db/migrations/019_writer_prompt_profile_style.sql index 77e373c..0377aef 100644 --- a/db/migrations/019_writer_prompt_profile_style.sql +++ b/db/migrations/019_writer_prompt_profile_style.sql @@ -30,8 +30,6 @@ SET value_json = to_jsonb($prompt$ ФОРМАТ ТЕКСТА - 3-6 коротких строк или 2-5 компактных абзацев; -- можно использовать 1-3 тематических эмоджи как маркеры структуры, например: 🧵 материал/конструкция, 🎒 переноска, 🛡 защита, ⚙️ детали, 📦 комплект/наличие, 🎯 назначение; -- эмоджи должны помогать читать текст, а не украшать каждую строку; - не добавляй ссылки и хэштеги в text. СТИЛЬ diff --git a/db/migrations/022_audit_log_retention_and_indexes.sql b/db/migrations/022_audit_log_retention_and_indexes.sql new file mode 100644 index 0000000..9712a9c --- /dev/null +++ b/db/migrations/022_audit_log_retention_and_indexes.sql @@ -0,0 +1,15 @@ +DELETE FROM audit_log +WHERE created_at < NOW() - INTERVAL '30 days'; + +CREATE INDEX IF NOT EXISTS idx_audit_log_created_at +ON audit_log(created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_audit_log_action_time +ON audit_log(action, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_audit_log_actor_time +ON audit_log(actor_id, created_at DESC) +WHERE actor_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_audit_log_entity_time +ON audit_log(entity_type, entity_id, created_at DESC); diff --git a/db/migrations/022_sync_ai_worker_enabled.sql b/db/migrations/022_sync_ai_worker_enabled.sql new file mode 100644 index 0000000..52cdc62 --- /dev/null +++ b/db/migrations/022_sync_ai_worker_enabled.sql @@ -0,0 +1,15 @@ +UPDATE app_settings s +SET value_json = to_jsonb(wc.enabled), + updated_at = NOW() +FROM worker_controls wc +WHERE (wc.name, s.key) IN ( + ('ai-qualifier', 'ai_qualifier_enabled'), + ('ai-writer', 'ai_writer_enabled') +); + +UPDATE raw_posts +SET rewrite_status='failed', + rewrite_notes=COALESCE(rewrite_notes, 'recovered stale writer processing state'), + updated_at=NOW() +WHERE rewrite_status='processing' + AND updated_at < NOW() - INTERVAL '30 minutes'; diff --git a/db/migrations/023_writer_strict_separator.sql b/db/migrations/023_writer_strict_separator.sql new file mode 100644 index 0000000..96e6199 --- /dev/null +++ b/db/migrations/023_writer_strict_separator.sql @@ -0,0 +1,21 @@ +-- Keep the AI-writer divider deterministic. +-- Target separator length: 27 characters. + +UPDATE app_settings +SET value_json = to_jsonb( + replace( + value_json #>> '{}', + '2. Черта с точным количеством символов ━━━━━━━━━━━━━━━━━━━━━━━━━━━', + '2. Черта-разделитель: отдельная строка ровно из 27 символов. Используй строго эту строку без изменений: ━━━━━━━━━━━━━━━━━━━━━━━━━━━' + ) +), +updated_at = NOW() +WHERE key = 'ai_writer_prompt' + AND (value_json #>> '{}') LIKE '%Черта с точным количеством символов%'; + +UPDATE raw_posts +SET rewritten_text = regexp_replace(rewritten_text, E'(^━{3,}$)', '━━━━━━━━━━━━━━━━━━━━━━━━━━━', 'gm'), + final_text = regexp_replace(final_text, E'(^━{3,}$)', '━━━━━━━━━━━━━━━━━━━━━━━━━━━', 'gm'), + updated_at = NOW() +WHERE COALESCE(rewritten_text, '') LIKE '%━%' + OR COALESCE(final_text, '') LIKE '%━%'; diff --git a/db/migrations/024_tg_poster.sql b/db/migrations/024_tg_poster.sql new file mode 100644 index 0000000..e8145ea --- /dev/null +++ b/db/migrations/024_tg_poster.sql @@ -0,0 +1,93 @@ +ALTER TABLE raw_posts + ADD COLUMN IF NOT EXISTS publication_status TEXT NOT NULL DEFAULT 'pending', + ADD COLUMN IF NOT EXISTS publication_error TEXT, + ADD COLUMN IF NOT EXISTS published_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS tg_publication_chat_id BIGINT, + ADD COLUMN IF NOT EXISTS tg_publication_thread_id BIGINT, + ADD COLUMN IF NOT EXISTS tg_publication_message_ids BIGINT[], + ADD COLUMN IF NOT EXISTS tg_publication_url TEXT; + +CREATE INDEX IF NOT EXISTS idx_raw_posts_publication_queue +ON raw_posts(publication_status, editorial_status, rewrite_status, published_at, reviewed_at, id); + +CREATE INDEX IF NOT EXISTS idx_raw_posts_publication_rank +ON raw_posts(final_category_tag, final_source_tag, published_at DESC); + +CREATE TABLE IF NOT EXISTS publication_runs ( + id BIGSERIAL PRIMARY KEY, + poster TEXT NOT NULL, + schedule_id TEXT NOT NULL, + scheduled_for DATE NOT NULL, + scheduled_time TEXT NOT NULL, + planned_count INTEGER NOT NULL DEFAULT 0, + published_count INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'started', + error TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE(poster, schedule_id, scheduled_for) +); + +CREATE INDEX IF NOT EXISTS idx_publication_runs_poster_date +ON publication_runs(poster, scheduled_for DESC, scheduled_time); + +INSERT INTO worker_controls(name, enabled, settings_json) +VALUES ('tg-poster', FALSE, '{}'::jsonb) +ON CONFLICT (name) DO NOTHING; + +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('tg_poster_enabled', 'false'::jsonb, 'bool', 'TG poster enabled', 'Double safety flag for publishing accepted posts to Telegram.', 'TG Poster'), + ('tg_poster_bot_token', '""'::jsonb, 'secret', 'TG poster bot token', 'Bot token for final Telegram publication. For test can be the same as upload bot token.', 'TG Poster'), + ('tg_poster_chat_id', '"-1003712724327"'::jsonb, 'str', 'TG publication chat id', 'Target Telegram channel/group id. Supports chat_id:thread_id for topics.', 'TG Poster'), + ('tg_poster_schedule_json', '[{"id":"morning","time":"10:00","count":1,"enabled":true},{"id":"evening","time":"18:00","count":1,"enabled":true}]'::jsonb, 'json', 'TG publication schedule', 'EKB time schedule rows: id, time HH:MM, count, enabled.', 'TG Poster'), + ('tg_poster_interval_sec', '60'::jsonb, 'int', 'TG poster interval, sec', 'Pause between poster checks.', 'TG Poster'), + ('tg_poster_media_group_max_items', '10'::jsonb, 'int', 'TG media group max items', 'Telegram media group supports 2..10 media items.', 'TG Poster'), + ('tg_poster_caption_limit', '1024'::jsonb, 'int', 'TG caption limit', 'Telegram caption limit for media messages.', 'TG Poster'), + ('tg_poster_message_limit', '4096'::jsonb, 'int', 'TG message limit', 'Telegram text message limit.', 'TG Poster'), + ('tg_poster_max_attempts', '3'::jsonb, 'int', 'TG post retry attempts', 'Retries for Telegram API errors.', 'TG Poster'), + ('tg_poster_retry_backoff_max_sec', '30'::jsonb, 'int', 'TG retry backoff max, sec', 'Max retry sleep for Telegram API errors.', 'TG Poster'), + ('tg_poster_send_delay_sec', '1'::jsonb, 'float', 'TG send delay, sec', 'Delay between Telegram sends.', 'TG Poster'), + ('tg_poster_text_overflow_caption', '"⬇️ Описание под медиа"'::jsonb, 'str', 'TG overflow caption', 'Caption used when full text is longer than Telegram media caption limit.', 'TG Poster'), + ('tg_poster_recent_window', '20'::jsonb, 'int', 'Ranking recent window', 'How many published posts are checked to reduce repeated categories and sources.', 'TG Poster'), + ('tg_poster_category_repeat_penalty', '3'::jsonb, 'float', 'Category repeat penalty', 'Ranking penalty for recently published same category.', 'TG Poster'), + ('tg_poster_source_repeat_penalty', '4'::jsonb, 'float', 'Source repeat penalty', 'Ranking penalty for recently published same source.', 'TG Poster') +ON CONFLICT (key) DO NOTHING; + +UPDATE app_settings +SET description='Обрезает исходный raw-текст перед отправкой в LLM. Итоговый rewrite должен быть компактным: Telegram caption 1024 символа минус будущая строка хэштегов.' +WHERE key='ai_writer_max_text_chars'; + +UPDATE app_settings +SET value_json = to_jsonb($contract$ +OUTPUT SCHEMA (return array matching input order): +{"rewrites":[{"id":123,"category_id":2,"text":"готовый текст без хэштегов","notes":"короткая заметка для редактора или null"}]} + +Rules: +- Input is a JSON object with key "posts" containing accepted posts. +- Input has key "categories" with objects: id, name, tag. +- Each post includes producer_name and producer_tag. producer_name can be a manufacturer, shop, or publishing source. +- Pick exactly one category from the categories list in input and return its numeric id as category_id. +- Do not add links or hashtags to rewritten text. +- Keep text compact enough for Telegram media caption: target 850 characters, hard maximum 900 characters. +- Remember the system appends category and source hashtags after text, so leave space for them. +- Return JSON only. No markdown. No text outside JSON. +- Never return an empty object. Never omit the "rewrites" key. +- Return one rewrite for every input post id. +- The "text" field must be a ready-to-publish Russian post with normal paragraph line breaks encoded as JSON string newlines. +- The "notes" field must be a short editor note in Russian or null. +$contract$::text), +updated_at = NOW() +WHERE key='ai_writer_contract'; + +UPDATE app_settings +SET value_json = to_jsonb( + regexp_replace( + value_json #>> '{}', + '- 3-6 коротких строк или 2-5 компактных абзацев;', + '- 3-6 коротких строк или 2-4 компактных абзаца; итоговый text должен целиться в 850 символов и не превышать 900 символов, потому что система добавит хэштеги и Telegram caption ограничен 1024 символами;', + 'g' + ) +) +WHERE key='ai_writer_prompt' + AND (value_json #>> '{}') LIKE '%3-6 коротких строк или 2-5 компактных абзацев%'; diff --git a/db/migrations/025_tg_poster_hourly_schedule.sql b/db/migrations/025_tg_poster_hourly_schedule.sql new file mode 100644 index 0000000..880f022 --- /dev/null +++ b/db/migrations/025_tg_poster_hourly_schedule.sql @@ -0,0 +1,46 @@ +UPDATE app_settings +SET value_json = '"⬇️ Описание"'::jsonb, + updated_at = NOW() +WHERE key='tg_poster_text_overflow_caption'; + +UPDATE app_settings +SET value_json = $schedule$[ + {"id":"hour-00","time":"00:00","count":4,"enabled":true}, + {"id":"hour-01","time":"01:00","count":4,"enabled":true}, + {"id":"hour-02","time":"02:00","count":4,"enabled":true}, + {"id":"hour-03","time":"03:00","count":4,"enabled":true}, + {"id":"hour-04","time":"04:00","count":4,"enabled":true}, + {"id":"hour-05","time":"05:00","count":4,"enabled":true}, + {"id":"hour-06","time":"06:00","count":4,"enabled":true}, + {"id":"hour-07","time":"07:00","count":4,"enabled":true}, + {"id":"hour-08","time":"08:00","count":4,"enabled":true}, + {"id":"hour-09","time":"09:00","count":4,"enabled":true}, + {"id":"hour-10","time":"10:00","count":4,"enabled":true}, + {"id":"hour-11","time":"11:00","count":4,"enabled":true}, + {"id":"hour-12","time":"12:00","count":4,"enabled":true}, + {"id":"hour-13","time":"13:00","count":4,"enabled":true}, + {"id":"hour-14","time":"14:00","count":4,"enabled":true}, + {"id":"hour-15","time":"15:00","count":4,"enabled":true}, + {"id":"hour-16","time":"16:00","count":4,"enabled":true}, + {"id":"hour-17","time":"17:00","count":4,"enabled":true}, + {"id":"hour-18","time":"18:00","count":4,"enabled":true}, + {"id":"hour-19","time":"19:00","count":4,"enabled":true}, + {"id":"hour-20","time":"20:00","count":4,"enabled":true}, + {"id":"hour-21","time":"21:00","count":4,"enabled":true}, + {"id":"hour-22","time":"22:00","count":4,"enabled":true}, + {"id":"hour-23","time":"23:00","count":4,"enabled":true} +]$schedule$::jsonb, + updated_at = NOW() +WHERE key='tg_poster_schedule_json'; + +UPDATE raw_posts +SET editorial_status='published', + updated_at=NOW() +WHERE publication_status='published' + AND editorial_status='accepted'; + +UPDATE raw_posts +SET editorial_status='publish_failed', + updated_at=NOW() +WHERE publication_status='publish_failed' + AND editorial_status='accepted'; diff --git a/db/migrations/026_writer_separator_17_chars.sql b/db/migrations/026_writer_separator_17_chars.sql new file mode 100644 index 0000000..a415f53 --- /dev/null +++ b/db/migrations/026_writer_separator_17_chars.sql @@ -0,0 +1,44 @@ +-- Keep the AI-writer divider deterministic. +-- Target separator length: 17 characters. + +UPDATE app_settings +SET value_json = to_jsonb( + regexp_replace( + value_json #>> '{}', + 'Черта-разделитель:[^\n]*', + 'Черта-разделитель: отдельная строка ровно из 17 символов. Используй строго эту строку без изменений: ━━━━━━━━━━━━━━━━━', + 'g' + ) +), +updated_at = NOW() +WHERE key = 'ai_writer_prompt' + AND (value_json #>> '{}') LIKE '%Черта-разделитель%'; + +UPDATE app_settings +SET value_json = to_jsonb( + CASE + WHEN (value_json #>> '{}') LIKE '%Separator line must be exactly%' + THEN regexp_replace( + value_json #>> '{}', + 'Separator line must be exactly[^\n]*', + 'Separator line must be exactly this 17-character line when used: ━━━━━━━━━━━━━━━━━', + 'g' + ) + ELSE (value_json #>> '{}') || E'\n- Separator line must be exactly this 17-character line when used: ━━━━━━━━━━━━━━━━━' + END +), +updated_at = NOW() +WHERE key = 'ai_writer_contract'; + +UPDATE raw_posts +SET rewritten_text = CASE + WHEN rewritten_text IS NULL THEN NULL + ELSE regexp_replace(rewritten_text, E'^[━─—-]{3,}\\s*$', '━━━━━━━━━━━━━━━━━', 'gm') + END, + final_text = CASE + WHEN final_text IS NULL THEN NULL + ELSE regexp_replace(final_text, E'^[━─—-]{3,}\\s*$', '━━━━━━━━━━━━━━━━━', 'gm') + END, + updated_at = NOW() +WHERE COALESCE(rewritten_text, '') ~ E'(?m)^[━─—-]{3,}\\s*$' + OR COALESCE(final_text, '') ~ E'(?m)^[━─—-]{3,}\\s*$'; diff --git a/db/migrations/027_raw_manual_rewrite_and_nck_reject.sql b/db/migrations/027_raw_manual_rewrite_and_nck_reject.sql new file mode 100644 index 0000000..714f855 --- /dev/null +++ b/db/migrations/027_raw_manual_rewrite_and_nck_reject.sql @@ -0,0 +1,34 @@ +-- Manual Raw-to-rewrite flow is implemented in admin.py. +-- Non-target category (sort_order/category_id 18, tag НЦК) must not enter editorial review. + +UPDATE app_settings +SET value_json = to_jsonb( + CASE + WHEN (value_json #>> '{}') LIKE '%category_id 18%' + THEN value_json #>> '{}' + ELSE (value_json #>> '{}') || E'\n- If the post is non-target content, choose category_id 18 ("Не целевой контент", tag "НЦК"); the app will reject it automatically.' + END +), +updated_at = NOW() +WHERE key = 'ai_writer_contract'; + +UPDATE raw_posts +SET editorial_status='rejected', + qualification_status='rejected', + qualification_decision='rejected', + qualification_model_decision='rejected', + qualification_reason=COALESCE(NULLIF(qualification_reason, ''), 'AI writer selected non-target category'), + qualification_reject_tag=COALESCE(NULLIF(qualification_reject_tag, ''), 'non_target_content'), + editor_notes=COALESCE(NULLIF(editor_notes, ''), 'Отклонено AI: Не целевой контент'), + reviewed_at=COALESCE(reviewed_at, NOW()), + updated_at=NOW() +WHERE rewrite_status='ready' + AND COALESCE(editorial_status, 'review')='review' + AND ( + rewrite_category_id=18 + OR final_category_id=18 + OR lower(COALESCE(rewrite_category_tag, '')) IN ('нцк', 'nck') + OR lower(COALESCE(final_category_tag, '')) IN ('нцк', 'nck') + OR lower(COALESCE(rewrite_category, '')) = lower('Не целевой контент') + OR lower(COALESCE(final_category, '')) = lower('Не целевой контент') + ); diff --git a/db/migrations/028_vk_poster_and_platform_publications.sql b/db/migrations/028_vk_poster_and_platform_publications.sql new file mode 100644 index 0000000..08706a1 --- /dev/null +++ b/db/migrations/028_vk_poster_and_platform_publications.sql @@ -0,0 +1,67 @@ +CREATE TABLE IF NOT EXISTS post_publications ( + id BIGSERIAL PRIMARY KEY, + raw_post_id BIGINT NOT NULL REFERENCES raw_posts(id) ON DELETE CASCADE, + platform TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + target_id TEXT, + external_id TEXT, + url TEXT, + error TEXT, + attachments_json JSONB NOT NULL DEFAULT '[]'::jsonb, + published_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE(raw_post_id, platform) +); + +CREATE INDEX IF NOT EXISTS idx_post_publications_platform_status +ON post_publications(platform, status, target_id, published_at DESC); + +CREATE INDEX IF NOT EXISTS idx_post_publications_raw_post +ON post_publications(raw_post_id); + +INSERT INTO post_publications(raw_post_id, platform, status, target_id, external_id, url, error, attachments_json, published_at, updated_at) +SELECT + id, + 'tg', + publication_status, + tg_publication_chat_id::text, + CASE WHEN tg_publication_message_ids IS NULL THEN NULL ELSE array_to_string(tg_publication_message_ids, ',') END, + tg_publication_url, + publication_error, + COALESCE(to_jsonb(tg_publication_message_ids), '[]'::jsonb), + published_at, + NOW() +FROM raw_posts +WHERE COALESCE(publication_status, 'pending') <> 'pending' +ON CONFLICT (raw_post_id, platform) DO UPDATE +SET status=EXCLUDED.status, + target_id=EXCLUDED.target_id, + external_id=EXCLUDED.external_id, + url=EXCLUDED.url, + error=EXCLUDED.error, + attachments_json=EXCLUDED.attachments_json, + published_at=EXCLUDED.published_at, + updated_at=NOW(); + +INSERT INTO worker_controls(name, enabled, settings_json) +VALUES ('vk-poster', FALSE, '{}'::jsonb) +ON CONFLICT (name) DO NOTHING; + +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('vk_poster_enabled', 'false'::jsonb, 'bool', 'VK poster enabled', 'Double safety flag for publishing accepted posts to VK.', 'VK Poster'), + ('vk_poster_access_token', '""'::jsonb, 'secret', 'VK poster access token', 'Group/user token with wall.post and photos permissions. If empty, env VK_GROUP_ACCESS_TOKEN/VK_ACCESS_TOKEN is used.', 'VK Poster'), + ('vk_poster_owner_id', '0'::jsonb, 'int', 'VK target owner_id', 'Target VK group owner_id, usually negative: -123456789.', 'VK Poster'), + ('vk_poster_from_group', 'true'::jsonb, 'bool', 'Post from group', 'Use from_group=1 when publishing to a community wall.', 'VK Poster'), + ('vk_poster_schedule_json', '[{"id":"vk-morning","time":"10:05","count":1,"enabled":true},{"id":"vk-evening","time":"18:05","count":1,"enabled":true}]'::jsonb, 'json', 'VK publication schedule', 'EKB time schedule rows: id, time HH:MM, count, enabled.', 'VK Poster'), + ('vk_poster_interval_sec', '60'::jsonb, 'int', 'VK poster interval, sec', 'Pause between poster checks.', 'VK Poster'), + ('vk_poster_timeout_sec', '90'::jsonb, 'int', 'VK API timeout, sec', 'Total timeout for VK API calls and media uploads.', 'VK Poster'), + ('vk_poster_max_attempts', '3'::jsonb, 'int', 'VK post retry attempts', 'Retries for VK API/network errors.', 'VK Poster'), + ('vk_poster_retry_backoff_max_sec', '30'::jsonb, 'int', 'VK retry backoff max, sec', 'Max retry sleep for VK API/network errors.', 'VK Poster'), + ('vk_poster_send_delay_sec', '1'::jsonb, 'float', 'VK send delay, sec', 'Delay between VK posts.', 'VK Poster'), + ('vk_poster_recent_window', '20'::jsonb, 'int', 'Ranking recent window', 'How many VK-published posts are checked to reduce repeated categories and sources.', 'VK Poster'), + ('vk_poster_category_repeat_penalty', '3'::jsonb, 'float', 'Category repeat penalty', 'Ranking penalty for recently published same category.', 'VK Poster'), + ('vk_poster_source_repeat_penalty', '4'::jsonb, 'float', 'Source repeat penalty', 'Ranking penalty for recently published same source.', 'VK Poster'), + ('vk_poster_dry_run', 'false'::jsonb, 'bool', 'VK dry run', 'If true, prepares attachments but does not call wall.post. Keep false for production.', 'VK Poster') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/029_vk_poster_oauth_settings.sql b/db/migrations/029_vk_poster_oauth_settings.sql new file mode 100644 index 0000000..e24de00 --- /dev/null +++ b/db/migrations/029_vk_poster_oauth_settings.sql @@ -0,0 +1,5 @@ +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('vk_poster_app_id', '""'::jsonb, 'str', 'VK OAuth app id', 'VK application id used for user OAuth code flow.', 'VK Poster'), + ('vk_poster_client_secret', '""'::jsonb, 'secret', 'VK OAuth client secret', 'Protected key from VK application settings. Used server-side to exchange code for user access token.', 'VK Poster') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/030_vk_poster_refresh_token.sql b/db/migrations/030_vk_poster_refresh_token.sql new file mode 100644 index 0000000..db4b6f3 --- /dev/null +++ b/db/migrations/030_vk_poster_refresh_token.sql @@ -0,0 +1,6 @@ +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('vk_poster_refresh_token', '""'::jsonb, 'secret', 'VK OAuth refresh token', 'Refresh token from VK ID code flow. Used by VK poster to renew access token automatically.', 'VK Poster'), + ('vk_poster_token_device_id', '""'::jsonb, 'str', 'VK OAuth device id', 'VK ID device_id returned with authorization code/token exchange.', 'VK Poster'), + ('vk_poster_token_expires_at', '""'::jsonb, 'str', 'VK access token expires at', 'UTC ISO timestamp used by VK poster to refresh the access token before expiry.', 'VK Poster') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/031_writer_require_producer_name.sql b/db/migrations/031_writer_require_producer_name.sql new file mode 100644 index 0000000..94ca386 --- /dev/null +++ b/db/migrations/031_writer_require_producer_name.sql @@ -0,0 +1,18 @@ +-- Make producer mention mandatory in every AI writer result. + +UPDATE app_settings +SET value_json = to_jsonb( + CASE + WHEN (value_json #>> '{}') LIKE '%explicitly mention producer_name%' + OR (value_json #>> '{}') LIKE '%явно упоминает producer_name%' + THEN value_json #>> '{}' + ELSE replace( + value_json #>> '{}', + '- Each post includes producer_name and producer_tag. producer_name can be a manufacturer, shop, or publishing source.', + '- Each post includes producer_name and producer_tag. producer_name can be a manufacturer, shop, or publishing source.' || E'\n' || + '- The "text" field for every rewrite must explicitly mention producer_name at least once. Do not satisfy this only through producer_tag, hashtags, notes, or metadata.' + ) + END +), +updated_at = NOW() +WHERE key = 'ai_writer_contract'; diff --git a/db/migrations/032_tg_reactor.sql b/db/migrations/032_tg_reactor.sql new file mode 100644 index 0000000..b99e6a6 --- /dev/null +++ b/db/migrations/032_tg_reactor.sql @@ -0,0 +1,39 @@ +CREATE TABLE IF NOT EXISTS post_reactions ( + id BIGSERIAL PRIMARY KEY, + publication_id BIGINT NOT NULL REFERENCES post_publications(id) ON DELETE CASCADE, + raw_post_id BIGINT NOT NULL REFERENCES raw_posts(id) ON DELETE CASCADE, + platform TEXT NOT NULL, + reactor_key TEXT NOT NULL, + reaction TEXT NOT NULL, + target_id TEXT NOT NULL, + message_id TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + error TEXT, + attempts INTEGER NOT NULL DEFAULT 0, + reacted_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE(publication_id, reactor_key, message_id) +); + +CREATE INDEX IF NOT EXISTS idx_post_reactions_status +ON post_reactions(platform, status, updated_at); + +CREATE INDEX IF NOT EXISTS idx_post_reactions_raw_post +ON post_reactions(raw_post_id); + +INSERT INTO worker_controls(name, enabled, settings_json) +VALUES ('tg-reactor', FALSE, '{}'::jsonb) +ON CONFLICT (name) DO NOTHING; + +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('tg_reactor_enabled', 'false'::jsonb, 'bool', 'TG reactor enabled', 'Double safety flag for Telegram reaction worker.', 'TG Reactor'), + ('tg_reactor_bot_tokens', '""'::jsonb, 'secret', 'TG reactor bot tokens', 'One or more Telegram bot tokens separated by comma/newline. If empty, uses tg_poster_bot_token/TG_BOT_TOKEN.', 'TG Reactor'), + ('tg_reactor_reactions_json', '["👍"]'::jsonb, 'json', 'TG reactions', 'Emoji reactions. One bot can set one reaction; multiple tokens can set multiple reactions.', 'TG Reactor'), + ('tg_reactor_delay_sec', '60'::jsonb, 'int', 'Delay after publication, sec', 'How long to wait after TG publication before reacting.', 'TG Reactor'), + ('tg_reactor_interval_sec', '60'::jsonb, 'int', 'TG reactor interval, sec', 'Pause between reaction worker checks.', 'TG Reactor'), + ('tg_reactor_react_all_messages', 'false'::jsonb, 'bool', 'React to all media messages', 'If false, reacts only to the first message of each published post.', 'TG Reactor'), + ('tg_reactor_max_attempts', '3'::jsonb, 'int', 'TG reaction retry attempts', 'Retries per publication/message/reactor.', 'TG Reactor'), + ('tg_reactor_retry_backoff_max_sec', '30'::jsonb, 'int', 'TG reaction retry backoff max, sec', 'Max sleep for Telegram reaction retry errors.', 'TG Reactor') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/033_tg_reactor_since.sql b/db/migrations/033_tg_reactor_since.sql new file mode 100644 index 0000000..b96ae14 --- /dev/null +++ b/db/migrations/033_tg_reactor_since.sql @@ -0,0 +1,4 @@ +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('tg_reactor_since', '""'::jsonb, 'str', 'React only since', 'UTC ISO timestamp. If set, TG reactor ignores publications older than this moment.', 'TG Reactor') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/034_tg_reactor_reaction_pause.sql b/db/migrations/034_tg_reactor_reaction_pause.sql new file mode 100644 index 0000000..ff6098e --- /dev/null +++ b/db/migrations/034_tg_reactor_reaction_pause.sql @@ -0,0 +1,4 @@ +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('tg_reactor_reaction_pause_sec', '2'::jsonb, 'int', 'Pause between TG reactions, sec', 'Small pause after each successful reaction to avoid a burst of Telegram API calls.', 'TG Reactor') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/035_uploader_video_format_limits.sql b/db/migrations/035_uploader_video_format_limits.sql new file mode 100644 index 0000000..67a2ac3 --- /dev/null +++ b/db/migrations/035_uploader_video_format_limits.sql @@ -0,0 +1,14 @@ +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('video_max_height', '720'::jsonb, 'int', 'Video max height', 'Maximum VK video height selected by yt-dlp before uploading to Telegram.', 'Uploader') +ON CONFLICT (key) DO NOTHING; + +UPDATE app_settings +SET value_json='600'::jsonb, + updated_at=NOW() +WHERE key='uploader_yt_dlp_timeout_sec' + AND CASE + WHEN jsonb_typeof(value_json) = 'number' THEN (value_json #>> '{}')::int + WHEN jsonb_typeof(value_json) = 'string' AND (value_json #>> '{}') ~ '^\d+$' THEN (value_json #>> '{}')::int + ELSE 0 + END < 600; diff --git a/db/migrations/036_daily_report.sql b/db/migrations/036_daily_report.sql new file mode 100644 index 0000000..fe0a41d --- /dev/null +++ b/db/migrations/036_daily_report.sql @@ -0,0 +1,13 @@ +INSERT INTO worker_controls(name, enabled, settings_json) +VALUES ('daily-report', TRUE, '{}'::jsonb) +ON CONFLICT (name) DO NOTHING; + +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('daily_report_enabled', 'true'::jsonb, 'bool', 'Daily report enabled', 'Double safety flag for daily Telegram operational report.', 'Daily Report'), + ('daily_report_bot_token', '""'::jsonb, 'secret', 'Daily report bot token', 'Telegram bot token for reports. If empty, uses tg_poster_bot_token or TG_BOT_TOKEN.', 'Daily Report'), + ('daily_report_recipient_ids', '[442509142]'::jsonb, 'json', 'Daily report recipients', 'Telegram user/chat IDs that receive the daily report.', 'Daily Report'), + ('daily_report_time', '"00:04"'::jsonb, 'str', 'Daily report time', 'EKB time HH:MM when yesterday report is sent.', 'Daily Report'), + ('daily_report_interval_sec', '60'::jsonb, 'int', 'Daily report interval, sec', 'Pause between checks.', 'Daily Report'), + ('daily_report_last_sent_date', '""'::jsonb, 'str', 'Last sent report date', 'Internal YYYY-MM-DD report date already sent automatically.', 'Daily Report') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/037_site_category_fields.sql b/db/migrations/037_site_category_fields.sql new file mode 100644 index 0000000..26f7f00 --- /dev/null +++ b/db/migrations/037_site_category_fields.sql @@ -0,0 +1,57 @@ +ALTER TABLE content_categories + ADD COLUMN IF NOT EXISTS site_name TEXT, + ADD COLUMN IF NOT EXISTS site_slug TEXT, + ADD COLUMN IF NOT EXISTS site_enabled BOOLEAN NOT NULL DEFAULT TRUE; + +UPDATE content_categories +SET site_name = CASE sort_order + WHEN 1 THEN 'Одежда' + WHEN 2 THEN 'Снаряжение' + WHEN 3 THEN 'Броня' + WHEN 4 THEN 'Аксессуары' + WHEN 5 THEN 'Связь' + WHEN 6 THEN 'Патчи' + WHEN 7 THEN 'Навигация' + WHEN 8 THEN 'Прицелы' + WHEN 9 THEN 'Ночные приборы' + WHEN 10 THEN 'Медицина' + WHEN 11 THEN 'Свет' + WHEN 12 THEN 'Электроника' + WHEN 13 THEN 'Ножи' + WHEN 14 THEN 'Оружейный обвес' + WHEN 15 THEN 'Практика' + WHEN 16 THEN 'Теория' + WHEN 17 THEN 'Пушки' + WHEN 18 THEN 'НЦК' + ELSE initcap(replace(tag, '_', ' ')) + END, + site_slug = CASE sort_order + WHEN 1 THEN 'odezhda' + WHEN 2 THEN 'snaryazhenie' + WHEN 3 THEN 'bronya' + WHEN 4 THEN 'aksessuary' + WHEN 5 THEN 'svyaz' + WHEN 6 THEN 'patchi' + WHEN 7 THEN 'navigatsiya' + WHEN 8 THEN 'pricely' + WHEN 9 THEN 'nochnye-pribory' + WHEN 10 THEN 'medicina' + WHEN 11 THEN 'svet' + WHEN 12 THEN 'elektronika' + WHEN 13 THEN 'nozhi' + WHEN 14 THEN 'oruzheynyy-obves' + WHEN 15 THEN 'praktika' + WHEN 16 THEN 'teoriya' + WHEN 17 THEN 'pushki' + WHEN 18 THEN 'nck' + ELSE site_slug + END, + site_enabled = sort_order <> 18, + updated_at = NOW(); + +ALTER TABLE content_categories + ALTER COLUMN site_name SET NOT NULL, + ALTER COLUMN site_slug SET NOT NULL; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_content_categories_site_slug +ON content_categories(site_slug); diff --git a/db/migrations/038_site_poster.sql b/db/migrations/038_site_poster.sql new file mode 100644 index 0000000..3c761cb --- /dev/null +++ b/db/migrations/038_site_poster.sql @@ -0,0 +1,13 @@ +INSERT INTO worker_controls(name, enabled, settings_json) +VALUES ('site-poster', TRUE, '{}'::jsonb) +ON CONFLICT (name) DO NOTHING; + +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('site_poster_enabled', 'true'::jsonb, 'bool', 'Site poster enabled', 'Double safety flag for automatic Ghost publishing.', 'Site Poster'), + ('site_poster_interval_sec', '300'::jsonb, 'int', 'Site poster interval, sec', 'Pause between automatic queue checks.', 'Site Poster'), + ('site_poster_photo_batch_size', '10'::jsonb, 'int', 'Photo batch size', 'Maximum photo-ready posts per automatic run.', 'Site Poster'), + ('site_poster_video_batch_size', '1'::jsonb, 'int', 'Video batch size', 'Maximum video-only posts per automatic run.', 'Site Poster'), + ('site_poster_ghost_url', '"https://f-n8.ru"'::jsonb, 'str', 'Ghost URL', 'Target Ghost public URL.', 'Site Poster'), + ('site_poster_key_file', '"/opt/vk-parser/.site-poster-key"'::jsonb, 'str', 'Ghost key file', 'Server path to the Ghost Admin API key.', 'Site Poster') +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/039_max_poster.sql b/db/migrations/039_max_poster.sql new file mode 100644 index 0000000..d36ab8c --- /dev/null +++ b/db/migrations/039_max_poster.sql @@ -0,0 +1,28 @@ +INSERT INTO worker_controls(name, enabled, settings_json) +VALUES ('max-poster', FALSE, '{}'::jsonb) +ON CONFLICT (name) DO NOTHING; + +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES + ('max_poster_enabled', 'false'::jsonb, 'bool', 'MAX poster enabled', 'Double safety flag for publishing accepted posts to MAX.', 'MAX Poster'), + ('max_poster_bot_token', '""'::jsonb, 'secret', 'MAX poster bot token', 'Bot access token for MAX API. Keep empty in git and set on server.', 'MAX Poster'), + ('max_poster_chat_id', '""'::jsonb, 'str', 'MAX target chat id', 'Target MAX channel/group chat_id.', 'MAX Poster'), + ('max_poster_api_base_url', '"https://platform-api2.max.ru"'::jsonb, 'str', 'MAX API base URL', 'MAX Bot API base URL. Since 2026 use platform-api2.max.ru.', 'MAX Poster'), + ('max_poster_schedule_json', '[{"id":"hour-00","time":"00:00","count":4,"enabled":true},{"id":"hour-01","time":"01:00","count":4,"enabled":true},{"id":"hour-02","time":"02:00","count":4,"enabled":true},{"id":"hour-03","time":"03:00","count":4,"enabled":true},{"id":"hour-04","time":"04:00","count":4,"enabled":true},{"id":"hour-05","time":"05:00","count":4,"enabled":true},{"id":"hour-06","time":"06:00","count":4,"enabled":true},{"id":"hour-07","time":"07:00","count":4,"enabled":true},{"id":"hour-08","time":"08:00","count":4,"enabled":true},{"id":"hour-09","time":"09:00","count":4,"enabled":true},{"id":"hour-10","time":"10:00","count":4,"enabled":true},{"id":"hour-11","time":"11:00","count":4,"enabled":true},{"id":"hour-12","time":"12:00","count":4,"enabled":true},{"id":"hour-13","time":"13:00","count":4,"enabled":true},{"id":"hour-14","time":"14:00","count":4,"enabled":true},{"id":"hour-15","time":"15:00","count":4,"enabled":true},{"id":"hour-16","time":"16:00","count":4,"enabled":true},{"id":"hour-17","time":"17:00","count":4,"enabled":true},{"id":"hour-18","time":"18:00","count":4,"enabled":true},{"id":"hour-19","time":"19:00","count":4,"enabled":true},{"id":"hour-20","time":"20:00","count":4,"enabled":true},{"id":"hour-21","time":"21:00","count":4,"enabled":true},{"id":"hour-22","time":"22:00","count":4,"enabled":true},{"id":"hour-23","time":"23:00","count":4,"enabled":true}]'::jsonb, 'json', 'MAX publication schedule', 'EKB time schedule rows: id, time HH:MM, count, enabled. Defaults to the TG hourly schedule.', 'MAX Poster'), + ('max_poster_interval_sec', '60'::jsonb, 'int', 'MAX poster interval, sec', 'Pause between poster checks.', 'MAX Poster'), + ('max_poster_message_limit', '4000'::jsonb, 'int', 'MAX message limit', 'MAX text message limit.', 'MAX Poster'), + ('max_poster_media_max_items', '10'::jsonb, 'int', 'MAX media max items', 'Maximum media attachments per MAX message.', 'MAX Poster'), + ('max_poster_media_process_delay_sec', '3'::jsonb, 'float', 'MAX media process delay, sec', 'Pause after media upload before sending the message.', 'MAX Poster'), + ('max_poster_video_ready_attempts', '5'::jsonb, 'int', 'MAX video ready attempts', 'Retries for sending a message while MAX is still processing uploaded video.', 'MAX Poster'), + ('max_poster_video_ready_delay_sec', '10'::jsonb, 'float', 'MAX video ready delay, sec', 'Initial retry delay while MAX is processing uploaded video. Delay doubles after each attempt.', 'MAX Poster'), + ('max_poster_max_attempts', '3'::jsonb, 'int', 'MAX post retry attempts', 'Retries for MAX API/network errors and attachment processing.', 'MAX Poster'), + ('max_poster_retry_backoff_max_sec', '30'::jsonb, 'int', 'MAX retry backoff max, sec', 'Max retry sleep for MAX API/network errors.', 'MAX Poster'), + ('max_poster_send_delay_sec', '1'::jsonb, 'float', 'MAX send delay, sec', 'Delay between MAX sends.', 'MAX Poster'), + ('max_poster_recent_window', '20'::jsonb, 'int', 'Ranking recent window', 'How many MAX-published posts are checked to reduce repeated categories and sources.', 'MAX Poster'), + ('max_poster_category_repeat_penalty', '3'::jsonb, 'float', 'Category repeat penalty', 'Ranking penalty for recently published same category.', 'MAX Poster'), + ('max_poster_source_repeat_penalty', '4'::jsonb, 'float', 'Source repeat penalty', 'Ranking penalty for recently published same source.', 'MAX Poster'), + ('max_poster_auto_reaction_enabled', 'true'::jsonb, 'bool', 'MAX auto reaction enabled', 'Try to set a reaction after publishing. If MAX API rejects it, publication remains successful.', 'MAX Poster'), + ('max_poster_auto_reaction', '"👍"'::jsonb, 'str', 'MAX auto reaction', 'Emoji reaction to try after publishing.', 'MAX Poster'), + ('max_poster_reaction_path_template', '"/messages/{message_id}/reactions"'::jsonb, 'str', 'MAX reaction path template', 'Relative API path for reaction call if supported by MAX API.', 'MAX Poster'), + ('max_poster_dry_run', 'false'::jsonb, 'bool', 'MAX dry run', 'If true, prepares publication but does not call MAX send message.', 'MAX Poster') +ON CONFLICT (key) DO NOTHING; diff --git a/deploy/max-poster.service b/deploy/max-poster.service new file mode 100644 index 0000000..d1a8c30 --- /dev/null +++ b/deploy/max-poster.service @@ -0,0 +1,15 @@ +[Unit] +Description=VK Parser MAX Poster +After=network.target postgresql.service + +[Service] +Type=simple +WorkingDirectory=/opt/vk-parser +Environment=PYTHONPATH=/opt/vk-parser/src +ExecStart=/opt/vk-parser/.venv/bin/python -m vk_parser_app.workers.max_poster +Restart=always +RestartSec=10 +User=root + +[Install] +WantedBy=multi-user.target diff --git a/deploy/site-poster.service b/deploy/site-poster.service new file mode 100644 index 0000000..1874803 --- /dev/null +++ b/deploy/site-poster.service @@ -0,0 +1,14 @@ +[Unit] +Description=VK Parser Site Poster +After=network.target postgresql.service + +[Service] +Type=simple +WorkingDirectory=/opt/vk-parser +Environment=PYTHONPATH=/opt/vk-parser/src +ExecStart=/opt/vk-parser/.venv/bin/python -m vk_parser_app.workers.site_poster +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/favi.png b/favi.png new file mode 100644 index 0000000..23a1268 Binary files /dev/null and b/favi.png differ diff --git a/forma-n8-social-preview-v2.jpg b/forma-n8-social-preview-v2.jpg new file mode 100644 index 0000000..d06cbd2 Binary files /dev/null and b/forma-n8-social-preview-v2.jpg differ diff --git a/forma_n8_site/.env.example b/forma_n8_site/.env.example new file mode 100644 index 0000000..e075a5d --- /dev/null +++ b/forma_n8_site/.env.example @@ -0,0 +1,8 @@ +DATABASE_URL=postgresql://forma_n8:change-me@127.0.0.1:5432/forma_n8 +SITE_BASE_URL=http://127.0.0.1:8090 +SITE_NAME=Forma N8 +SITE_TAGLINE=Новости снаряжения +SITE_INDEXING_ENABLED=false +PUBLISH_API_TOKEN=change-me +UPLOAD_DIR=/var/lib/forma-n8/uploads +PAGE_SIZE=12 diff --git a/forma_n8_site/README.md b/forma_n8_site/README.md new file mode 100644 index 0000000..2435175 --- /dev/null +++ b/forma_n8_site/README.md @@ -0,0 +1,55 @@ +# Forma N8 Site + +Отдельное SSR-приложение публичного сайта Forma N8. Контент и постоянные URL не зависят от будущего дизайна. + +## Возможности + +- лента новостей, страницы статей, категорий и производителей; +- PostgreSQL со стабильными `external_id` и `slug`; +- idempotent API публикации и отдельный upload API; +- RSS 2.0, sitemap, robots.txt; +- canonical, Open Graph и JSON-LD `NewsArticle`; +- системный деплой через nginx + systemd. + +## Локальный запуск + +```bash +python -m venv .venv +.venv/bin/pip install -r requirements.txt +export DATABASE_URL=postgresql://forma_n8:forma_n8@127.0.0.1:5432/forma_n8 +PYTHONPATH=src .venv/bin/python scripts/migrate.py +PYTHONPATH=src .venv/bin/uvicorn forma_n8_site.app:app --port 8090 +``` + +## Публикационный API + +Сначала медиа загружается через `POST /api/v1/media` с заголовком `X-Forma-Token`, затем полученный относительный URL передаётся в статью. + +`POST /api/v1/articles`: + +```json +{ + "external_id": "parser:518", + "title": "Заголовок новости", + "lead": "Краткое описание", + "body": "Основной текст без HTML", + "category_name": "Снаряжение", + "category_tag": "snaryazhenie", + "producer_name": "Производитель", + "producer_tag": "producer", + "source_url": "https://example.com/source", + "status": "published", + "published_at": "2026-06-21T10:00:00+05:00", + "media": [ + {"type": "image", "url": "/media/ab/hash.jpg", "alt": "Описание изображения"} + ] +} +``` + +Повторный запрос с тем же `external_id` обновляет материал, но сохраняет первоначальные slug и дату первой публикации. Это предотвращает дубли URL и потерю поисковой истории. + +Публичное название категории берётся из таблицы `categories` по нормализованному `category_tag`. Описательные названия категорий из AI-контура не используются в URL, H1 и SEO-метаданных. + +После подключения домена меняется только `SITE_BASE_URL` в `/etc/forma-n8.env`; затем настраивается TLS и отправляются sitemap в Google Search Console и Яндекс Вебмастер. + +До подключения домена `SITE_INDEXING_ENABLED=false`: страницы получают `noindex`, а `robots.txt` запрещает обход IP-адреса. После настройки домена, HTTPS и 301-редиректа с IP значение меняется на `true`. diff --git a/forma_n8_site/db/migrations/001_initial.sql b/forma_n8_site/db/migrations/001_initial.sql new file mode 100644 index 0000000..cf55ccf --- /dev/null +++ b/forma_n8_site/db/migrations/001_initial.sql @@ -0,0 +1,46 @@ +CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS articles ( + id BIGSERIAL PRIMARY KEY, + external_id TEXT NOT NULL UNIQUE, + slug TEXT NOT NULL UNIQUE, + title TEXT NOT NULL, + lead TEXT NOT NULL DEFAULT '', + body TEXT NOT NULL, + category_name TEXT NOT NULL, + category_tag TEXT NOT NULL, + producer_name TEXT NOT NULL, + producer_tag TEXT NOT NULL, + source_url TEXT, + seo_title TEXT NOT NULL DEFAULT '', + seo_description TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'draft' CHECK (status IN ('draft', 'published')), + published_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_articles_feed + ON articles(status, published_at DESC, id DESC); +CREATE INDEX IF NOT EXISTS idx_articles_category + ON articles(category_tag, status, published_at DESC); +CREATE INDEX IF NOT EXISTS idx_articles_producer + ON articles(producer_tag, status, published_at DESC); + +CREATE TABLE IF NOT EXISTS article_media ( + id BIGSERIAL PRIMARY KEY, + article_id BIGINT NOT NULL REFERENCES articles(id) ON DELETE CASCADE, + media_type TEXT NOT NULL CHECK (media_type IN ('image', 'video')), + url TEXT NOT NULL, + alt_text TEXT NOT NULL DEFAULT '', + width INTEGER, + height INTEGER, + sort_order INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_article_media_order + ON article_media(article_id, sort_order, id); diff --git a/forma_n8_site/db/migrations/002_categories.sql b/forma_n8_site/db/migrations/002_categories.sql new file mode 100644 index 0000000..fa547b0 --- /dev/null +++ b/forma_n8_site/db/migrations/002_categories.sql @@ -0,0 +1,35 @@ +CREATE TABLE IF NOT EXISTS categories ( + tag TEXT PRIMARY KEY, + name TEXT NOT NULL, + sort_order INTEGER NOT NULL DEFAULT 100, + is_active BOOLEAN NOT NULL DEFAULT TRUE +); + +INSERT INTO categories(tag, name, sort_order) VALUES + ('odezhda', 'Одежда', 1), + ('snaryazhenie', 'Снаряжение', 2), + ('bronya', 'Броня', 3), + ('aksessuary', 'Аксессуары', 4), + ('svyaz', 'Связь', 5), + ('patchi', 'Патчи', 6), + ('navigaciya', 'Навигация', 7), + ('pricely', 'Прицелы', 8), + ('nochnye-pribory', 'Ночные приборы', 9), + ('medicina', 'Медицина', 10), + ('svet', 'Свет', 11), + ('elektronika', 'Электроника', 12), + ('nozhi', 'Ножи и мультитулы', 13), + ('oruzheynyy-obves', 'Оружейный обвес', 14), + ('praktika', 'Практика', 15), + ('teoriya', 'Теория', 16), + ('pushki', 'Оружие', 17) +ON CONFLICT(tag) DO UPDATE SET + name=EXCLUDED.name, + sort_order=EXCLUDED.sort_order; + +UPDATE articles a +SET category_name=c.name, + updated_at=NOW() +FROM categories c +WHERE a.category_tag=c.tag + AND a.category_name<>c.name; diff --git a/forma_n8_site/deploy/forma-n8.service b/forma_n8_site/deploy/forma-n8.service new file mode 100644 index 0000000..7f7d14b --- /dev/null +++ b/forma_n8_site/deploy/forma-n8.service @@ -0,0 +1,23 @@ +[Unit] +Description=Forma N8 public website +After=network.target postgresql.service +Wants=postgresql.service + +[Service] +Type=simple +User=forma-n8 +Group=forma-n8 +WorkingDirectory=/opt/forma-n8 +EnvironmentFile=/etc/forma-n8.env +Environment=PYTHONPATH=/opt/forma-n8/src +ExecStart=/opt/forma-n8/.venv/bin/uvicorn forma_n8_site.app:app --host 127.0.0.1 --port 8090 --proxy-headers +Restart=always +RestartSec=3 +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/forma-n8 + +[Install] +WantedBy=multi-user.target diff --git a/forma_n8_site/deploy/nginx.conf b/forma_n8_site/deploy/nginx.conf new file mode 100644 index 0000000..55a63f5 --- /dev/null +++ b/forma_n8_site/deploy/nginx.conf @@ -0,0 +1,31 @@ +server { + listen 80 default_server; + listen [::]:80 default_server; + server_name _; + + client_max_body_size 52m; + + location /static/ { + alias /opt/forma-n8/static/; + expires 30d; + add_header Cache-Control "public, immutable"; + access_log off; + } + + location /media/ { + alias /var/lib/forma-n8/uploads/; + expires 30d; + add_header Cache-Control "public"; + add_header X-Content-Type-Options "nosniff"; + } + + location / { + proxy_pass http://127.0.0.1:8090; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 60s; + } +} diff --git a/forma_n8_site/design/concept-home.png b/forma_n8_site/design/concept-home.png new file mode 100644 index 0000000..cb19625 Binary files /dev/null and b/forma_n8_site/design/concept-home.png differ diff --git a/forma_n8_site/ghost-theme/forma-n8/assets/css/screen.css b/forma_n8_site/ghost-theme/forma-n8/assets/css/screen.css new file mode 100644 index 0000000..fa2f0d4 --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/assets/css/screen.css @@ -0,0 +1,431 @@ +@font-face { + font-family: "Roboto Condensed"; + src: url("../fonts/roboto-condensed-cyrillic-400.woff2") format("woff2"); + font-weight: 400; + font-display: swap; +} + +@font-face { + font-family: "Roboto Condensed"; + src: url("../fonts/roboto-condensed-cyrillic-700.woff2") format("woff2"); + font-weight: 700; + font-display: swap; +} + +:root { + color-scheme: dark; + --bg: #090c0d; + --surface: #101415; + --surface-strong: #151a1b; + --text: #eceee8; + --muted: #9ba19b; + --faint: #686f6b; + --border: #29302e; + --border-strong: #3a423f; + --olive: #a5ad62; + --red: #df4a40; + --max: 1240px; + --gutter: 24px; + --font: "Roboto Condensed", "Arial Narrow", Arial, sans-serif; +} + +* { box-sizing: border-box; } + +html { background: var(--bg); scroll-behavior: smooth; } + +body { + margin: 0; + min-width: 320px; + background: var(--bg); + color: var(--text); + font-family: var(--font); + font-size: 18px; + line-height: 1.55; + letter-spacing: 0; + -webkit-font-smoothing: antialiased; +} + +body::before { + content: ""; + position: fixed; + inset: 0; + pointer-events: none; + opacity: .22; + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='120' height='120' viewBox='0 0 120 120'%3E%3Cfilter id='n'%3E%3CfeTurbulence type='fractalNoise' baseFrequency='.8' numOctaves='2' stitchTiles='stitch'/%3E%3C/filter%3E%3Crect width='100%25' height='100%25' filter='url(%23n)' opacity='.08'/%3E%3C/svg%3E"); +} + +a { color: inherit; text-decoration: none; } +a:hover { color: var(--olive); } +img, video { display: block; max-width: 100%; } +button, input { font: inherit; letter-spacing: 0; } + +:focus-visible { outline: 2px solid var(--olive); outline-offset: 4px; } + +.skip-link { + position: fixed; + z-index: 100; + top: 10px; + left: 10px; + transform: translateY(-160%); + padding: 10px 14px; + background: var(--text); + color: var(--bg); +} +.skip-link:focus { transform: none; } + +.site-header { + position: sticky; + z-index: 20; + top: 0; + border-bottom: 1px solid var(--border); + background: rgba(9, 12, 13, .96); + backdrop-filter: blur(12px); +} + +.header-inner { + width: min(var(--max), calc(100% - var(--gutter) * 2)); + min-height: 76px; + margin: 0 auto; + display: grid; + grid-template-columns: 170px 1fr auto; + align-items: center; + gap: 32px; +} + +.brand { + display: inline-flex; + width: fit-content; + align-items: center; + gap: 11px; + font-size: 15px; + font-weight: 700; + line-height: .95; +} +.brand img { width: 42px; height: 42px; border: 1px solid var(--border-strong); } + +.main-nav { display: flex; justify-content: center; align-self: stretch; gap: 38px; } +.main-nav .nav { display: contents; margin: 0; padding: 0; list-style: none; } +.main-nav .nav li { display: contents; } +.nav-dropdown { position: relative; display: flex; align-items: center; } +.nav-dropdown-toggle { display: inline-flex; height: 100%; align-items: center; gap: 8px; padding: 0; border: 0; background: transparent; color: var(--muted); cursor: pointer; } +.nav-dropdown-toggle:hover, .nav-dropdown-toggle[aria-expanded="true"] { color: var(--text); } +.nav-dropdown-icon { width: 7px; height: 7px; border-right: 1.5px solid currentColor; border-bottom: 1.5px solid currentColor; transform: translateY(-2px) rotate(45deg); transition: transform .18s ease; } +.nav-dropdown-toggle[aria-expanded="true"] .nav-dropdown-icon { transform: translateY(2px) rotate(225deg); } +.category-menu { position: absolute; z-index: 30; top: calc(100% - 1px); left: 50%; display: none; grid-template-columns: repeat(2, minmax(170px, 1fr)); width: 430px; padding: 12px; border: 1px solid var(--border-strong); background: var(--bg); transform: translateX(-50%); box-shadow: 0 18px 44px rgba(0,0,0,.38); } +.category-menu.is-open { display: grid; } +.category-menu a { min-height: 40px; padding: 9px 11px; border-bottom: 1px solid var(--border); color: var(--muted); } +.category-menu a:hover { background: var(--surface-strong); color: var(--text); } +.main-nav a { + position: relative; + display: inline-flex; + align-items: center; + color: var(--muted); + font-size: 16px; + white-space: nowrap; +} +.main-nav a:hover, .main-nav a[aria-current="page"] { color: var(--text); } +.main-nav a[aria-current="page"]::after { + content: ""; + position: absolute; + right: 0; + bottom: -1px; + left: 0; + height: 3px; + background: var(--olive); +} + +.header-actions { display: flex; gap: 8px; } +.header-actions > .social-links { margin-right: 5px; padding-right: 10px; border-right: 1px solid var(--border); } +.icon-button { + display: grid; + width: 40px; + height: 40px; + padding: 0; + place-items: center; + border: 0; + background: transparent; + color: var(--text); + cursor: pointer; +} +.icon-button:hover { background: var(--surface-strong); color: var(--olive); } +.icon-button svg { width: 23px; height: 23px; fill: none; stroke: currentColor; stroke-width: 1.8; stroke-linecap: square; } +.menu-button { display: none; } + +.search-form { + width: min(760px, calc(100% - var(--gutter) * 2)); + margin: 0 auto; + padding: 16px 0 20px; +} +.search-form[hidden] { display: none; } +.search-form label { display: block; margin-bottom: 8px; color: var(--muted); font-size: 14px; } +.search-form > div { display: grid; grid-template-columns: 1fr auto; } +.search-form input { + min-width: 0; + height: 48px; + padding: 0 16px; + border: 1px solid var(--border-strong); + border-right: 0; + border-radius: 0; + background: var(--surface); + color: var(--text); + font-size: 17px; +} +.search-form button, .read-link, .empty-state a { + border: 1px solid var(--olive); + border-radius: 0; + background: transparent; + color: var(--text); + cursor: pointer; +} +.search-form button { padding: 0 24px; } +.search-form button:hover, .read-link:hover, .empty-state a:hover { background: var(--olive); color: var(--bg); } + +.feed-shell, .article-page, .related-section, .site-footer { + position: relative; + width: min(var(--max), calc(100% - var(--gutter) * 2)); + margin: 0 auto; +} + +.page-heading { padding: 56px 0 46px; } +.page-heading h1 { + max-width: 900px; + margin: 0; + font-size: 68px; + font-weight: 700; + line-height: .98; +} +@media (min-width: 981px) { + .home-template .page-heading { + position: relative; + left: 50%; + width: min(1600px, calc(100vw - var(--gutter) * 2)); + transform: translateX(-50%); + } + .home-template .page-heading h1 { + max-width: none; + font-size: clamp(58px, 4vw, 68px); + text-align: center; + } +} +.page-heading > p { margin: 20px 0 0; color: var(--muted); } + +.lead-story { + display: grid; + grid-template-columns: minmax(0, 1.25fr) minmax(340px, .95fr); + min-height: 480px; + max-height: 680px; + border-bottom: 1px solid var(--border-strong); +} +.lead-media { min-height: 480px; max-height: 680px; overflow: hidden; background: var(--surface); } +.lead-media img, .lead-media video { width: 100%; height: 100%; object-fit: cover; transition: transform .45s ease; } +.lead-media:hover img { transform: scale(1.015); } +.lead-copy { display: flex; flex-direction: column; justify-content: center; padding: 34px 0 36px 44px; } +.lead-copy h2 { margin: 24px 0 22px; font-size: 40px; line-height: 1.14; } +.lead-copy h2 a:hover { color: var(--text); text-decoration: underline; text-decoration-color: var(--olive); text-underline-offset: 7px; } +.lead-copy > p { max-width: 580px; margin: 0; color: var(--muted); font-size: 20px; line-height: 1.5; } +.read-link { width: fit-content; margin-top: 30px; padding: 10px 15px; font-size: 15px; } +.read-link span { margin-left: 8px; } + +.article-meta { display: flex; align-items: center; flex-wrap: wrap; gap: 0; color: var(--muted); font-size: 14px; } +.article-meta > * { display: inline-flex; align-items: center; } +.article-meta > * + *::before { content: ""; width: 1px; height: 18px; margin: 0 16px; background: var(--border-strong); } +.article-meta .category-link { color: var(--olive); } +.article-meta .producer-link { color: var(--red); } + +.feed-list { width: 100%; } +.feed-list-top { border-top: 1px solid var(--border-strong); } +.feed-row { + display: grid; + grid-template-columns: 350px minmax(0, 1fr); + gap: 34px; + padding: 22px 0; + border-bottom: 1px solid var(--border); +} +.feed-row-media { aspect-ratio: 16 / 9; overflow: hidden; background: var(--surface); } +.feed-row-media img, .feed-row-media video { width: 100%; height: 100%; object-fit: cover; transition: transform .35s ease; } +.feed-row-media:hover img { transform: scale(1.02); } +.feed-row-copy { align-self: center; min-width: 0; } +.feed-row h2 { margin: 16px 0 8px; font-size: 25px; line-height: 1.2; } +.feed-row h2 a:hover { color: var(--text); text-decoration: underline; text-decoration-color: var(--olive); text-underline-offset: 5px; } +.feed-row p { max-width: 760px; margin: 0; color: var(--muted); font-size: 17px; line-height: 1.5; } +.preview-social { margin-top: 14px; } +.media-placeholder { display: grid; width: 100%; height: 100%; place-items: center; background: var(--surface); } +.media-placeholder img { width: 76px; height: 76px; opacity: .28; object-fit: contain; } +.feed-media { position: relative; } +.feed-media-feature, .feed-media-feature > img, .feed-media-feature > .media-placeholder { display: block; width: 100%; height: 100%; } +.feed-media-track { display: flex; width: 100%; height: 100%; overflow-x: auto; overflow-y: hidden; scroll-snap-type: x mandatory; scrollbar-width: none; } +.feed-media-track::-webkit-scrollbar { display: none; } +.feed-media-slide { flex: 0 0 100%; width: 100%; height: 100%; scroll-snap-align: start; } +.feed-media-slide > a, .feed-media-slide img, .feed-media-slide video { display: block; width: 100%; height: 100%; } +.feed-media-slide img, .feed-media-slide video { object-fit: cover; } +.feed-media-count { position: absolute; top: 10px; right: 10px; z-index: 2; padding: 4px 8px; border-radius: 999px; background: rgba(0,0,0,.68); color: #fff; font-size: 12px; line-height: 1; pointer-events: none; } + +.pagination { display: grid; grid-template-columns: 1fr auto 1fr; align-items: center; min-height: 90px; font-size: 15px; } +.pagination > :last-child { justify-self: end; } +.pagination span { color: var(--faint); } +.pagination .older-posts { grid-column: 3; justify-self: end; } +.pagination .newer-posts { grid-column: 1; } +.pagination .page-number { grid-column: 2; grid-row: 1; } + +.home-seo { + max-width: 960px; + margin: 54px auto 0; + padding: 40px 0 0; + border-top: 1px solid var(--border-strong); +} +.home-seo h2 { margin: 0 0 24px; font-size: 34px; line-height: 1.15; } +.home-seo p { margin: 0 0 18px; color: var(--muted); font-size: 17px; line-height: 1.7; } +.home-seo p:last-child { margin-bottom: 0; } + +.taxonomy-heading { padding-bottom: 34px; } +.taxonomy-heading > p { margin: 0 0 10px; color: var(--olive); font-size: 15px; text-transform: uppercase; } +.taxonomy-heading h1 { font-size: 56px; } + +.article-page { padding-top: 56px; } +.article-header { max-width: 960px; margin: 0 auto 42px; } +.article-breadcrumbs { display: flex; gap: 12px; margin-bottom: 28px; color: var(--muted); font-size: 14px; } +.article-breadcrumbs span { color: var(--faint); } +.article-header h1 { margin: 0; font-size: 58px; line-height: 1.06; } +.article-lead { max-width: 850px; margin: 26px 0 0; color: var(--muted); font-size: 23px; line-height: 1.45; } +.article-byline { display: flex; gap: 24px; margin-top: 28px; color: var(--muted); font-size: 14px; } +.article-byline a { color: var(--red); } +.article-media { display: grid; gap: 12px; margin-bottom: 46px; } +.article-media figure { margin: 0; background: var(--surface); } +.article-media img, .article-media video { width: 100%; max-height: 760px; object-fit: contain; } +.article-layout { display: grid; grid-template-columns: minmax(0, 760px) 260px; justify-content: center; gap: 78px; } +.article-body { font-size: 21px; line-height: 1.7; } +.article-body p { margin: 0 0 26px; } +.gh-content > * { max-width: 100%; } +.gh-content a { color: var(--olive); text-decoration: underline; text-underline-offset: 3px; } +.gh-content figure, .gh-content .kg-card { margin: 32px 0; } +.gh-content img, .gh-content video { width: 100%; height: auto; } +.gh-content blockquote { margin: 30px 0; padding-left: 24px; border-left: 3px solid var(--olive); color: var(--muted); } +.kg-width-wide, .kg-width-full { width: 100%; } +.kg-gallery-container { display: flex; flex-direction: column; gap: 10px; } +.kg-gallery-row { display: flex; gap: 10px; } +.kg-gallery-image { flex: 1 1 0; min-width: 0; margin: 0; overflow: hidden; background: var(--surface); } +.kg-gallery-image img { width: 100%; height: 100%; margin: 0; object-fit: cover; cursor: zoom-in; } +.media-lightbox { width: 100%; height: 100%; max-width: none; max-height: none; padding: 0; border: 0; background: rgba(0,0,0,.94); } +.media-lightbox::backdrop { background: rgba(0,0,0,.94); } +.media-lightbox img { width: 100%; height: 100%; object-fit: contain; } +.media-lightbox button { position: fixed; top: 18px; right: 18px; z-index: 2; width: 44px; height: 44px; border: 1px solid var(--border-strong); background: var(--bg); color: var(--text); font-size: 28px; cursor: pointer; } +.article-facts { align-self: start; border-top: 2px solid var(--olive); } +.article-facts > div { padding: 18px 0; border-bottom: 1px solid var(--border); } +.article-facts span { display: block; margin-bottom: 4px; color: var(--faint); font-size: 13px; text-transform: uppercase; } +.article-facts a { font-size: 17px; } +.article-facts .source-link { display: block; margin-top: 22px; color: var(--muted); font-size: 14px; } +.social-links { display: flex; align-items: center; flex-wrap: wrap; gap: 8px; margin-top: 13px; } +.social-link { display: inline-flex; min-height: 34px; align-items: center; gap: 7px; padding: 5px 10px; border: 1px solid var(--border); color: var(--muted); font-size: 14px; line-height: 1; transition: border-color .18s ease, color .18s ease, background .18s ease; } +.social-link:hover { border-color: var(--border-strong); background: var(--surface-strong); color: var(--text); } +.social-link svg { width: 18px; height: 18px; fill: currentColor; } +.social-link--vk:hover { color: #7ca8e8; } +.social-link--tg:hover { color: #58b9e8; } +.social-links--compact { gap: 4px; margin-top: 0; } +.social-links--compact .social-link { width: 34px; min-height: 34px; justify-content: center; padding: 0; } +.social-links--compact .social-link svg { width: 17px; height: 17px; } +.about-social { margin-top: 42px; padding-top: 24px; border-top: 1px solid var(--border-strong); } +.about-social > span { color: var(--faint); font-size: 13px; text-transform: uppercase; } +.related-section { margin-top: 90px; border-top: 1px solid var(--border-strong); } +.related-section > h2 { margin: 0; padding: 34px 0 18px; font-size: 32px; } + +.empty-state, .not-found { min-height: 56vh; display: flex; flex-direction: column; justify-content: center; align-items: flex-start; } +.empty-state h2, .not-found h1 { margin: 0; font-size: 42px; } +.empty-state p { color: var(--muted); } +.empty-state a { margin-top: 16px; padding: 10px 15px; } +.not-found > p { margin: 0; color: var(--red); font-size: 18px; } +.not-found a { margin-top: 24px; color: var(--olive); } + +.site-footer { + display: flex; + justify-content: space-between; + align-items: flex-end; + margin-top: 84px; + padding: 34px 0 46px; + border-top: 1px solid var(--border-strong); + color: var(--muted); + font-size: 14px; +} +.footer-brand { color: var(--text); font-size: 20px; font-weight: 700; } +.site-footer p { margin: 7px 0 0; } +.site-footer nav { display: flex; gap: 24px; } +.site-footer .social-links { margin-top: 18px; } + +@media (max-width: 980px) { + .header-inner { grid-template-columns: 150px 1fr auto; gap: 16px; } + .main-nav { gap: 22px; } + .main-nav a { font-size: 14px; } + .page-heading h1 { font-size: 56px; } + .lead-story { grid-template-columns: 1.1fr .9fr; min-height: 420px; } + .lead-media { min-height: 420px; } + .lead-copy { padding-left: 30px; } + .lead-copy h2 { font-size: 33px; } + .feed-row { grid-template-columns: 290px minmax(0, 1fr); gap: 26px; } + .article-layout { grid-template-columns: minmax(0, 680px) 220px; gap: 44px; } +} + +@media (max-width: 760px) { + :root { --gutter: 16px; } + body { font-size: 17px; } + .home-seo { margin-top: 38px; padding-top: 30px; } + .home-seo h2 { font-size: 28px; } + .home-seo p { font-size: 16px; line-height: 1.65; } + .header-inner { min-height: 64px; grid-template-columns: 1fr auto; } + .brand img { width: 38px; height: 38px; } + .main-nav { + position: absolute; + top: 64px; + right: 0; + left: 0; + display: none; + padding: 12px var(--gutter) 18px; + border-bottom: 1px solid var(--border); + background: var(--bg); + } + .main-nav.is-open { display: grid; } + .main-nav a { min-height: 44px; border-bottom: 1px solid var(--border); font-size: 16px; } + .nav-dropdown { display: block; } + .nav-dropdown-toggle { width: 100%; min-height: 44px; border-bottom: 1px solid var(--border); text-align: left; font-size: 16px; } + .category-menu { position: static; width: 100%; grid-template-columns: 1fr; padding: 0 0 8px 14px; border: 0; box-shadow: none; transform: none; } + .category-menu a { font-size: 15px; } + .main-nav a[aria-current="page"]::after { display: none; } + .menu-button { display: grid; } + .page-heading { padding: 38px 0 30px; } + .page-heading h1, .taxonomy-heading h1 { font-size: 44px; line-height: 1; } + .lead-story { display: block; min-height: 0; max-height: none; } + .lead-media { min-height: 0; max-height: min(72vh, 560px); aspect-ratio: 4 / 3; } + .lead-copy { padding: 25px 0 34px; } + .lead-copy h2 { margin: 20px 0 15px; font-size: 32px; } + .lead-copy > p { font-size: 18px; } + .feed-row { display: block; padding: 24px 0; } + .feed-row-media { width: 100%; max-height: min(72vh, 560px); aspect-ratio: 4 / 3; } + .feed-row-copy { padding-top: 16px; } + .feed-row h2 { margin: 10px 0 8px; font-size: 24px; } + .feed-row p { display: block; font-size: 16px; } + .preview-social { margin-top: 10px; } + .preview-social .social-link { width: 30px; min-height: 30px; } + .article-meta { font-size: 12px; } + .article-meta time { display: none; } + .article-meta > * + *::before { height: 14px; margin: 0 10px; } + .article-meta .category-link::before { display: none; } + .article-header { margin-bottom: 30px; } + .article-header h1 { font-size: 42px; } + .article-lead { font-size: 20px; } + .article-layout { display: block; } + .article-body { font-size: 19px; } + .article-facts { margin-top: 42px; } + .related-section { margin-top: 64px; } +} + +@media (max-width: 460px) { + .feed-row h2 { font-size: 22px; line-height: 1.2; } + .article-meta .producer-link::before { display: none; } + .article-meta .category-link { display: none; } + .article-header h1 { font-size: 36px; } + .article-byline { display: grid; gap: 7px; } + .site-footer { display: grid; gap: 24px; } +} + +@media (prefers-reduced-motion: reduce) { + html { scroll-behavior: auto; } + *, *::before, *::after { transition-duration: .01ms !important; } +} diff --git a/forma_n8_site/ghost-theme/forma-n8/assets/fonts/roboto-condensed-cyrillic-400.woff2 b/forma_n8_site/ghost-theme/forma-n8/assets/fonts/roboto-condensed-cyrillic-400.woff2 new file mode 100644 index 0000000..10138a0 Binary files /dev/null and b/forma_n8_site/ghost-theme/forma-n8/assets/fonts/roboto-condensed-cyrillic-400.woff2 differ diff --git a/forma_n8_site/ghost-theme/forma-n8/assets/fonts/roboto-condensed-cyrillic-700.woff2 b/forma_n8_site/ghost-theme/forma-n8/assets/fonts/roboto-condensed-cyrillic-700.woff2 new file mode 100644 index 0000000..c001da3 Binary files /dev/null and b/forma_n8_site/ghost-theme/forma-n8/assets/fonts/roboto-condensed-cyrillic-700.woff2 differ diff --git a/forma_n8_site/ghost-theme/forma-n8/assets/images/favi.png b/forma_n8_site/ghost-theme/forma-n8/assets/images/favi.png new file mode 100644 index 0000000..23a1268 Binary files /dev/null and b/forma_n8_site/ghost-theme/forma-n8/assets/images/favi.png differ diff --git a/forma_n8_site/ghost-theme/forma-n8/assets/js/site.js b/forma_n8_site/ghost-theme/forma-n8/assets/js/site.js new file mode 100644 index 0000000..af1bfa8 --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/assets/js/site.js @@ -0,0 +1,86 @@ +(() => { + document.querySelectorAll('[data-feed-media]').forEach((container) => { + const source = container.querySelector('.feed-media-source'); + const feature = container.querySelector('.feed-media-feature'); + if (!source || !feature) return; + + const videos = [...source.querySelectorAll('video')].map((item) => { + const video = item.cloneNode(true); + video.removeAttribute('poster'); + video.controls = true; + video.playsInline = true; + video.preload = 'metadata'; + return video; + }); + const images = [...source.querySelectorAll('.kg-gallery-card img, .kg-image-card img')].map((item) => { + const link = document.createElement('a'); + link.href = item.currentSrc || item.src; + link.setAttribute('aria-label', item.alt ? `Открыть: ${item.alt}` : 'Открыть изображение'); + link.appendChild(item.cloneNode(true)); + return link; + }); + + const items = videos.length ? [...videos, ...images] : [feature, ...images]; + if (items.length === 1 && !videos.length) return; + const track = document.createElement('div'); + track.className = 'feed-media-track'; + items.forEach((item) => { + const slide = document.createElement('div'); + slide.className = 'feed-media-slide'; + slide.appendChild(item); + track.appendChild(slide); + }); + container.replaceChildren(track, source); + if (items.length > 1) { + const count = document.createElement('span'); + count.className = 'feed-media-count'; + count.textContent = `1 / ${items.length}`; + container.appendChild(count); + track.addEventListener('scroll', () => { + const index = Math.round(track.scrollLeft / Math.max(track.clientWidth, 1)); + count.textContent = `${Math.min(index + 1, items.length)} / ${items.length}`; + }, { passive: true }); + } + }); +})(); + +(() => { + const button = document.querySelector('[data-menu-toggle]'); + const nav = document.querySelector('#main-nav'); + if (!button || !nav) return; + button.addEventListener('click', () => { + const open = nav.classList.toggle('is-open'); + button.setAttribute('aria-expanded', String(open)); + }); +})(); + +(() => { + const toggle = document.querySelector('[data-category-toggle]'); + const menu = document.querySelector('[data-category-menu]'); + if (!toggle || !menu) return; + const close = () => { menu.classList.remove('is-open'); toggle.setAttribute('aria-expanded', 'false'); }; + toggle.addEventListener('click', (event) => { + event.stopPropagation(); + const open = menu.classList.toggle('is-open'); + toggle.setAttribute('aria-expanded', String(open)); + }); + document.addEventListener('click', (event) => { if (!menu.contains(event.target)) close(); }); + document.addEventListener('keydown', (event) => { if (event.key === 'Escape') close(); }); +})(); + +(() => { + const images = document.querySelectorAll('.gh-content .kg-gallery-image img'); + if (!images.length || !('HTMLDialogElement' in window)) return; + const dialog = document.createElement('dialog'); + dialog.className = 'media-lightbox'; + dialog.innerHTML = ''; + const preview = dialog.querySelector('img'); + dialog.querySelector('button').addEventListener('click', () => dialog.close()); + dialog.addEventListener('click', (event) => { if (event.target === dialog) dialog.close(); }); + document.body.appendChild(dialog); + images.forEach((image) => image.addEventListener('click', () => { + preview.src = image.currentSrc || image.src; + preview.alt = image.alt || ''; + dialog.showModal(); + })); +})(); diff --git a/forma_n8_site/ghost-theme/forma-n8/default.hbs b/forma_n8_site/ghost-theme/forma-n8/default.hbs new file mode 100644 index 0000000..647acc2 --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/default.hbs @@ -0,0 +1,81 @@ + + + + + + + + + {{#is "home"}} + Forma N8 — все новости снаряжения в одном месте + {{else}} + {{#is "post"}} + {{#post}}Forma N8 — {{title}}{{/post}} + {{else}} + {{#is "tag"}} + {{#tag}}Forma N8 — {{name}}{{/tag}} + {{else}} + Forma N8 — {{meta_title}} + {{/is}} + {{/is}} + {{/is}} + + + {{ghost_head}} + + + + + + +
+
+ + + FORMA
N8 + + +
+ {{> "social-links" compact=true}} + + +
+
+
+
{{{body}}}
+ + + + + {{ghost_foot}} + + diff --git a/forma_n8_site/ghost-theme/forma-n8/error.hbs b/forma_n8_site/ghost-theme/forma-n8/error.hbs new file mode 100644 index 0000000..765c750 --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/error.hbs @@ -0,0 +1,2 @@ +{{!< default}} +

{{statusCode}}

{{message}}

Вернуться в ленту
diff --git a/forma_n8_site/ghost-theme/forma-n8/index.hbs b/forma_n8_site/ghost-theme/forma-n8/index.hbs new file mode 100644 index 0000000..bf08724 --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/index.hbs @@ -0,0 +1,38 @@ +{{!< default}} +
+

Новости тактического снаряжения и экипировки

+ {{#foreach posts}} + {{#if @first}} + {{#is "home"}} +
+
+ {{#if feature_image}}{{#if feature_image_alt}}{{feature_image_alt}}{{else}}{{title}}{{/if}}{{else}}{{/if}} + +
+
+
{{#if primary_tag}}{{primary_tag.name}}{{/if}}
+

{{title}}

+

{{excerpt words="42"}}

+ Читать новость +
{{> "social-links" compact=true}}
+
+
+ {{else}} + {{> "post-row"}} + {{/is}} + {{else}} + {{> "post-row"}} + {{/if}} + {{else}} +

Публикаций пока нет

Лента скоро наполнится новостями.

+ {{/foreach}} + {{> "pagination-ru"}} + {{#is "home"}} +
+

О новостной ленте Forma N8

+

Forma N8 — лента новостей о тактическом снаряжении, экипировке и EDC. Мы собираем новинки с десятков источников и публикуем в одном месте: плитоносцы и бронезащита, подсумки и разгрузочные системы, боевые брюки и рубашки, прицелы и оптика, тактические фонари, средства связи, медицина и ночные приборы.

+

В ленте — новинки российских и зарубежных производителей: Ars Arma, Wartech, SSO, Stich Profi, Giena Tactical, 5.45 Design, Red Cliff, VOIN, Sturmer, БАРС, SRVV, ANA Tactical, Mordor Tac, Centurion Gear, ENOTACTICAL, Black Buckle, MILGEAR, SKIF Armor, Atlant Armour, БронеЁж, R5 GEAR, VINDEX, JAEGER Equipment, Margul Gear, Gordeev Tactical, IGLA TAC, STRIX TAC, Колчуга, APEX, Пересвет, Гарсинг и многих других. Отслеживаем оптику Диполь, ARTELV и ARKON, гарнитуры VULCAN, Fidem и ОКБ Октава, фонари Nicron, патчи BAD GRINGO и RedWar Patch, ножи DAGGERR, Morpheus Knives и N.C.Custom.

+

Темы ленты охватывают весь комплект тактической экипировки: системы MOLLE и Molle-Minus, плиты стандартов Гранит, SAPI и ESAPI, ткани Cordura и NYCO, расцветки Multicam, Tiger Stripe и EMR. Новинки магазинов Академия Снаряжения, Semper Fi, ФИНИСТ и Империя ножей, обновления линеек, цены и отзывы пользователей — всё в одной новостной ленте о снаряжении.

+
+ {{/is}} +
diff --git a/forma_n8_site/ghost-theme/forma-n8/package.json b/forma_n8_site/ghost-theme/forma-n8/package.json new file mode 100644 index 0000000..4dc677a --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/package.json @@ -0,0 +1,18 @@ +{ + "name": "forma-n8", + "description": "Strict editorial theme for Forma N8", + "version": "1.0.0", + "author": { + "name": "Forma N8", + "email": "admin@f-n8.ru" + }, + "keywords": ["ghost-theme", "forma-n8", "news"], + "engines": { + "ghost": ">=6.0.0" + }, + "license": "UNLICENSED", + "config": { + "posts_per_page": 25, + "card_assets": true + } +} diff --git a/forma_n8_site/ghost-theme/forma-n8/page.hbs b/forma_n8_site/ghost-theme/forma-n8/page.hbs new file mode 100644 index 0000000..8c0d99d --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/page.hbs @@ -0,0 +1,2 @@ +{{!< default}} +{{#post}}
{{#if @page.show_title_and_feature_image}}

{{title}}

{{#if custom_excerpt}}

{{custom_excerpt}}

{{/if}}
{{#if feature_image}}
{{title}}
{{/if}}{{/if}}
{{content}}
{{/post}} diff --git a/forma_n8_site/ghost-theme/forma-n8/partials/pagination-ru.hbs b/forma_n8_site/ghost-theme/forma-n8/partials/pagination-ru.hbs new file mode 100644 index 0000000..4be7bf4 --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/partials/pagination-ru.hbs @@ -0,0 +1,9 @@ +{{#if pagination.pages}} + {{#match pagination.pages ">" 1}} + + {{/match}} +{{/if}} diff --git a/forma_n8_site/ghost-theme/forma-n8/partials/post-row.hbs b/forma_n8_site/ghost-theme/forma-n8/partials/post-row.hbs new file mode 100644 index 0000000..bf38e4a --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/partials/post-row.hbs @@ -0,0 +1,12 @@ +
+
+ {{#if feature_image}}{{#if feature_image_alt}}{{feature_image_alt}}{{else}}{{title}}{{/if}}{{else}}{{/if}} + +
+
+
{{#if primary_tag}}{{primary_tag.name}}{{/if}}
+

{{title}}

+

{{excerpt words="28"}}

+
{{> "social-links" compact=true}}
+
+
diff --git a/forma_n8_site/ghost-theme/forma-n8/partials/social-links.hbs b/forma_n8_site/ghost-theme/forma-n8/partials/social-links.hbs new file mode 100644 index 0000000..119b5f4 --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/partials/social-links.hbs @@ -0,0 +1,10 @@ +
+ + + {{#unless compact}}ВКонтакте{{/unless}} + + + + {{#unless compact}}Telegram{{/unless}} + +
diff --git a/forma_n8_site/ghost-theme/forma-n8/post.hbs b/forma_n8_site/ghost-theme/forma-n8/post.hbs new file mode 100644 index 0000000..4108cdf --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/post.hbs @@ -0,0 +1,20 @@ +{{!< default}} +{{#post}} +
+
+
Лента{{#if primary_tag}}/{{primary_tag.name}}{{/if}}
+

{{title}}

+ {{#if custom_excerpt}}

{{custom_excerpt}}

{{/if}} +
+
+ {{#if feature_image}}
{{#if feature_image_alt}}{{feature_image_alt}}{{else}}{{title}}{{/if}}
{{/if}} +
+
{{content}}
+ +
+
+{{/post}} diff --git a/forma_n8_site/ghost-theme/forma-n8/tag.hbs b/forma_n8_site/ghost-theme/forma-n8/tag.hbs new file mode 100644 index 0000000..61e8ee2 --- /dev/null +++ b/forma_n8_site/ghost-theme/forma-n8/tag.hbs @@ -0,0 +1,6 @@ +{{!< default}} +
+ {{#tag}}

Тема

{{name}}

{{#if description}}

{{description}}

{{/if}}
{{/tag}} +
{{#foreach posts}}{{> "post-row"}}{{/foreach}}
+ {{> "pagination-ru"}} +
diff --git a/forma_n8_site/requirements.txt b/forma_n8_site/requirements.txt new file mode 100644 index 0000000..d02a6d6 --- /dev/null +++ b/forma_n8_site/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.115.12 +uvicorn[standard]==0.34.2 +asyncpg==0.30.0 +jinja2==3.1.6 +python-multipart==0.0.20 +pydantic-settings==2.9.1 diff --git a/forma_n8_site/scripts/migrate.py b/forma_n8_site/scripts/migrate.py new file mode 100644 index 0000000..a1ea41f --- /dev/null +++ b/forma_n8_site/scripts/migrate.py @@ -0,0 +1,28 @@ +import asyncio +import os +from pathlib import Path + +import asyncpg + + +async def main() -> None: + root = Path(__file__).resolve().parents[1] + conn = await asyncpg.connect(os.environ["DATABASE_URL"]) + try: + await conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW())" + ) + for path in sorted((root / "db" / "migrations").glob("*.sql")): + exists = await conn.fetchval("SELECT 1 FROM schema_migrations WHERE version=$1", path.name) + if exists: + continue + async with conn.transaction(): + await conn.execute(path.read_text(encoding="utf-8")) + await conn.execute("INSERT INTO schema_migrations(version) VALUES($1)", path.name) + print(f"applied {path.name}") + finally: + await conn.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/forma_n8_site/src/forma_n8_site/__init__.py b/forma_n8_site/src/forma_n8_site/__init__.py new file mode 100644 index 0000000..805a5c2 --- /dev/null +++ b/forma_n8_site/src/forma_n8_site/__init__.py @@ -0,0 +1 @@ +"""Forma N8 public website.""" diff --git a/forma_n8_site/src/forma_n8_site/app.py b/forma_n8_site/src/forma_n8_site/app.py new file mode 100644 index 0000000..069ef3f --- /dev/null +++ b/forma_n8_site/src/forma_n8_site/app.py @@ -0,0 +1,448 @@ +import hashlib +import hmac +import json +from contextlib import asynccontextmanager +from datetime import datetime, timezone +from pathlib import Path +from zoneinfo import ZoneInfo +from xml.etree import ElementTree as ET + +from fastapi import Depends, FastAPI, File, Header, HTTPException, Query, Request, UploadFile +from fastapi.responses import FileResponse, HTMLResponse, PlainTextResponse, Response +from fastapi.staticfiles import StaticFiles +from fastapi.templating import Jinja2Templates + +from .config import settings +from .db import connect, disconnect, get_pool +from .schemas import ArticleUpsert +from .text import excerpt, paragraphs, slugify + + +ROOT = Path(__file__).resolve().parents[2] +STATIC_DIR = ROOT / "static" +TEMPLATE_DIR = ROOT / "templates" +ALLOWED_UPLOADS = { + "image/jpeg": ".jpg", + "image/png": ".png", + "image/webp": ".webp", + "image/gif": ".gif", + "video/mp4": ".mp4", + "video/webm": ".webm", +} + + +@asynccontextmanager +async def lifespan(_: FastAPI): + settings.upload_dir.mkdir(parents=True, exist_ok=True) + await connect() + yield + await disconnect() + + +app = FastAPI(title=settings.site_name, docs_url=None, redoc_url=None, lifespan=lifespan) +app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static") +app.mount("/media", StaticFiles(directory=settings.upload_dir, check_dir=False), name="media") +templates = Jinja2Templates(directory=TEMPLATE_DIR) +templates.env.filters["paragraphs"] = paragraphs +templates.env.filters["excerpt"] = excerpt +LOCAL_TZ = ZoneInfo("Asia/Yekaterinburg") +templates.env.filters["ekb_date"] = lambda value: value.astimezone(LOCAL_TZ).strftime("%d.%m.%Y") if value else "" +templates.env.filters["ekb_time"] = lambda value: value.astimezone(LOCAL_TZ).strftime("%H:%M") if value else "" + + +def site_url(path: str = "") -> str: + return f"{settings.base_url}{path if path.startswith('/') else '/' + path if path else ''}" + + +def article_path(article: dict) -> str: + return f"/news/{article['slug']}" + + +templates.env.globals.update( + site_name=settings.site_name, + site_tagline=settings.site_tagline, + site_url=site_url, + article_path=article_path, +) + + +@app.middleware("http") +async def security_headers(request: Request, call_next): + response = await call_next(request) + response.headers.setdefault("X-Content-Type-Options", "nosniff") + response.headers.setdefault("X-Frame-Options", "DENY") + response.headers.setdefault("Referrer-Policy", "strict-origin-when-cross-origin") + response.headers.setdefault("Permissions-Policy", "camera=(), microphone=(), geolocation=()") + return response + + +def base_context(request: Request, **extra): + return { + "request": request, + "current_path": request.url.path, + "canonical_url": site_url(request.url.path), + "meta_title": f"{settings.site_tagline} — {settings.site_name}", + "meta_description": "Новости производителей тактического снаряжения, одежды, брони и аксессуаров.", + "noindex": not settings.site_indexing_enabled or bool(request.query_params.get("q")), + **extra, + } + + +ARTICLE_COLUMNS = """ + a.*, + COALESCE( + jsonb_agg( + jsonb_build_object( + 'type', m.media_type, + 'url', m.url, + 'alt', m.alt_text, + 'width', m.width, + 'height', m.height + ) ORDER BY m.sort_order, m.id + ) FILTER (WHERE m.id IS NOT NULL), + '[]'::jsonb + ) AS media +""" + + +def normalize_article(row) -> dict: + article = dict(row) + media = article.get("media") or [] + if isinstance(media, str): + try: + media = json.loads(media) + except json.JSONDecodeError: + media = [] + article["media"] = media if isinstance(media, list) else [] + return article + + +async def fetch_feed(*, page: int, q: str = "", category: str = "", producer: str = "", page_size: int | None = None): + pool = await get_pool() + where = ["a.status='published'", "a.published_at <= NOW()"] + args: list[object] = [] + if q: + args.append(f"%{q}%") + where.append(f"(a.title ILIKE ${len(args)} OR a.lead ILIKE ${len(args)} OR a.body ILIKE ${len(args)} OR a.producer_name ILIKE ${len(args)})") + if category: + args.append(category) + where.append(f"a.category_tag=${len(args)}") + if producer: + args.append(producer) + where.append(f"a.producer_tag=${len(args)}") + where_sql = " AND ".join(where) + total = int(await pool.fetchval(f"SELECT COUNT(*) FROM articles a WHERE {where_sql}", *args) or 0) + limit = max(1, min(50, page_size or settings.page_size)) + offset = (page - 1) * limit + rows = await pool.fetch( + f""" + SELECT {ARTICLE_COLUMNS} + FROM articles a + LEFT JOIN article_media m ON m.article_id=a.id + WHERE {where_sql} + GROUP BY a.id + ORDER BY a.published_at DESC, a.id DESC + LIMIT ${len(args) + 1} OFFSET ${len(args) + 2} + """, + *args, + limit, + offset, + ) + return [normalize_article(row) for row in rows], total, limit + + +@app.get("/", response_class=HTMLResponse) +async def feed(request: Request, page: int = Query(1, ge=1), q: str = Query("", max_length=100)): + articles, total, limit = await fetch_feed(page=page, q=q.strip()) + pages = max(1, (total + limit - 1) // limit) + canonical = site_url("/") if page == 1 else site_url(f"/?page={page}") + return templates.TemplateResponse( + "feed.html", + base_context( + request, + articles=articles, + page=page, + pages=pages, + total=total, + q=q.strip(), + canonical_url=canonical, + ), + ) + + +@app.get("/category/{tag}", response_class=HTMLResponse) +async def category_feed(request: Request, tag: str, page: int = Query(1, ge=1)): + pool = await get_pool() + category_name = await pool.fetchval("SELECT name FROM categories WHERE tag=$1 AND is_active=TRUE", tag) + if not category_name: + raise HTTPException(status_code=404) + articles, total, limit = await fetch_feed(page=page, category=tag) + name = str(category_name) + pages = max(1, (total + limit - 1) // limit) + return templates.TemplateResponse( + "taxonomy.html", + base_context( + request, + articles=articles, + page=page, + pages=pages, + heading=name, + taxonomy_label="Категория", + canonical_url=site_url(f"/category/{tag}" + (f"?page={page}" if page > 1 else "")), + meta_title=f"{name}: новости — {settings.site_name}", + meta_description=f"Последние новости в категории «{name}»: производители, новинки и обзоры снаряжения.", + noindex=total == 0, + ), + ) + + +@app.get("/brand/{tag}", response_class=HTMLResponse) +async def producer_feed(request: Request, tag: str, page: int = Query(1, ge=1)): + articles, total, limit = await fetch_feed(page=page, producer=tag) + if not articles and page == 1: + raise HTTPException(status_code=404) + name = articles[0]["producer_name"] if articles else tag + pages = max(1, (total + limit - 1) // limit) + return templates.TemplateResponse( + "taxonomy.html", + base_context( + request, + articles=articles, + page=page, + pages=pages, + heading=name, + taxonomy_label="Производитель", + canonical_url=site_url(f"/brand/{tag}" + (f"?page={page}" if page > 1 else "")), + meta_title=f"{name}: новости и новинки — {settings.site_name}", + meta_description=f"Новости и новинки производителя {name}: снаряжение, одежда и экипировка.", + ), + ) + + +@app.get("/news/{slug}", response_class=HTMLResponse) +async def article(request: Request, slug: str): + pool = await get_pool() + row = await pool.fetchrow( + f""" + SELECT {ARTICLE_COLUMNS} + FROM articles a + LEFT JOIN article_media m ON m.article_id=a.id + WHERE a.slug=$1 AND a.status='published' AND a.published_at <= NOW() + GROUP BY a.id + """, + slug, + ) + if not row: + raise HTTPException(status_code=404) + item = normalize_article(row) + related = await pool.fetch( + f""" + SELECT {ARTICLE_COLUMNS} + FROM articles a + LEFT JOIN article_media m ON m.article_id=a.id + WHERE a.status='published' AND a.id<>$1 + AND (a.category_tag=$2 OR a.producer_tag=$3) + GROUP BY a.id + ORDER BY (a.producer_tag=$3) DESC, a.published_at DESC + LIMIT 3 + """, + item["id"], + item["category_tag"], + item["producer_tag"], + ) + canonical = site_url(article_path(item)) + image = item["media"][0]["url"] if item["media"] else site_url("/static/favi.png") + if image.startswith("/"): + image = site_url(image) + json_ld = { + "@context": "https://schema.org", + "@type": "NewsArticle", + "headline": item["title"], + "description": item["seo_description"] or item["lead"] or excerpt(item["body"]), + "datePublished": item["published_at"].isoformat(), + "dateModified": item["updated_at"].isoformat(), + "mainEntityOfPage": canonical, + "image": [image], + "author": {"@type": "Organization", "name": settings.site_name}, + "publisher": { + "@type": "Organization", + "name": settings.site_name, + "logo": {"@type": "ImageObject", "url": site_url("/static/favi.png")}, + }, + } + return templates.TemplateResponse( + "article.html", + base_context( + request, + article=item, + related=[normalize_article(x) for x in related], + canonical_url=canonical, + meta_title=item["seo_title"] or f"{item['title']} — {settings.site_name}", + meta_description=item["seo_description"] or item["lead"] or excerpt(item["body"]), + og_image=image, + json_ld=json.dumps(json_ld, ensure_ascii=False), + ), + ) + + +def require_api_token(x_forma_token: str = Header(default="")) -> None: + if not settings.publish_api_token or not hmac.compare_digest(x_forma_token, settings.publish_api_token): + raise HTTPException(status_code=401, detail="invalid token") + + +@app.post("/api/v1/media", dependencies=[Depends(require_api_token)]) +async def upload_media(file: UploadFile = File(...)): + content_type = (file.content_type or "").split(";", 1)[0].lower() + if content_type not in ALLOWED_UPLOADS: + raise HTTPException(status_code=415, detail="unsupported media type") + data = await file.read(50 * 1024 * 1024 + 1) + if len(data) > 50 * 1024 * 1024: + raise HTTPException(status_code=413, detail="file too large") + digest = hashlib.sha256(data).hexdigest() + suffix = ALLOWED_UPLOADS[content_type] + target = settings.upload_dir / digest[:2] / f"{digest}{suffix}" + target.parent.mkdir(parents=True, exist_ok=True) + if not target.exists(): + target.write_bytes(data) + return {"url": f"/media/{digest[:2]}/{target.name}", "sha256": digest, "content_type": content_type} + + +@app.post("/api/v1/articles", dependencies=[Depends(require_api_token)]) +async def upsert_article(payload: ArticleUpsert): + pool = await get_pool() + existing = await pool.fetchrow("SELECT id, slug, published_at FROM articles WHERE external_id=$1", payload.external_id) + requested_slug = slugify(payload.slug or payload.title) + stable_slug = existing["slug"] if existing else f"{requested_slug}-{slugify(payload.external_id)[-24:]}" + published_at = payload.published_at or datetime.now(timezone.utc) + category_tag = slugify(payload.category_tag) + category_name = await pool.fetchval("SELECT name FROM categories WHERE tag=$1 AND is_active=TRUE", category_tag) + category_name = str(category_name or payload.category_name).strip() + async with pool.acquire() as conn: + async with conn.transaction(): + row = await conn.fetchrow( + """ + INSERT INTO articles( + external_id, slug, title, lead, body, category_name, category_tag, + producer_name, producer_tag, source_url, seo_title, seo_description, + status, published_at + ) + VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) + ON CONFLICT(external_id) DO UPDATE SET + title=EXCLUDED.title, + lead=EXCLUDED.lead, + body=EXCLUDED.body, + category_name=EXCLUDED.category_name, + category_tag=EXCLUDED.category_tag, + producer_name=EXCLUDED.producer_name, + producer_tag=EXCLUDED.producer_tag, + source_url=EXCLUDED.source_url, + seo_title=EXCLUDED.seo_title, + seo_description=EXCLUDED.seo_description, + status=EXCLUDED.status, + published_at=COALESCE(articles.published_at, EXCLUDED.published_at), + updated_at=NOW() + RETURNING id, slug, status, published_at, updated_at + """, + payload.external_id, + stable_slug, + payload.title.strip(), + payload.lead.strip(), + payload.body.strip(), + category_name, + category_tag, + payload.producer_name.strip(), + slugify(payload.producer_tag), + str(payload.source_url) if payload.source_url else None, + (payload.seo_title or payload.title).strip(), + (payload.seo_description or payload.lead or excerpt(payload.body)).strip(), + payload.status, + published_at, + ) + await conn.execute("DELETE FROM article_media WHERE article_id=$1", row["id"]) + for index, media in enumerate(payload.media): + await conn.execute( + """ + INSERT INTO article_media(article_id, media_type, url, alt_text, width, height, sort_order) + VALUES($1,$2,$3,$4,$5,$6,$7) + """, + row["id"], media.type, media.url, media.alt or payload.title, + media.width, media.height, media.sort_order or index, + ) + return { + "id": row["id"], + "external_id": payload.external_id, + "status": row["status"], + "url": site_url(f"/news/{row['slug']}") if row["status"] == "published" else None, + "published_at": row["published_at"], + "updated_at": row["updated_at"], + } + + +@app.get("/rss.xml") +async def rss(): + articles, _, _ = await fetch_feed(page=1, page_size=50) + root = ET.Element("rss", {"version": "2.0", "xmlns:media": "http://search.yahoo.com/mrss/"}) + channel = ET.SubElement(root, "channel") + ET.SubElement(channel, "title").text = f"{settings.site_name} — {settings.site_tagline}" + ET.SubElement(channel, "link").text = settings.base_url + ET.SubElement(channel, "description").text = "Новости производителей тактического снаряжения и экипировки." + ET.SubElement(channel, "language").text = "ru" + for item in articles[:50]: + node = ET.SubElement(channel, "item") + url = site_url(article_path(item)) + ET.SubElement(node, "title").text = item["title"] + ET.SubElement(node, "link").text = url + ET.SubElement(node, "guid", {"isPermaLink": "true"}).text = url + ET.SubElement(node, "description").text = item["lead"] or excerpt(item["body"]) + ET.SubElement(node, "pubDate").text = item["published_at"].strftime("%a, %d %b %Y %H:%M:%S +0000") + ET.SubElement(node, "category").text = item["category_name"] + if item["media"]: + ET.SubElement(node, "media:content", {"url": site_url(item["media"][0]["url"]) if item["media"][0]["url"].startswith("/") else item["media"][0]["url"], "medium": "image"}) + return Response(ET.tostring(root, encoding="utf-8", xml_declaration=True), media_type="application/rss+xml") + + +@app.get("/sitemap.xml") +async def sitemap(): + pool = await get_pool() + rows = await pool.fetch("SELECT slug, updated_at FROM articles WHERE status='published' ORDER BY id") + categories = await pool.fetch("SELECT category_tag AS tag, MAX(updated_at) AS updated_at FROM articles WHERE status='published' GROUP BY category_tag") + producers = await pool.fetch("SELECT producer_tag AS tag, MAX(updated_at) AS updated_at FROM articles WHERE status='published' GROUP BY producer_tag") + root = ET.Element("urlset", {"xmlns": "http://www.sitemaps.org/schemas/sitemap/0.9"}) + entries = [(site_url("/"), None)] + entries += [(site_url(f"/category/{row['tag']}"), row["updated_at"]) for row in categories] + entries += [(site_url(f"/brand/{row['tag']}"), row["updated_at"]) for row in producers] + entries += [(site_url(f"/news/{row['slug']}"), row["updated_at"]) for row in rows] + for loc, updated in entries: + node = ET.SubElement(root, "url") + ET.SubElement(node, "loc").text = loc + if updated: + ET.SubElement(node, "lastmod").text = updated.date().isoformat() + return Response(ET.tostring(root, encoding="utf-8", xml_declaration=True), media_type="application/xml") + + +@app.get("/robots.txt", response_class=PlainTextResponse) +async def robots(): + if not settings.site_indexing_enabled: + return "User-agent: *\nDisallow: /\n" + return f"User-agent: *\nAllow: /\nDisallow: /api/\nSitemap: {site_url('/sitemap.xml')}\n" + + +@app.get("/favicon.ico", include_in_schema=False) +async def favicon(): + return FileResponse(STATIC_DIR / "favi.png", media_type="image/png") + + +@app.get("/health") +async def health(): + pool = await get_pool() + await pool.fetchval("SELECT 1") + return {"status": "ok"} + + +@app.exception_handler(404) +async def not_found(request: Request, _): + return templates.TemplateResponse( + "404.html", + base_context(request, meta_title=f"Страница не найдена — {settings.site_name}", noindex=True), + status_code=404, + ) diff --git a/forma_n8_site/src/forma_n8_site/config.py b/forma_n8_site/src/forma_n8_site/config.py new file mode 100644 index 0000000..cbe716b --- /dev/null +++ b/forma_n8_site/src/forma_n8_site/config.py @@ -0,0 +1,23 @@ +from pathlib import Path + +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + database_url: str = "postgresql://forma_n8:forma_n8@127.0.0.1:5432/forma_n8" + site_base_url: str = "http://127.0.0.1:8090" + site_name: str = "Forma N8" + site_tagline: str = "Новости снаряжения" + site_indexing_enabled: bool = False + publish_api_token: str = "" + upload_dir: Path = Path("/var/lib/forma-n8/uploads") + page_size: int = 12 + + model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") + + @property + def base_url(self) -> str: + return self.site_base_url.rstrip("/") + + +settings = Settings() diff --git a/forma_n8_site/src/forma_n8_site/db.py b/forma_n8_site/src/forma_n8_site/db.py new file mode 100644 index 0000000..696ba9d --- /dev/null +++ b/forma_n8_site/src/forma_n8_site/db.py @@ -0,0 +1,24 @@ +import asyncpg + +from .config import settings + + +pool: asyncpg.Pool | None = None + + +async def connect() -> asyncpg.Pool: + global pool + if pool is None: + pool = await asyncpg.create_pool(settings.database_url, min_size=1, max_size=10, command_timeout=30) + return pool + + +async def disconnect() -> None: + global pool + if pool is not None: + await pool.close() + pool = None + + +async def get_pool() -> asyncpg.Pool: + return pool or await connect() diff --git a/forma_n8_site/src/forma_n8_site/schemas.py b/forma_n8_site/src/forma_n8_site/schemas.py new file mode 100644 index 0000000..d9945b4 --- /dev/null +++ b/forma_n8_site/src/forma_n8_site/schemas.py @@ -0,0 +1,31 @@ +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel, Field, HttpUrl + + +class MediaInput(BaseModel): + type: Literal["image", "video"] = "image" + url: str = Field(min_length=1, max_length=2000) + alt: str = Field(default="", max_length=500) + width: int | None = Field(default=None, ge=1) + height: int | None = Field(default=None, ge=1) + sort_order: int = Field(default=0, ge=0) + + +class ArticleUpsert(BaseModel): + external_id: str = Field(min_length=1, max_length=160) + slug: str | None = Field(default=None, max_length=220) + title: str = Field(min_length=5, max_length=300) + lead: str = Field(default="", max_length=1000) + body: str = Field(min_length=20, max_length=50000) + category_name: str = Field(min_length=1, max_length=120) + category_tag: str = Field(min_length=1, max_length=120) + producer_name: str = Field(min_length=1, max_length=200) + producer_tag: str = Field(min_length=1, max_length=160) + source_url: HttpUrl | None = None + seo_title: str | None = Field(default=None, max_length=300) + seo_description: str | None = Field(default=None, max_length=500) + status: Literal["draft", "published"] = "published" + published_at: datetime | None = None + media: list[MediaInput] = Field(default_factory=list, max_length=20) diff --git a/forma_n8_site/src/forma_n8_site/text.py b/forma_n8_site/src/forma_n8_site/text.py new file mode 100644 index 0000000..0dcbe29 --- /dev/null +++ b/forma_n8_site/src/forma_n8_site/text.py @@ -0,0 +1,37 @@ +import re +from html import escape + +from markupsafe import Markup + + +TRANSLIT = str.maketrans( + { + "а": "a", "б": "b", "в": "v", "г": "g", "д": "d", "е": "e", "ё": "e", + "ж": "zh", "з": "z", "и": "i", "й": "y", "к": "k", "л": "l", "м": "m", + "н": "n", "о": "o", "п": "p", "р": "r", "с": "s", "т": "t", "у": "u", + "ф": "f", "х": "h", "ц": "c", "ч": "ch", "ш": "sh", "щ": "sch", + "ъ": "", "ы": "y", "ь": "", "э": "e", "ю": "yu", "я": "ya", + } +) + + +def slugify(value: str) -> str: + value = value.strip().lower().translate(TRANSLIT) + value = re.sub(r"[^a-z0-9]+", "-", value).strip("-") + return value[:180] or "article" + + +def paragraphs(value: str) -> Markup: + blocks = [] + for part in re.split(r"\n\s*\n", value.strip()): + lines = "
".join(escape(line.strip()) for line in part.splitlines() if line.strip()) + if lines: + blocks.append(f"

{lines}

") + return Markup("".join(blocks)) + + +def excerpt(value: str, limit: int = 180) -> str: + compact = re.sub(r"\s+", " ", value or "").strip() + if len(compact) <= limit: + return compact + return compact[: limit - 1].rsplit(" ", 1)[0] + "…" diff --git a/forma_n8_site/static/favi.png b/forma_n8_site/static/favi.png new file mode 100644 index 0000000..23a1268 Binary files /dev/null and b/forma_n8_site/static/favi.png differ diff --git a/forma_n8_site/static/fonts/roboto-condensed-cyrillic-400.woff2 b/forma_n8_site/static/fonts/roboto-condensed-cyrillic-400.woff2 new file mode 100644 index 0000000..10138a0 Binary files /dev/null and b/forma_n8_site/static/fonts/roboto-condensed-cyrillic-400.woff2 differ diff --git a/forma_n8_site/static/fonts/roboto-condensed-cyrillic-700.woff2 b/forma_n8_site/static/fonts/roboto-condensed-cyrillic-700.woff2 new file mode 100644 index 0000000..c001da3 Binary files /dev/null and b/forma_n8_site/static/fonts/roboto-condensed-cyrillic-700.woff2 differ diff --git a/forma_n8_site/static/site.css b/forma_n8_site/static/site.css new file mode 100644 index 0000000..9c590af --- /dev/null +++ b/forma_n8_site/static/site.css @@ -0,0 +1,348 @@ +@font-face { + font-family: "Roboto Condensed"; + src: url("/static/fonts/roboto-condensed-cyrillic-400.woff2") format("woff2"); + font-weight: 400; + font-display: swap; +} + +@font-face { + font-family: "Roboto Condensed"; + src: url("/static/fonts/roboto-condensed-cyrillic-700.woff2") format("woff2"); + font-weight: 700; + font-display: swap; +} + +:root { + color-scheme: dark; + --bg: #090c0d; + --surface: #101415; + --surface-strong: #151a1b; + --text: #eceee8; + --muted: #9ba19b; + --faint: #686f6b; + --border: #29302e; + --border-strong: #3a423f; + --olive: #a5ad62; + --red: #df4a40; + --max: 1240px; + --gutter: 24px; + --font: "Roboto Condensed", "Arial Narrow", Arial, sans-serif; +} + +* { box-sizing: border-box; } + +html { background: var(--bg); scroll-behavior: smooth; } + +body { + margin: 0; + min-width: 320px; + background: var(--bg); + color: var(--text); + font-family: var(--font); + font-size: 18px; + line-height: 1.55; + letter-spacing: 0; + -webkit-font-smoothing: antialiased; +} + +body::before { + content: ""; + position: fixed; + inset: 0; + pointer-events: none; + opacity: .22; + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='120' height='120' viewBox='0 0 120 120'%3E%3Cfilter id='n'%3E%3CfeTurbulence type='fractalNoise' baseFrequency='.8' numOctaves='2' stitchTiles='stitch'/%3E%3C/filter%3E%3Crect width='100%25' height='100%25' filter='url(%23n)' opacity='.08'/%3E%3C/svg%3E"); +} + +a { color: inherit; text-decoration: none; } +a:hover { color: var(--olive); } +img, video { display: block; max-width: 100%; } +button, input { font: inherit; letter-spacing: 0; } + +:focus-visible { outline: 2px solid var(--olive); outline-offset: 4px; } + +.skip-link { + position: fixed; + z-index: 100; + top: 10px; + left: 10px; + transform: translateY(-160%); + padding: 10px 14px; + background: var(--text); + color: var(--bg); +} +.skip-link:focus { transform: none; } + +.site-header { + position: sticky; + z-index: 20; + top: 0; + border-bottom: 1px solid var(--border); + background: rgba(9, 12, 13, .96); + backdrop-filter: blur(12px); +} + +.header-inner { + width: min(var(--max), calc(100% - var(--gutter) * 2)); + min-height: 76px; + margin: 0 auto; + display: grid; + grid-template-columns: 170px 1fr auto; + align-items: center; + gap: 32px; +} + +.brand { + display: inline-flex; + width: fit-content; + align-items: center; + gap: 11px; + font-size: 15px; + font-weight: 700; + line-height: .95; +} +.brand img { width: 42px; height: 42px; border: 1px solid var(--border-strong); } + +.main-nav { display: flex; justify-content: center; align-self: stretch; gap: 38px; } +.main-nav a { + position: relative; + display: inline-flex; + align-items: center; + color: var(--muted); + font-size: 16px; + white-space: nowrap; +} +.main-nav a:hover, .main-nav a[aria-current="page"] { color: var(--text); } +.main-nav a[aria-current="page"]::after { + content: ""; + position: absolute; + right: 0; + bottom: -1px; + left: 0; + height: 3px; + background: var(--olive); +} + +.header-actions { display: flex; gap: 8px; } +.icon-button { + display: grid; + width: 40px; + height: 40px; + padding: 0; + place-items: center; + border: 0; + background: transparent; + color: var(--text); + cursor: pointer; +} +.icon-button:hover { background: var(--surface-strong); color: var(--olive); } +.icon-button svg { width: 23px; height: 23px; fill: none; stroke: currentColor; stroke-width: 1.8; stroke-linecap: square; } +.menu-button { display: none; } + +.search-form { + width: min(760px, calc(100% - var(--gutter) * 2)); + margin: 0 auto; + padding: 16px 0 20px; +} +.search-form[hidden] { display: none; } +.search-form label { display: block; margin-bottom: 8px; color: var(--muted); font-size: 14px; } +.search-form > div { display: grid; grid-template-columns: 1fr auto; } +.search-form input { + min-width: 0; + height: 48px; + padding: 0 16px; + border: 1px solid var(--border-strong); + border-right: 0; + border-radius: 0; + background: var(--surface); + color: var(--text); + font-size: 17px; +} +.search-form button, .read-link, .empty-state a { + border: 1px solid var(--olive); + border-radius: 0; + background: transparent; + color: var(--text); + cursor: pointer; +} +.search-form button { padding: 0 24px; } +.search-form button:hover, .read-link:hover, .empty-state a:hover { background: var(--olive); color: var(--bg); } + +.feed-shell, .article-page, .related-section, .site-footer { + position: relative; + width: min(var(--max), calc(100% - var(--gutter) * 2)); + margin: 0 auto; +} + +.page-heading { padding: 56px 0 46px; } +.page-heading h1 { + max-width: 900px; + margin: 0; + font-size: 68px; + font-weight: 700; + line-height: .98; +} +.page-heading > p { margin: 20px 0 0; color: var(--muted); } + +.lead-story { + display: grid; + grid-template-columns: minmax(0, 1.25fr) minmax(340px, .95fr); + min-height: 480px; + border-bottom: 1px solid var(--border-strong); +} +.lead-media { min-height: 480px; overflow: hidden; background: var(--surface); } +.lead-media img, .lead-media video { width: 100%; height: 100%; object-fit: cover; transition: transform .45s ease; } +.lead-media:hover img { transform: scale(1.015); } +.lead-copy { display: flex; flex-direction: column; justify-content: center; padding: 34px 0 36px 44px; } +.lead-copy h2 { margin: 24px 0 22px; font-size: 40px; line-height: 1.14; } +.lead-copy h2 a:hover { color: var(--text); text-decoration: underline; text-decoration-color: var(--olive); text-underline-offset: 7px; } +.lead-copy > p { max-width: 580px; margin: 0; color: var(--muted); font-size: 20px; line-height: 1.5; } +.read-link { width: fit-content; margin-top: 30px; padding: 10px 15px; font-size: 15px; } +.read-link span { margin-left: 8px; } + +.article-meta { display: flex; align-items: center; flex-wrap: wrap; gap: 0; color: var(--muted); font-size: 14px; } +.article-meta > * { display: inline-flex; align-items: center; } +.article-meta > * + *::before { content: ""; width: 1px; height: 18px; margin: 0 16px; background: var(--border-strong); } +.article-meta .category-link { color: var(--olive); } +.article-meta .producer-link { color: var(--red); } + +.feed-list { width: 100%; } +.feed-list-top { border-top: 1px solid var(--border-strong); } +.feed-row { + display: grid; + grid-template-columns: 350px minmax(0, 1fr); + gap: 34px; + padding: 22px 0; + border-bottom: 1px solid var(--border); +} +.feed-row-media { aspect-ratio: 16 / 9; overflow: hidden; background: var(--surface); } +.feed-row-media img, .feed-row-media video { width: 100%; height: 100%; object-fit: cover; transition: transform .35s ease; } +.feed-row-media:hover img { transform: scale(1.02); } +.feed-row-copy { align-self: center; min-width: 0; } +.feed-row h2 { margin: 16px 0 8px; font-size: 25px; line-height: 1.2; } +.feed-row h2 a:hover { color: var(--text); text-decoration: underline; text-decoration-color: var(--olive); text-underline-offset: 5px; } +.feed-row p { max-width: 760px; margin: 0; color: var(--muted); font-size: 17px; line-height: 1.5; } +.media-placeholder { display: grid; width: 100%; height: 100%; place-items: center; background: var(--surface); } +.media-placeholder img { width: 76px; height: 76px; opacity: .28; object-fit: contain; } + +.pagination { display: grid; grid-template-columns: 1fr auto 1fr; align-items: center; min-height: 90px; font-size: 15px; } +.pagination > :last-child { justify-self: end; } +.pagination span { color: var(--faint); } + +.taxonomy-heading { padding-bottom: 34px; } +.taxonomy-heading > p { margin: 0 0 10px; color: var(--olive); font-size: 15px; text-transform: uppercase; } +.taxonomy-heading h1 { font-size: 56px; } + +.article-page { padding-top: 56px; } +.article-header { max-width: 960px; margin: 0 auto 42px; } +.article-breadcrumbs { display: flex; gap: 12px; margin-bottom: 28px; color: var(--muted); font-size: 14px; } +.article-breadcrumbs span { color: var(--faint); } +.article-header h1 { margin: 0; font-size: 58px; line-height: 1.06; } +.article-lead { max-width: 850px; margin: 26px 0 0; color: var(--muted); font-size: 23px; line-height: 1.45; } +.article-byline { display: flex; gap: 24px; margin-top: 28px; color: var(--muted); font-size: 14px; } +.article-byline a { color: var(--red); } +.article-media { display: grid; gap: 12px; margin-bottom: 46px; } +.article-media figure { margin: 0; background: var(--surface); } +.article-media img, .article-media video { width: 100%; max-height: 760px; object-fit: contain; } +.article-layout { display: grid; grid-template-columns: minmax(0, 760px) 260px; justify-content: center; gap: 78px; } +.article-body { font-size: 21px; line-height: 1.7; } +.article-body p { margin: 0 0 26px; } +.article-facts { align-self: start; border-top: 2px solid var(--olive); } +.article-facts > div { padding: 18px 0; border-bottom: 1px solid var(--border); } +.article-facts span { display: block; margin-bottom: 4px; color: var(--faint); font-size: 13px; text-transform: uppercase; } +.article-facts a { font-size: 17px; } +.article-facts .source-link { display: block; margin-top: 22px; color: var(--muted); font-size: 14px; } +.related-section { margin-top: 90px; border-top: 1px solid var(--border-strong); } +.related-section > h2 { margin: 0; padding: 34px 0 18px; font-size: 32px; } + +.empty-state, .not-found { min-height: 56vh; display: flex; flex-direction: column; justify-content: center; align-items: flex-start; } +.empty-state h2, .not-found h1 { margin: 0; font-size: 42px; } +.empty-state p { color: var(--muted); } +.empty-state a { margin-top: 16px; padding: 10px 15px; } +.not-found > p { margin: 0; color: var(--red); font-size: 18px; } +.not-found a { margin-top: 24px; color: var(--olive); } + +.site-footer { + display: flex; + justify-content: space-between; + align-items: flex-end; + margin-top: 84px; + padding: 34px 0 46px; + border-top: 1px solid var(--border-strong); + color: var(--muted); + font-size: 14px; +} +.footer-brand { color: var(--text); font-size: 20px; font-weight: 700; } +.site-footer p { margin: 7px 0 0; } +.site-footer nav { display: flex; gap: 24px; } + +@media (max-width: 980px) { + .header-inner { grid-template-columns: 150px 1fr auto; gap: 16px; } + .main-nav { gap: 22px; } + .main-nav a { font-size: 14px; } + .page-heading h1 { font-size: 56px; } + .lead-story { grid-template-columns: 1.1fr .9fr; min-height: 420px; } + .lead-media { min-height: 420px; } + .lead-copy { padding-left: 30px; } + .lead-copy h2 { font-size: 33px; } + .feed-row { grid-template-columns: 290px minmax(0, 1fr); gap: 26px; } + .article-layout { grid-template-columns: minmax(0, 680px) 220px; gap: 44px; } +} + +@media (max-width: 760px) { + :root { --gutter: 16px; } + body { font-size: 17px; } + .header-inner { min-height: 64px; grid-template-columns: 1fr auto; } + .brand img { width: 38px; height: 38px; } + .main-nav { + position: absolute; + top: 64px; + right: 0; + left: 0; + display: none; + padding: 12px var(--gutter) 18px; + border-bottom: 1px solid var(--border); + background: var(--bg); + } + .main-nav.is-open { display: grid; } + .main-nav a { min-height: 44px; border-bottom: 1px solid var(--border); font-size: 16px; } + .main-nav a[aria-current="page"]::after { display: none; } + .menu-button { display: grid; } + .page-heading { padding: 38px 0 30px; } + .page-heading h1, .taxonomy-heading h1 { font-size: 44px; line-height: 1; } + .lead-story { display: block; min-height: 0; } + .lead-media { min-height: 0; aspect-ratio: 4 / 3; } + .lead-copy { padding: 25px 0 34px; } + .lead-copy h2 { margin: 20px 0 15px; font-size: 32px; } + .lead-copy > p { font-size: 18px; } + .feed-row { grid-template-columns: 132px minmax(0, 1fr); gap: 16px; padding: 18px 0; } + .feed-row-media { aspect-ratio: 4 / 3; } + .feed-row h2 { margin: 10px 0 0; font-size: 20px; } + .feed-row p { display: none; } + .article-meta { font-size: 12px; } + .article-meta time { display: none; } + .article-meta > * + *::before { height: 14px; margin: 0 10px; } + .article-meta .category-link::before { display: none; } + .article-header { margin-bottom: 30px; } + .article-header h1 { font-size: 42px; } + .article-lead { font-size: 20px; } + .article-layout { display: block; } + .article-body { font-size: 19px; } + .article-facts { margin-top: 42px; } + .related-section { margin-top: 64px; } +} + +@media (max-width: 460px) { + .feed-row { grid-template-columns: 112px minmax(0, 1fr); gap: 13px; } + .feed-row h2 { font-size: 18px; line-height: 1.18; } + .article-meta .producer-link::before { display: none; } + .article-meta .category-link { display: none; } + .article-header h1 { font-size: 36px; } + .article-byline { display: grid; gap: 7px; } + .site-footer { display: grid; gap: 24px; } +} + +@media (prefers-reduced-motion: reduce) { + html { scroll-behavior: auto; } + *, *::before, *::after { transition-duration: .01ms !important; } +} diff --git a/forma_n8_site/static/site.js b/forma_n8_site/static/site.js new file mode 100644 index 0000000..debf74e --- /dev/null +++ b/forma_n8_site/static/site.js @@ -0,0 +1,16 @@ +const menuButton = document.querySelector('[data-menu-toggle]'); +const nav = document.querySelector('#main-nav'); +const searchButton = document.querySelector('[data-search-toggle]'); +const searchForm = document.querySelector('[data-search-form]'); + +menuButton?.addEventListener('click', () => { + const open = menuButton.getAttribute('aria-expanded') === 'true'; + menuButton.setAttribute('aria-expanded', String(!open)); + nav?.classList.toggle('is-open', !open); +}); + +searchButton?.addEventListener('click', () => { + const opening = searchForm?.hasAttribute('hidden'); + searchForm?.toggleAttribute('hidden', !opening); + if (opening) searchForm?.querySelector('input')?.focus(); +}); diff --git a/forma_n8_site/templates/404.html b/forma_n8_site/templates/404.html new file mode 100644 index 0000000..459108c --- /dev/null +++ b/forma_n8_site/templates/404.html @@ -0,0 +1,4 @@ +{% extends "base.html" %} +{% block content %} +

404

Страница не найдена

Вернуться в ленту
+{% endblock %} diff --git a/forma_n8_site/templates/_article_rows.html b/forma_n8_site/templates/_article_rows.html new file mode 100644 index 0000000..a173c47 --- /dev/null +++ b/forma_n8_site/templates/_article_rows.html @@ -0,0 +1,29 @@ +{% macro media(article, eager=false) %} + {% set item = article.media[0] if article.media else none %} + {% if item and item.type == 'image' %} + {{ item.alt or article.title }} + {% elif item and item.type == 'video' %} + + {% else %} + + {% endif %} +{% endmacro %} + +{% macro meta(article) %} +
+ + {{ article.category_name }} + {{ article.producer_name }} +
+{% endmacro %} + +{% macro row(article) %} +
+ {{ media(article) }} +
+ {{ meta(article) }} +

{{ article.title }}

+

{{ article.lead or article.body|excerpt(210) }}

+
+
+{% endmacro %} diff --git a/forma_n8_site/templates/_pagination.html b/forma_n8_site/templates/_pagination.html new file mode 100644 index 0000000..fb581f3 --- /dev/null +++ b/forma_n8_site/templates/_pagination.html @@ -0,0 +1,7 @@ +{% if pages > 1 %} + +{% endif %} diff --git a/forma_n8_site/templates/article.html b/forma_n8_site/templates/article.html new file mode 100644 index 0000000..45ad529 --- /dev/null +++ b/forma_n8_site/templates/article.html @@ -0,0 +1,38 @@ +{% extends "base.html" %} +{% from "_article_rows.html" import media, row %} +{% block og_type %}article{% endblock %} +{% block content %} +
+
+
Лента/{{ article.category_name }}
+

{{ article.title }}

+ {% if article.lead %}

{{ article.lead }}

{% endif %} +
+ + {{ article.producer_name }} +
+
+ {% if article.media %} +
+ {% for item in article.media %} + {% if item.type == 'image' %}
{{ item.alt or article.title }}
{% endif %} + {% if item.type == 'video' %}
{% endif %} + {% endfor %} +
+ {% endif %} +
+
{{ article.body|paragraphs }}
+ +
+
+ {% if related %} +
+

По теме

+
{% for item in related %}{{ row(item) }}{% endfor %}
+
+ {% endif %} +{% endblock %} diff --git a/forma_n8_site/templates/base.html b/forma_n8_site/templates/base.html new file mode 100644 index 0000000..1769bf8 --- /dev/null +++ b/forma_n8_site/templates/base.html @@ -0,0 +1,67 @@ + + + + + + {{ meta_title }} + + {% if noindex %}{% else %}{% endif %} + + + + + + + + + + {% if og_image %}{% endif %} + + {% if json_ld %}{% endif %} + + + +
+
+ + + FORMA
N8 + + +
+ + +
+
+
+ +
+ + +
+
+
+
{% block content %}{% endblock %}
+ + + + diff --git a/forma_n8_site/templates/feed.html b/forma_n8_site/templates/feed.html new file mode 100644 index 0000000..c336afc --- /dev/null +++ b/forma_n8_site/templates/feed.html @@ -0,0 +1,34 @@ +{% extends "base.html" %} +{% from "_article_rows.html" import media, meta, row %} +{% block content %} +
+
+

Новости снаряжения

+ {% if q %}

Результаты поиска: «{{ q }}»

{% endif %} +
+ {% if articles %} + {% if page == 1 and not q %} + {% set lead = articles[0] %} +
+ {{ media(lead, true) }} +
+ {{ meta(lead) }} +

{{ lead.title }}

+

{{ lead.lead or lead.body|excerpt(340) }}

+ Читать новость +
+
+
+ {% for article in articles[1:] %}{{ row(article) }}{% endfor %} +
+ {% else %} +
+ {% for article in articles %}{{ row(article) }}{% endfor %} +
+ {% endif %} + {% include "_pagination.html" %} + {% else %} +

Ничего не найдено

Попробуйте изменить запрос или вернуться к ленте.

Вернуться в ленту
+ {% endif %} +
+{% endblock %} diff --git a/forma_n8_site/templates/taxonomy.html b/forma_n8_site/templates/taxonomy.html new file mode 100644 index 0000000..bdd89f5 --- /dev/null +++ b/forma_n8_site/templates/taxonomy.html @@ -0,0 +1,18 @@ +{% extends "base.html" %} +{% from "_article_rows.html" import row %} +{% block content %} +
+
+

{{ taxonomy_label }}

+

{{ heading }}

+
+ {% if articles %} +
+ {% for article in articles %}{{ row(article) }}{% endfor %} +
+ {% include "_pagination.html" %} + {% else %} +

Публикаций пока нет

Новые материалы появятся здесь после публикации.

Вернуться в ленту
+ {% endif %} +
+{% endblock %} diff --git a/requirements.txt b/requirements.txt index 72a1627..7c83571 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ jinja2==3.1.6 loguru==0.7.3 litellm==1.76.0 pydantic-settings==2.10.1 +Pillow==11.3.0 python-multipart==0.0.20 uvicorn[standard]==0.35.0 yt-dlp==2026.6.9 diff --git a/scripts/backfill_pending_videos.py b/scripts/backfill_pending_videos.py new file mode 100644 index 0000000..5b0813d --- /dev/null +++ b/scripts/backfill_pending_videos.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import asyncio +import html +import json +import os +from pathlib import Path +from typing import Any + +from aiogram.types import FSInputFile + +from vk_parser_app.constants import MEDIA_STATUS_FAILED, MEDIA_STATUS_LINK_ONLY +from vk_parser_app.db import get_pool +from vk_parser_app.workers.vk_storage_uploader import TMP_DIR, TMP_PREFIX, TelegramStorageUploader + + +def jlog(event: str, **payload: Any) -> None: + print(json.dumps({"event": event, **payload}, ensure_ascii=False, default=str), flush=True) + + +async def load_pending_videos(worker: TelegramStorageUploader, limit: int) -> list[dict[str, Any]]: + rows = await worker.pool.fetch( + """ + SELECT m.*, rp.original_url AS post_url + FROM raw_post_media m + JOIN raw_posts rp ON rp.id=m.raw_post_id + WHERE m.media_type='video' + AND m.status='pending' + AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = '' + AND COALESCE(m.error, '') ILIKE '%timeout%' + AND COALESCE(m.duration_sec, 0) <= $1 + ORDER BY COALESCE(rp.publication_status, 'pending')='published', m.id DESC + LIMIT $2 + """, + worker.video_max_duration_sec, + limit, + ) + return [dict(row) for row in rows] + + +async def send_video_to_storage(worker: TelegramStorageUploader, media: dict[str, Any], path: str) -> None: + caption = ( + f"Backfill video media #{int(media['id'])} for raw_post #{int(media['raw_post_id'])}\n" + f'source post' + ) + message = await worker.tg_retry( + lambda: worker.bot.send_video( + video=FSInputFile(path, filename=f"video_{int(media['id'])}.mp4"), + caption=caption[: worker.caption_limit], + parse_mode="HTML", + **worker.chat_kwargs(), + ) + ) + if not message.video: + raise RuntimeError("Telegram accepted message without video payload") + await worker.mark_media_uploaded(int(media["id"]), message.video.file_id, message.video.file_unique_id) + + +async def main() -> None: + limit = int(os.getenv("BACKFILL_LIMIT", "200")) + worker = TelegramStorageUploader() + await worker.init() + videos = await load_pending_videos(worker, limit) + jlog("start", count=len(videos), limit=limit) + ok = 0 + failed = 0 + link_only = 0 + try: + for media in videos: + media_id = int(media["id"]) + raw_post_id = int(media["raw_post_id"]) + path = str(TMP_DIR / f"{TMP_PREFIX}backfill_{raw_post_id}_{media_id}.mp4") + jlog( + "video_start", + media_id=media_id, + raw_post_id=raw_post_id, + url=media.get("original_url"), + duration_sec=media.get("duration_sec"), + ) + try: + info = await worker.download_video(str(media.get("original_url") or ""), path) + if not info: + await worker.mark_media_failed_attempt(media_id, "video download failed") + failed += 1 + jlog("video_failed", media_id=media_id, raw_post_id=raw_post_id, error="video download failed") + continue + if info.get("error"): + if info.get("permanent"): + await worker.mark_media_link_only(media_id, str(info["error"])) + link_only += 1 + else: + await worker.mark_media_failed_attempt(media_id, str(info["error"])) + failed += 1 + jlog("video_failed", media_id=media_id, raw_post_id=raw_post_id, error=info.get("error")) + continue + await send_video_to_storage(worker, media, path) + ok += 1 + jlog("video_uploaded", media_id=media_id, raw_post_id=raw_post_id, size_bytes=info.get("size_bytes")) + await asyncio.sleep(worker.media_upload_delay_sec) + except Exception as exc: + await worker.mark_media_failed_attempt(media_id, str(exc)) + failed += 1 + jlog("video_failed", media_id=media_id, raw_post_id=raw_post_id, error=str(exc)) + finally: + Path(path).unlink(missing_ok=True) + finally: + await worker.close() + if worker.pool: + await worker.pool.close() + jlog("done", ok=ok, failed=failed, link_only=link_only, total=len(videos)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/publish_site_batch.py b/scripts/publish_site_batch.py new file mode 100644 index 0000000..d240f5b --- /dev/null +++ b/scripts/publish_site_batch.py @@ -0,0 +1,524 @@ +from __future__ import annotations + +import argparse +import asyncio +import base64 +import hashlib +import hmac +import html +import io +import json +import os +import re +import subprocess +import tempfile +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +try: + import fcntl +except ImportError: # pragma: no cover - production runs on Linux + fcntl = None + +import aiohttp +from aiogram import Bot +from aiogram.client.session.aiohttp import AiohttpSession +from aiogram.client.telegram import TelegramAPIServer +from PIL import Image, ImageOps + +from vk_parser_app.config import settings +from vk_parser_app.db import close_pool, fetch_setting, get_pool + + +SEPARATOR_RE = re.compile(r"^━+$") +HASHTAG_LINE_RE = re.compile(r"^(?:#[\wа-яё]+\s*)+$", re.IGNORECASE) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Publish selected accepted posts to Ghost") + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--ids", help="Comma-separated raw_post ids") + group.add_argument("--all-ready", action="store_true", help="Publish every photo-ready post not yet published to site") + group.add_argument("--all-ready-videos", action="store_true", help="Publish video-only ready posts") + parser.add_argument("--ghost-url", default="https://f-n8.ru") + parser.add_argument("--key-file", default="/opt/vk-parser/.site-poster-key") + parser.add_argument("--limit", type=int, default=0, help="Maximum posts to publish in this run; 0 means unlimited") + parser.add_argument("--retry-failed", action="store_true", help="Include previous site publish_failed rows") + return parser.parse_args() + + +def ghost_token(key: str) -> str: + key_id, secret = key.strip().split(":", 1) + now = int(time.time()) + + def encode(value: dict[str, Any] | bytes) -> str: + raw = value if isinstance(value, bytes) else json.dumps(value, separators=(",", ":")).encode() + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode() + + header = encode({"alg": "HS256", "typ": "JWT", "kid": key_id}) + payload = encode({"iat": now, "exp": now + 300, "aud": "/admin/"}) + signature = encode(hmac.new(bytes.fromhex(secret), f"{header}.{payload}".encode(), hashlib.sha256).digest()) + return f"{header}.{payload}.{signature}" + + +def split_post_text(value: str) -> tuple[str, str]: + lines = [line.strip() for line in str(value or "").replace("\r", "").split("\n")] + while lines and not lines[0]: + lines.pop(0) + title = lines.pop(0) if lines else "Новость Forma N8" + while lines and (not lines[0] or SEPARATOR_RE.fullmatch(lines[0])): + lines.pop(0) + while lines and (not lines[-1] or HASHTAG_LINE_RE.fullmatch(lines[-1])): + lines.pop() + body = "\n".join(lines).strip() + return title[:255], body + + +def paragraphs(value: str) -> str: + chunks = [chunk.strip() for chunk in re.split(r"\n\s*\n", value) if chunk.strip()] + return "".join(f"

{html.escape(chunk).replace(chr(10), '
')}

" for chunk in chunks) + + +def excerpt(value: str, limit: int = 260) -> str: + clean = re.sub(r"\s+", " ", value).strip() + if len(clean) <= limit: + return clean + return clean[: limit - 1].rsplit(" ", 1)[0] + "…" + + +def compress_photo(data: bytes) -> tuple[bytes, int, int]: + with Image.open(io.BytesIO(data)) as source: + image = ImageOps.exif_transpose(source) + if image.mode not in {"RGB", "RGBA"}: + image = image.convert("RGB") + image.thumbnail((2000, 2000), Image.Resampling.LANCZOS) + output = io.BytesIO() + image.save(output, "WEBP", quality=82, method=6) + return output.getvalue(), image.width, image.height + + +def video_thumbnail(data: bytes) -> tuple[bytes, int, int]: + video_path = "" + image_path = "" + try: + with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as video_file: + video_file.write(data) + video_path = video_file.name + with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as image_file: + image_path = image_file.name + command = [ + "ffmpeg", "-y", "-ss", "1", "-i", video_path, + "-frames:v", "1", "-vf", "scale='min(2000,iw)':-2", "-q:v", "3", image_path, + ] + result = subprocess.run(command, capture_output=True, timeout=90) + if result.returncode != 0 or not Path(image_path).exists() or Path(image_path).stat().st_size == 0: + command[3] = "0" + result = subprocess.run(command, capture_output=True, timeout=90) + if result.returncode != 0: + raise RuntimeError("ffmpeg thumbnail extraction failed") + return compress_photo(Path(image_path).read_bytes()) + finally: + for path in (video_path, image_path): + if path: + try: + os.unlink(path) + except FileNotFoundError: + pass + + +def normalize_video(data: bytes, duration: int, height: int | None) -> bytes: + max_bytes = 40 * 1024 * 1024 + if len(data) <= max_bytes and (not height or height <= 720): + return data + input_path = "" + output_path = "" + try: + with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as input_file: + input_file.write(data) + input_path = input_file.name + with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as output_file: + output_path = output_file.name + safe_duration = max(1, int(duration or 1)) + target_total_kbps = int((38 * 1024 * 8) / safe_duration) + video_kbps = max(280, min(2800, target_total_kbps - 96)) + command = [ + "ffmpeg", "-y", "-i", input_path, + "-map_metadata", "-1", "-c:v", "libx264", "-preset", "fast", + "-b:v", f"{video_kbps}k", "-maxrate", f"{video_kbps}k", "-bufsize", f"{video_kbps * 2}k", + ] + if height and int(height) > 720: + command.extend(["-vf", "scale=-2:720"]) + command.extend(["-c:a", "aac", "-b:a", "96k", "-movflags", "+faststart", output_path]) + result = subprocess.run(command, capture_output=True, timeout=max(180, safe_duration * 3)) + if result.returncode != 0 or not Path(output_path).exists(): + raise RuntimeError("ffmpeg video normalization failed") + normalized = Path(output_path).read_bytes() + if len(normalized) > 48 * 1024 * 1024: + raise RuntimeError(f"normalized video remains too large: {len(normalized)} bytes") + return normalized + finally: + for path in (input_path, output_path): + if path: + try: + os.unlink(path) + except FileNotFoundError: + pass + + +def gallery_html(images: list[dict[str, Any]], title: str) -> str: + if not images: + return "" + if len(images) == 1: + image = images[0] + return ( + '
' + f'' + "
" + ) + rows = [] + for offset in range(0, len(images), 3): + cells = [] + for image in images[offset : offset + 3]: + cells.append( + '
' + f'' + "
" + ) + rows.append('
' + "".join(cells) + "
") + return '
' + "".join(rows) + "
" + + +def videos_html(videos: list[dict[str, Any]], title: str, poster_url: str) -> str: + return "".join( + '
' + f'' + "
" + for video in videos + ) + + +def lexical_paragraph(text: str) -> dict[str, Any]: + return { + "children": [{"detail": 0, "format": 0, "mode": "normal", "style": "", "text": text, "type": "extended-text", "version": 1}], + "direction": None, + "format": "", + "indent": 0, + "type": "paragraph", + "version": 1, + } + + +def video_lexical(videos: list[dict[str, Any]], body: str, thumbnail: dict[str, Any]) -> str: + children: list[dict[str, Any]] = [] + for video in videos: + children.append( + { + "type": "video", + "version": 1, + "src": video["url"], + "caption": "", + "fileName": video["file_name"], + "mimeType": "video/mp4", + "width": video.get("width"), + "height": video.get("height"), + "duration": video.get("duration") or 0, + "thumbnailSrc": thumbnail["url"], + "customThumbnailSrc": "", + "thumbnailWidth": thumbnail["width"], + "thumbnailHeight": thumbnail["height"], + "cardWidth": "wide", + "loop": False, + } + ) + children.extend(lexical_paragraph(chunk.strip()) for chunk in re.split(r"\n\s*\n", body) if chunk.strip()) + return json.dumps( + {"root": {"children": children, "direction": None, "format": "", "indent": 0, "type": "root", "version": 1}}, + ensure_ascii=False, + separators=(",", ":"), + ) + + +class GhostClient: + def __init__(self, session: aiohttp.ClientSession, base_url: str, key: str) -> None: + self.session = session + self.base_url = base_url.rstrip("/") + self.key = key + + def headers(self) -> dict[str, str]: + return {"Authorization": f"Ghost {ghost_token(self.key)}", "Accept-Version": "v6.0"} + + async def upload_image(self, data: bytes, name: str, ref: str) -> str: + form = aiohttp.FormData() + form.add_field("file", data, filename=name, content_type="image/webp") + form.add_field("purpose", "image") + form.add_field("ref", ref) + async with self.session.post( + f"{self.base_url}/ghost/api/admin/images/upload/", data=form, headers=self.headers() + ) as response: + payload = await response.json(content_type=None) + if response.status >= 300: + raise RuntimeError(f"Ghost image upload {response.status}: {payload}") + return str(payload["images"][0]["url"]) + + async def upload_media(self, data: bytes, name: str, content_type: str) -> str: + form = aiohttp.FormData() + form.add_field("file", data, filename=name, content_type=content_type) + async with self.session.post( + f"{self.base_url}/ghost/api/admin/media/upload/", data=form, headers=self.headers() + ) as response: + payload = await response.json(content_type=None) + if response.status >= 300: + raise RuntimeError(f"Ghost media upload {response.status}: {payload}") + return str(payload["media"][0]["url"]) + + async def create_post(self, post: dict[str, Any]) -> dict[str, Any]: + suffix = "/" if "lexical" in post else "/?source=html" + async with self.session.post( + f"{self.base_url}/ghost/api/admin/posts{suffix}", + json={"posts": [post]}, + headers=self.headers(), + ) as response: + payload = await response.json(content_type=None) + if response.status >= 300: + raise RuntimeError(f"Ghost post create {response.status}: {payload}") + return dict(payload["posts"][0]) + + +async def main() -> None: + args = parse_args() + key = Path(args.key_file).read_text(encoding="utf-8").strip() + pool = await get_pool() + if args.all_ready or args.all_ready_videos: + media_condition = ( + """EXISTS ( + SELECT 1 FROM raw_post_media rpm + WHERE rpm.raw_post_id=rp.id AND rpm.media_type='video' + AND COALESCE(rpm.editor_hidden,FALSE)=FALSE + AND COALESCE(rpm.tg_file_id,rpm.storage_attachment_id,'')<>'' + ) AND NOT EXISTS ( + SELECT 1 FROM raw_post_media rpm + WHERE rpm.raw_post_id=rp.id AND rpm.media_type='photo' + AND COALESCE(rpm.editor_hidden,FALSE)=FALSE + AND COALESCE(rpm.tg_file_id,rpm.storage_attachment_id,'')<>'' + )""" + if args.all_ready_videos + else """EXISTS ( + SELECT 1 FROM raw_post_media rpm + WHERE rpm.raw_post_id=rp.id AND rpm.media_type='photo' + AND COALESCE(rpm.editor_hidden,FALSE)=FALSE + AND COALESCE(rpm.tg_file_id,rpm.storage_attachment_id,'')<>'' + )""" + ) + candidates = [ + dict(row) + for row in await pool.fetch( + f""" + SELECT rp.id, + COALESCE(rp.final_category_tag,rp.rewrite_category_tag,'') AS category, + COALESCE(rp.final_source_tag,rp.rewrite_source_tag,s.tag,'') AS source + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + JOIN content_categories cc ON cc.sort_order=COALESCE(rp.final_category_id,rp.rewrite_category_id) + LEFT JOIN post_publications pp ON pp.raw_post_id=rp.id AND pp.platform='site' + WHERE rp.status='storage_ready' + AND rp.rewrite_status='ready' + AND COALESCE(rp.editorial_status,'review') IN ('accepted','published','publish_failed') + AND cc.site_enabled=TRUE + AND COALESCE(pp.status,'pending') {"<> 'published'" if args.retry_failed else "= 'pending'"} + AND {media_condition} + ORDER BY COALESCE(rp.reviewed_at,rp.edited_at,rp.rewritten_at,rp.created_at),rp.id + """ + ) + ] + recent = [ + dict(row) + for row in await pool.fetch( + """ + SELECT COALESCE(rp.final_category_tag,rp.rewrite_category_tag,'') AS category, + COALESCE(rp.final_source_tag,rp.rewrite_source_tag,s.tag,'') AS source + FROM post_publications pp + JOIN raw_posts rp ON rp.id=pp.raw_post_id + JOIN sources s ON s.id=rp.source_id + WHERE pp.platform='site' AND pp.status='published' + ORDER BY pp.published_at DESC NULLS LAST,pp.id DESC LIMIT 20 + """ + ) + ] + selected: list[dict[str, Any]] = [] + while candidates: + previous = selected[-1] if selected else (recent[0] if recent else {}) + recent_window = recent + selected[-20:] + best = min( + range(len(candidates)), + key=lambda index: ( + int(bool(candidates[index]["category"]) and candidates[index]["category"] == previous.get("category")) + + int(bool(candidates[index]["source"]) and candidates[index]["source"] == previous.get("source")), + sum(3 for row in recent_window if row.get("category") == candidates[index]["category"]) + + sum(4 for row in recent_window if row.get("source") == candidates[index]["source"]), + index, + ), + ) + selected.append(candidates.pop(best)) + ids = [int(row["id"]) for row in selected] + if args.limit > 0: + ids = ids[: args.limit] + else: + ids = [int(item) for item in str(args.ids or "").split(",") if item.strip()] + if not ids: + print("NO_READY_POSTS") + await close_pool() + return + print(f"QUEUE {len(ids)}") + rows = await pool.fetch( + """ + SELECT rp.*, s.name AS source_name, s.tag AS source_tag, + cc.site_name, cc.site_slug, cc.site_enabled, + pp.status AS site_publication_status + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + JOIN content_categories cc ON cc.sort_order=COALESCE(rp.final_category_id, rp.rewrite_category_id) + LEFT JOIN post_publications pp ON pp.raw_post_id=rp.id AND pp.platform='site' + WHERE rp.id=ANY($1::bigint[]) + """, + ids, + ) + by_id = {int(row["id"]): dict(row) for row in rows} + local_api = str(await fetch_setting("local_bot_api_url", settings.local_bot_api_url) or "").strip() + bot_session = AiohttpSession(api=TelegramAPIServer.from_base(local_api.rstrip("/"), is_local=True)) if local_api else AiohttpSession() + bot = Bot(settings.tg_bot_token, session=bot_session) + timeout = aiohttp.ClientTimeout(total=120) + published_base = datetime.now(timezone.utc) - timedelta(minutes=len(ids)) + try: + async with aiohttp.ClientSession(timeout=timeout) as http_session: + ghost = GhostClient(http_session, args.ghost_url, key) + for position, raw_post_id in enumerate(ids): + post = by_id.get(raw_post_id) + if not post: + print(f"SKIP {raw_post_id}: missing or unmapped category") + continue + if not post.get("site_enabled"): + print(f"SKIP {raw_post_id}: category disabled for site") + continue + if post.get("site_publication_status") == "published": + print(f"SKIP {raw_post_id}: already published") + continue + try: + media_rows = await pool.fetch( + """ + SELECT * FROM raw_post_media + WHERE raw_post_id=$1 AND media_type IN ('photo','video') + AND COALESCE(editor_hidden,FALSE)=FALSE + AND COALESCE(tg_file_id,storage_attachment_id,'')<>'' + ORDER BY sort_order,id + """, + raw_post_id, + ) + uploaded_photos = [] + uploaded_videos = [] + thumbnail = None + for index, media in enumerate(media_rows): + file_id = str(media["tg_file_id"] or media["storage_attachment_id"]) + telegram_file = await bot.get_file(file_id) + buffer = io.BytesIO() + await bot.download_file(telegram_file.file_path, destination=buffer) + data = buffer.getvalue() + if media["media_type"] == "photo": + compressed, width, height = compress_photo(data) + url = await ghost.upload_image(compressed, f"post-{raw_post_id}-{index + 1}.webp", f"raw-post-{raw_post_id}-{index + 1}") + uploaded_photos.append({"type": "photo", "url": url, "width": width, "height": height, "bytes": len(compressed)}) + else: + data = normalize_video(data, int(media.get("duration_sec") or 0), media.get("height")) + url = await ghost.upload_media(data, f"post-{raw_post_id}-{index + 1}.mp4", "video/mp4") + uploaded_videos.append({ + "type": "video", "url": url, "bytes": len(data), + "file_name": f"post-{raw_post_id}-{index + 1}.mp4", + "width": media.get("width"), "height": media.get("height"), + "duration": media.get("duration_sec") or 0, + }) + if thumbnail is None: + compressed, width, height = video_thumbnail(data) + thumb_url = await ghost.upload_image(compressed, f"post-{raw_post_id}-video-cover.webp", f"raw-post-{raw_post_id}-video-cover") + thumbnail = {"type": "thumbnail", "url": thumb_url, "width": width, "height": height, "bytes": len(compressed)} + feature = uploaded_photos[0] if uploaded_photos else thumbnail + if not feature: + raise RuntimeError("no downloadable site media") + title, body = split_post_text(post.get("final_text") or post.get("rewritten_text") or "") + producer = str(post.get("source_name") or "").strip() + content = gallery_html(uploaded_photos[1:], title) + videos_html(uploaded_videos, title, feature["url"]) + paragraphs(body) + attachments = uploaded_photos + uploaded_videos + ([thumbnail] if thumbnail else []) + payload = { + "title": title, + "status": "published", + "published_at": (published_base + timedelta(minutes=position)).isoformat(), + "custom_excerpt": excerpt(body), + "meta_description": excerpt(body), + "feature_image": feature["url"], + "feature_image_alt": title, + "tags": [ + {"name": str(post["site_name"]), "slug": str(post["site_slug"])}, + {"name": f"#producer:{producer}"}, + {"name": f"#raw-post:{raw_post_id}"}, + ], + } + if uploaded_videos and not uploaded_photos: + payload["lexical"] = video_lexical(uploaded_videos, body, thumbnail) + else: + payload["html"] = content + created = await ghost.create_post(payload) + await pool.execute( + """ + INSERT INTO post_publications(raw_post_id,platform,status,target_id,external_id,url,error,attachments_json,published_at) + VALUES($1,'site','published',$2,$3,$4,NULL,$5::jsonb,NOW()) + ON CONFLICT(raw_post_id,platform) DO UPDATE SET + status='published',target_id=EXCLUDED.target_id,external_id=EXCLUDED.external_id, + url=EXCLUDED.url,error=NULL,attachments_json=EXCLUDED.attachments_json, + published_at=NOW(),updated_at=NOW() + """, + raw_post_id, + args.ghost_url, + str(created["id"]), + str(created["url"]), + json.dumps(attachments, ensure_ascii=False), + ) + print(f"PUBLISHED {raw_post_id} {created['url']} media={len(attachments)} bytes={sum(x['bytes'] for x in attachments)}") + except Exception as exc: + await pool.execute( + """ + INSERT INTO post_publications(raw_post_id,platform,status,target_id,error,updated_at) + VALUES($1,'site','publish_failed',$2,$3,NOW()) + ON CONFLICT(raw_post_id,platform) DO UPDATE SET + status='publish_failed',target_id=EXCLUDED.target_id,error=EXCLUDED.error,updated_at=NOW() + """, + raw_post_id, + args.ghost_url, + str(exc)[:2000], + ) + print(f"FAILED {raw_post_id}: {exc}") + finally: + await bot.session.close() + await close_pool() + + +def run_locked() -> None: + if fcntl is None: + asyncio.run(main()) + return + with Path("/tmp/forma-n8-site-poster.lock").open("w") as lock_file: + try: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + print("ALREADY_RUNNING") + return + asyncio.run(main()) + + +if __name__ == "__main__": + run_locked() diff --git a/scripts/update_ghost_about.mjs b/scripts/update_ghost_about.mjs new file mode 100644 index 0000000..9ac96ae --- /dev/null +++ b/scripts/update_ghost_about.mjs @@ -0,0 +1,35 @@ +import {createHmac} from "node:crypto"; +import {readFileSync} from "node:fs"; + +const baseUrl = "https://f-n8.ru"; +const [keyId, secret] = readFileSync("/root/ghost-admin-api-key", "utf8").trim().split(":"); +const encode = (value) => Buffer.from(JSON.stringify(value)).toString("base64url"); +const now = Math.floor(Date.now() / 1000); +const unsigned = `${encode({alg: "HS256", typ: "JWT", kid: keyId})}.${encode({iat: now, exp: now + 300, aud: "/admin/"})}`; +const token = `${unsigned}.${createHmac("sha256", Buffer.from(secret, "hex")).update(unsigned).digest("base64url")}`; +const headers = {Authorization: `Ghost ${token}`, "Content-Type": "application/json"}; + +const oldText = "Мы собираем материалы производителей и профильных магазинов, проверяем их релевантность и приводим к единому удобному формату."; +const newText = "Ежедневно мы собираем новости снаряжения с сотен источников, отбираем лучшее и публикуем в одном месте!"; + +const response = await fetch(`${baseUrl}/ghost/api/admin/pages/slug/about/?formats=html`, {headers}); +if (!response.ok) throw new Error(`Ghost page read failed: ${response.status} ${await response.text()}`); +const page = (await response.json()).pages[0]; +const html = String(page.html || ""); +if (!html.includes(oldText) && !String(page.custom_excerpt || "").includes(oldText)) { + throw new Error("Expected about text was not found"); +} +const update = { + id: page.id, + updated_at: page.updated_at, + html: html.replaceAll(oldText, newText), + custom_excerpt: String(page.custom_excerpt || "").replaceAll(oldText, newText), +}; +const saved = await fetch(`${baseUrl}/ghost/api/admin/pages/${page.id}/?source=html`, { + method: "PUT", + headers, + body: JSON.stringify({pages: [update]}), +}); +if (!saved.ok) throw new Error(`Ghost page update failed: ${saved.status} ${await saved.text()}`); +const result = (await saved.json()).pages[0]; +console.log(JSON.stringify({id: result.id, slug: result.slug, updated_at: result.updated_at})); diff --git a/src/vk_parser_app/admin.py b/src/vk_parser_app/admin.py index 771030d..c5d0d36 100644 --- a/src/vk_parser_app/admin.py +++ b/src/vk_parser_app/admin.py @@ -16,16 +16,16 @@ from zoneinfo import ZoneInfo import aiohttp from fastapi import FastAPI, File, Form, Request, UploadFile, status -from fastapi.responses import HTMLResponse, RedirectResponse +from fastapi.responses import FileResponse, HTMLResponse, RedirectResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates from loguru import logger from .config import settings from .constants import PLATFORM_VK -from .db import fetch_setting, get_pool +from .db import fetch_int_setting, fetch_setting, get_pool from .security import hash_password, new_token, token_hash, verify_password -from .text_utils import normalize_hash_tag, parse_categories +from .text_utils import build_publication_text, normalize_hash_tag, parse_categories from .vk_api import VKAPIClient, normalize_vk_source from .workers.ai_qualifier import normalize_model, response_usage @@ -33,6 +33,8 @@ COOKIE_NAME = "vk_parser_admin" VK_OAUTH_VERIFIER_COOKIE = "vk_oauth_verifier" VK_OAUTH_STATE_COOKIE = "vk_oauth_state" BASE_DIR = Path(__file__).resolve().parent +PROJECT_ROOT = BASE_DIR.parents[1] +FAVICON_PATH = PROJECT_ROOT / "favi.png" templates = Jinja2Templates(directory=str(BASE_DIR / "templates")) app = FastAPI(title="VK Parser Admin") LOCAL_TZ = ZoneInfo("Asia/Yekaterinburg") @@ -42,6 +44,16 @@ EDITOR_MEDIA_DIR = UPLOAD_ROOT / "editor_media" EDITOR_MEDIA_DIR.mkdir(parents=True, exist_ok=True) app.mount("/uploads", StaticFiles(directory=str(UPLOAD_ROOT)), name="uploads") + +@app.get("/favi.png") +async def favicon_png(): + return FileResponse(FAVICON_PATH) + + +@app.get("/favicon.ico") +async def favicon_ico(): + return FileResponse(FAVICON_PATH, media_type="image/png") + PROVIDER_OPTIONS = [ {"value": "openrouter", "label": "OpenRouter"}, {"value": "openai", "label": "OpenAI"}, @@ -92,7 +104,7 @@ PROMPT_HINTS = { ), "safe": "Обычно не трогаем. Если промпт райтера переписывается, свободную часть меняем выше, контракт оставляем стабильным.", "input": "Контракт склеивается после свободной инструкции и отправляется как system prompt.", - "contract": '{"rewrites":[{"id":123,"category_id":2,"text":"готовый текст без хэштегов","notes":"короткая заметка для редактора"}]}', + "contract": '{"rewrites":[{"id":123,"category_id":2,"text":"готовый текст до 900 символов без хэштегов; явно упоминает producer_name","notes":"короткая заметка для редактора"}]}', }, } @@ -130,6 +142,11 @@ PREFERRED_MODELS = { CATEGORY_TITLES = { "AI Qualifier": "AI-квалификатор", "AI Writer": "AI-райтер", + "TG Poster": "TG-постер", + "TG Reactor": "TG-реактор", + "VK Poster": "VK-постер", + "MAX Poster": "MAX-постер", + "Daily Report": "Ежедневный отчет", "Parser": "Парсер", "Uploader": "Аплоадер", "VK": "VK API", @@ -139,15 +156,31 @@ CATEGORY_TITLES = { CATEGORY_ORDER = { "AI Qualifier": 10, "AI Writer": 20, - "Parser": 30, - "VK": 40, - "Uploader": 50, + "TG Poster": 30, + "TG Reactor": 40, + "VK Poster": 50, + "MAX Poster": 51, + "Site Poster": 52, + "Daily Report": 55, + "Parser": 60, + "VK": 70, + "Uploader": 80, "General": 100, } +AI_WORKER_ENABLED_SETTINGS = { + "ai-qualifier": "ai_qualifier_enabled", + "ai-writer": "ai_writer_enabled", + "tg-poster": "tg_poster_enabled", + "tg-reactor": "tg_reactor_enabled", + "vk-poster": "vk_poster_enabled", + "max-poster": "max_poster_enabled", + "site-poster": "site_poster_enabled", + "daily-report": "daily_report_enabled", +} + SETTING_ORDER = { "AI Qualifier": [ - "ai_qualifier_enabled", "ai_qualifier_provider", "ai_qualifier_model", "ai_qualifier_api_key", @@ -162,7 +195,6 @@ SETTING_ORDER = { "ai_qualifier_interval_sec", ], "AI Writer": [ - "ai_writer_enabled", "ai_writer_provider", "ai_writer_model", "ai_writer_api_key", @@ -176,6 +208,86 @@ SETTING_ORDER = { "ai_writer_timeout_sec", "ai_writer_interval_sec", ], + "TG Poster": [ + "tg_poster_bot_token", + "tg_poster_chat_id", + "tg_poster_interval_sec", + "tg_poster_media_group_max_items", + "tg_poster_caption_limit", + "tg_poster_message_limit", + "tg_poster_max_attempts", + "tg_poster_retry_backoff_max_sec", + "tg_poster_send_delay_sec", + "tg_poster_text_overflow_caption", + "tg_poster_recent_window", + "tg_poster_category_repeat_penalty", + "tg_poster_source_repeat_penalty", + ], + "TG Reactor": [ + "tg_reactor_bot_tokens", + "tg_reactor_reactions_json", + "tg_reactor_since", + "tg_reactor_delay_sec", + "tg_reactor_interval_sec", + "tg_reactor_reaction_pause_sec", + "tg_reactor_react_all_messages", + "tg_reactor_max_attempts", + "tg_reactor_retry_backoff_max_sec", + ], + "VK Poster": [ + "vk_poster_app_id", + "vk_poster_client_secret", + "vk_poster_access_token", + "vk_poster_refresh_token", + "vk_poster_token_device_id", + "vk_poster_token_expires_at", + "vk_poster_owner_id", + "vk_poster_from_group", + "vk_poster_interval_sec", + "vk_poster_timeout_sec", + "vk_poster_max_attempts", + "vk_poster_retry_backoff_max_sec", + "vk_poster_send_delay_sec", + "vk_poster_recent_window", + "vk_poster_category_repeat_penalty", + "vk_poster_source_repeat_penalty", + "vk_poster_dry_run", + ], + "MAX Poster": [ + "max_poster_bot_token", + "max_poster_chat_id", + "max_poster_api_base_url", + "max_poster_interval_sec", + "max_poster_message_limit", + "max_poster_media_max_items", + "max_poster_media_process_delay_sec", + "max_poster_video_ready_attempts", + "max_poster_video_ready_delay_sec", + "max_poster_max_attempts", + "max_poster_retry_backoff_max_sec", + "max_poster_send_delay_sec", + "max_poster_recent_window", + "max_poster_category_repeat_penalty", + "max_poster_source_repeat_penalty", + "max_poster_auto_reaction_enabled", + "max_poster_auto_reaction", + "max_poster_reaction_path_template", + "max_poster_dry_run", + ], + "Site Poster": [ + "site_poster_interval_sec", + "site_poster_photo_batch_size", + "site_poster_video_batch_size", + "site_poster_ghost_url", + "site_poster_key_file", + ], + "Daily Report": [ + "daily_report_bot_token", + "daily_report_recipient_ids", + "daily_report_time", + "daily_report_interval_sec", + "daily_report_last_sent_date", + ], "Parser": [ "parser_interval_sec", "parser_new_source_lookback_days", @@ -211,6 +323,7 @@ SETTING_ORDER = { "tg_retry_attempts", "tg_retry_backoff_max_sec", "video_max_size_mb", + "video_max_height", "video_max_duration_sec", "uploader_yt_dlp_timeout_sec", ], @@ -235,6 +348,36 @@ async def add_security_headers(request: Request, call_next): return response +def should_audit_request(request: Request) -> bool: + path = request.url.path + if path in {"/health", "/favi.png", "/favicon.ico"}: + return False + if path.startswith("/uploads/"): + return False + if path == "/login" and request.method == "GET": + return False + return request.method in {"GET", "POST"} + + +@app.middleware("http") +async def add_request_audit(request: Request, call_next): + user = await get_current_user(request) if should_audit_request(request) else None + response = await call_next(request) + if user: + try: + await audit( + user["id"], + f"request.{request.method.lower()}", + "http_request", + None, + {"status_code": response.status_code}, + request, + ) + except Exception as exc: + logger.warning("Could not write request audit log: {}", exc) + return response + + def pkce_challenge(verifier: str) -> str: digest = hashlib.sha256(verifier.encode("ascii")).digest() return base64.urlsafe_b64encode(digest).decode("ascii").rstrip("=") @@ -457,6 +600,8 @@ EDITOR_STATUS_OPTIONS = [ {"value": "review", "label": "На проверке"}, {"value": "edited", "label": "Отредактировано"}, {"value": "accepted", "label": "Принято"}, + {"value": "published", "label": "Опубликовано"}, + {"value": "publish_failed", "label": "Ошибка публикации"}, {"value": "rejected", "label": "Отклонено"}, ] @@ -842,10 +987,58 @@ def prepare_raw_post(row: Any) -> dict[str, Any]: def prepare_editor_post(row: Any) -> dict[str, Any]: post = prepare_raw_post(row) + category_name = post.get("final_category") or post.get("rewrite_category") or "" + category_tag = post.get("resolved_category_tag") or post.get("final_category_tag") or post.get("rewrite_category_tag") or category_name + source_tag = post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "" post["review_text"] = post.get("final_text") or post.get("rewritten_text") or "" - post["review_category"] = post.get("final_category") or post.get("rewrite_category") or "" - post["review_category_tag"] = post.get("final_category_tag") or post.get("rewrite_category_tag") or "" - post["review_source_tag"] = post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "" + post["publication_preview_text"] = build_publication_text(post["review_text"], category_tag, source_tag) + post["review_category"] = category_name + post["review_category_tag"] = category_tag + post["review_source_tag"] = source_tag + publication_status = post.get("publication_status") or "pending" + if publication_status == "published": + post["publication_status_label"] = "опубликовано" + post["publication_status_class"] = "ok" + elif publication_status == "publish_failed": + post["publication_status_label"] = "ошибка публикации" + post["publication_status_class"] = "bad" + else: + post["publication_status_label"] = "не опубликовано" + post["publication_status_class"] = "" + platform_rows = post.get("platform_publications") or [] + if isinstance(platform_rows, str): + try: + platform_rows = json.loads(platform_rows) + except json.JSONDecodeError: + platform_rows = [] + platforms: dict[str, dict[str, Any]] = {} + for item in platform_rows or []: + if not isinstance(item, dict) or not item.get("platform"): + continue + platforms[str(item["platform"])] = item + if "tg" not in platforms and post.get("publication_status") and post.get("publication_status") != "pending": + platforms["tg"] = { + "platform": "tg", + "status": post.get("publication_status"), + "url": post.get("tg_publication_url"), + "error": post.get("publication_error"), + } + labels = {"tg": "TG", "vk": "VK", "max": "MAX", "site": "Сайт"} + status_labels = { + "published": "опубликовано", + "publish_failed": "ошибка", + "pending": "ожидает", + } + status_classes = {"published": "ok", "publish_failed": "bad", "pending": ""} + post["publication_platforms"] = [ + { + **platforms.get(platform, {"platform": platform, "status": "pending"}), + "label": label, + "status_label": status_labels.get(str(platforms.get(platform, {}).get("status") or "pending"), str(platforms.get(platform, {}).get("status") or "pending")), + "class": status_classes.get(str(platforms.get(platform, {}).get("status") or "pending"), ""), + } + for platform, label in labels.items() + ] post["editorial_status_label"] = next( (item["label"] for item in EDITOR_STATUS_OPTIONS if item["value"] == (post.get("editorial_status") or "review")), post.get("editorial_status") or "review", @@ -855,6 +1048,37 @@ def prepare_editor_post(row: Any) -> dict[str, Any]: return post +async def resolve_category_for_editor(name: str, category_id: int | None = None) -> dict[str, Any]: + pool = await get_pool() + row = None + if category_id is not None: + row = await pool.fetchrow( + """ + SELECT sort_order AS id, name, tag + FROM content_categories + WHERE sort_order=$1 + LIMIT 1 + """, + category_id, + ) + if row is None: + raw_name = (name or "").strip().strip("#") + row = await pool.fetchrow( + """ + SELECT sort_order AS id, name, tag + FROM content_categories + WHERE lower(name)=lower($1) OR lower(tag)=lower($1) + ORDER BY is_active DESC, sort_order, name + LIMIT 1 + """, + raw_name, + ) + if row: + return {"id": int(row["id"]), "name": str(row["name"]), "tag": normalize_hash_tag(str(row["tag"]), "category")} + fallback_name = (name or "").strip().strip("#") + return {"id": category_id, "name": fallback_name, "tag": normalize_hash_tag(fallback_name, "category")} + + def uploaded_editor_media_path(url: str) -> Path | None: raw = str(url or "").strip() prefix = "/uploads/editor_media/" @@ -898,6 +1122,109 @@ async def writer_categories() -> list[str]: ] +async def writer_category_options() -> list[dict[str, str]]: + pool = await get_pool() + rows = await pool.fetch( + """ + SELECT name, tag + FROM content_categories + WHERE is_active=TRUE + ORDER BY sort_order, name + """ + ) + options = [ + {"name": str(row["name"]), "tag": normalize_hash_tag(str(row["tag"]), "category")} + for row in rows + ] + if options: + return options + return [{"name": name, "tag": normalize_hash_tag(name, "category")} for name in await writer_categories()] + + +def normalize_tg_schedule(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list): + value = [] + rows: list[dict[str, Any]] = [] + for idx, item in enumerate(value): + if not isinstance(item, dict): + continue + row_id = str(item.get("id") or f"slot-{idx + 1}").strip()[:80] + time_value = str(item.get("time") or "").strip() + count = item.get("count") or 1 + try: + count_int = clamp_int(int(count), 1, 20) + except Exception: + count_int = 1 + if not re.fullmatch(r"(?:[01]\d|2[0-3]):[0-5]\d", time_value): + time_value = "10:00" + rows.append({"id": row_id, "time": time_value, "count": count_int, "enabled": bool(item.get("enabled", True))}) + return sorted(rows, key=lambda row: row["time"]) + + +async def tg_poster_schedule_rows() -> list[dict[str, Any]]: + value = await fetch_setting("tg_poster_schedule_json", []) + rows = normalize_tg_schedule(value) + if rows: + return rows + return [ + {"id": "morning", "time": "10:00", "count": 1, "enabled": True}, + {"id": "evening", "time": "18:00", "count": 1, "enabled": True}, + ] + + +async def vk_poster_schedule_rows() -> list[dict[str, Any]]: + value = await fetch_setting("vk_poster_schedule_json", []) + rows = normalize_tg_schedule(value) + if rows: + return rows + return [ + {"id": "vk-morning", "time": "10:05", "count": 1, "enabled": True}, + {"id": "vk-evening", "time": "18:05", "count": 1, "enabled": True}, + ] + + +async def save_tg_poster_schedule(rows: list[dict[str, Any]]) -> None: + pool = await get_pool() + await pool.execute( + """ + INSERT INTO app_settings(key, value_json, value_type, title, description, category) + VALUES( + 'tg_poster_schedule_json', + $1::jsonb, + 'json', + 'TG publication schedule', + 'EKB time schedule rows: id, time HH:MM, count, enabled.', + 'TG Poster' + ) + ON CONFLICT (key) DO UPDATE + SET value_json=EXCLUDED.value_json, + updated_at=NOW() + """, + json.dumps(normalize_tg_schedule(rows), ensure_ascii=False), + ) + + +async def save_vk_poster_schedule(rows: list[dict[str, Any]]) -> None: + pool = await get_pool() + await pool.execute( + """ + INSERT INTO app_settings(key, value_json, value_type, title, description, category) + VALUES( + 'vk_poster_schedule_json', + $1::jsonb, + 'json', + 'VK publication schedule', + 'EKB time schedule rows: id, time HH:MM, count, enabled.', + 'VK Poster' + ) + ON CONFLICT (key) DO UPDATE + SET value_json=EXCLUDED.value_json, + updated_at=NOW() + """, + json.dumps(normalize_tg_schedule(rows), ensure_ascii=False), + ) + + async def writer_category_payload() -> list[dict[str, Any]]: pool = await get_pool() rows = await pool.fetch( @@ -921,7 +1248,8 @@ async def category_rows() -> list[dict[str, Any]]: pool = await get_pool() rows = await pool.fetch( """ - SELECT id, name, tag, is_active, sort_order, created_at, updated_at + SELECT id, name, tag, site_name, site_slug, site_enabled, + is_active, sort_order, created_at, updated_at FROM content_categories ORDER BY is_active DESC, sort_order, name """ @@ -1258,8 +1586,36 @@ async def model_options_for_provider(provider: str, api_key: str = "") -> list[d return list(models) -async def audit(actor_id: int | None, action: str, entity_type: str, entity_id: int | None, after: Any = None) -> None: +async def prune_audit_log() -> None: pool = await get_pool() + await pool.execute("DELETE FROM audit_log WHERE created_at < NOW() - INTERVAL '30 days'") + + +def request_audit_meta(request: Request | None, extra: dict[str, Any] | None = None) -> dict[str, Any]: + meta = dict(extra or {}) + if request: + meta.update( + { + "ip": client_ip(request), + "method": request.method, + "path": request.url.path, + "query": str(request.url.query), + "user_agent": (request.headers.get("user-agent") or "")[:300], + } + ) + return meta + + +async def audit( + actor_id: int | None, + action: str, + entity_type: str, + entity_id: int | None, + after: Any = None, + request: Request | None = None, +) -> None: + pool = await get_pool() + payload = request_audit_meta(request, after if isinstance(after, dict) else {"value": after} if after is not None else None) await pool.execute( """ INSERT INTO audit_log(actor_id, action, entity_type, entity_id, after_json) @@ -1269,13 +1625,28 @@ async def audit(actor_id: int | None, action: str, entity_type: str, entity_id: action, entity_type, entity_id, - json.dumps(after, ensure_ascii=False) if after is not None else None, + json.dumps(payload, ensure_ascii=False) if payload else None, ) +def prepare_audit_row(row: Any) -> dict[str, Any]: + item = dict(row) + payload = item.get("after_json") + if isinstance(payload, str): + try: + payload = json.loads(payload) + except json.JSONDecodeError: + pass + item["created_at_fmt"] = format_dt(item.get("created_at")) + item["actor_label"] = item.get("actor_login") or (f"user #{item['actor_id']}" if item.get("actor_id") else "system") + item["details_pretty"] = json.dumps(payload, ensure_ascii=False, indent=2, default=str) if payload is not None else "" + return item + + @app.on_event("startup") async def startup() -> None: await bootstrap_admin() + await prune_audit_log() @app.get("/health") @@ -1339,6 +1710,162 @@ async def vk_group_oauth_start() -> RedirectResponse: return RedirectResponse(f"https://oauth.vk.com/authorize?{urlencode(params)}") +@app.get("/vk-oauth/start") +async def vk_poster_oauth_start(request: Request) -> RedirectResponse: + user = await get_current_user(request) + if not user: + return redirect("/login") + client_id = str(await fetch_setting("vk_poster_app_id", "") or "").strip() + if not client_id: + return redirect("/workers") + verifier = secrets.token_urlsafe(64) + state = secrets.token_urlsafe(24) + redirect_uri = "https://sw.exostring.xyz/vk-oauth/callback" + params = { + "client_id": client_id, + "redirect_uri": redirect_uri, + "response_type": "code", + "scope": "wall photos", + "state": state, + "code_challenge": pkce_challenge(verifier), + "code_challenge_method": "s256", + "origin": "https://sw.exostring.xyz", + "v": "5.199", + } + response = RedirectResponse(f"https://id.vk.ru/authorize?{urlencode(params)}") + response.set_cookie(VK_OAUTH_VERIFIER_COOKIE, verifier, httponly=True, secure=True, samesite="lax", max_age=15 * 60) + response.set_cookie(VK_OAUTH_STATE_COOKIE, state, httponly=True, secure=True, samesite="lax", max_age=15 * 60) + return response + + +@app.get("/vk-oauth/callback", response_class=HTMLResponse) +async def vk_poster_oauth_callback( + request: Request, + code: str = "", + state: str = "", + error: str = "", + error_description: str = "", +): + user = await get_current_user(request) + result: dict[str, Any] = { + "ok": False, + "error": error, + "error_description": error_description, + "state_ok": bool(state and state == request.cookies.get(VK_OAUTH_STATE_COOKIE, "")), + } + if not user: + result.update({"error": "auth_required", "error_description": "Сначала войдите в админку, затем повторите VK OAuth."}) + elif error: + pass + elif not code: + result.update({"error": "missing_code", "error_description": "VK callback пришёл без code."}) + elif not result["state_ok"]: + result.update({"error": "state_mismatch", "error_description": "OAuth state не совпал. Повторите авторизацию из админки."}) + else: + pool = await get_pool() + client_id = str(await fetch_setting("vk_poster_app_id", "") or "").strip() + client_secret = str(await fetch_setting("vk_poster_client_secret", "") or "").strip() + owner_id = int(await fetch_int_setting("vk_poster_owner_id", 0)) + redirect_uri = "https://sw.exostring.xyz/vk-oauth/callback" + code_verifier = request.cookies.get(VK_OAUTH_VERIFIER_COOKIE, "") + device_id = request.query_params.get("device_id", "") + try: + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=30)) as session: + async with session.post( + "https://id.vk.ru/oauth2/auth", + data={ + "grant_type": "authorization_code", + "client_id": client_id, + "client_secret": client_secret, + "redirect_uri": redirect_uri, + "code": code, + "code_verifier": code_verifier, + "device_id": device_id, + "state": state, + }, + ) as resp: + token_payload = await resp.json(content_type=None) + if token_payload.get("error"): + raise RuntimeError(f"{token_payload.get('error')}: {token_payload.get('error_description')}") + access_token = str(token_payload.get("access_token") or "").strip() + if not access_token: + raise RuntimeError(f"VK did not return access_token: {token_payload}") + refresh_token = str(token_payload.get("refresh_token") or "").strip() + expires_in = token_payload.get("expires_in") + expires_at = None + try: + expires_at = (now_utc() + timedelta(seconds=int(expires_in))).isoformat() if expires_in is not None else None + except Exception: + expires_at = None + updates = { + "vk_poster_access_token": access_token, + "vk_poster_refresh_token": refresh_token, + "vk_poster_token_device_id": str(token_payload.get("device_id") or device_id or ""), + "vk_poster_token_expires_at": expires_at or "", + } + for key, value in updates.items(): + await pool.execute( + """ + UPDATE app_settings + SET value_json=$2::jsonb, + updated_by=$3, + updated_at=NOW() + WHERE key=$1 + """, + key, + json.dumps(value), + user["id"], + ) + check: dict[str, Any] = {} + async with VKAPIClient(token=access_token, rps=2, timeout_total_sec=30, retry_attempts=1) as client: + try: + upload_url = await client.get_wall_upload_server(abs(owner_id)) + check["photo_upload_server_ok"] = bool(upload_url) + except Exception as exc: + check["photo_upload_server_ok"] = False + check["photo_upload_server_error"] = str(exc) + try: + response = await client.call("wall.get", owner_id=owner_id, count=1, filter="owner") + check["wall_get_ok"] = isinstance(response, dict) + except Exception as exc: + check["wall_get_ok"] = False + check["wall_get_error"] = str(exc) + await audit( + user["id"], + "vk_oauth.token_saved", + "vk_poster", + None, + { + "owner_id": owner_id, + "user_id": token_payload.get("user_id"), + "expires_in": token_payload.get("expires_in"), + "has_refresh_token": bool(refresh_token), + "expires_at": expires_at, + **check, + }, + request, + ) + result.update( + { + "ok": True, + "error": "", + "error_description": "", + "owner_id": owner_id, + "user_id": token_payload.get("user_id"), + "expires_in": token_payload.get("expires_in"), + "has_refresh_token": bool(refresh_token), + "expires_at": expires_at, + **check, + } + ) + except Exception as exc: + result.update({"error": "token_exchange_failed", "error_description": str(exc)}) + response = templates.TemplateResponse("vk_oauth_callback.html", base_context(request, user, result=result)) + response.delete_cookie(VK_OAUTH_STATE_COOKIE) + response.delete_cookie(VK_OAUTH_VERIFIER_COOKIE) + return response + + @app.get("/", response_class=HTMLResponse) async def index(request: Request): user = await get_current_user(request) @@ -1386,6 +1913,7 @@ async def login_post(request: Request, login: str = Form(...), password: str = F ip, login_value, ) + await audit(None, "auth.login_blocked", "admin_login", None, {"login": login_value}, request) return templates.TemplateResponse( "login.html", base_context(request, None, error="Слишком много попыток. Подожди 15 минут и попробуй снова."), @@ -1401,6 +1929,7 @@ async def login_post(request: Request, login: str = Form(...), password: str = F ip, login_value, ) + await audit(None, "auth.login_failed", "admin_login", None, {"login": login_value}, request) return templates.TemplateResponse("login.html", base_context(request, None, error="Неверный логин или пароль")) await pool.execute("INSERT INTO admin_login_attempts(ip, login, success) VALUES($1, $2, TRUE)", ip, login_value) @@ -1416,6 +1945,7 @@ async def login_post(request: Request, login: str = Form(...), password: str = F csrf, now_utc() + timedelta(days=14), ) + await audit(row["id"], "auth.login_success", "admin_user", int(row["id"]), {"login": login_value}, request) response = redirect("/sources") response.set_cookie(COOKIE_NAME, token, httponly=True, secure=True, samesite="lax", max_age=14 * 24 * 3600) return response @@ -1424,9 +1954,12 @@ async def login_post(request: Request, login: str = Form(...), password: str = F @app.post("/logout") async def logout(request: Request): token = request.cookies.get(COOKIE_NAME) + user = await get_current_user(request) if token else None if token: pool = await get_pool() await pool.execute("DELETE FROM admin_sessions WHERE token_hash=$1", token_hash(token, settings.app_secret_key)) + if user: + await audit(user["id"], "auth.logout", "admin_user", int(user["id"]), None, request) response = redirect("/login") response.delete_cookie(COOKIE_NAME) return response @@ -1800,6 +2333,7 @@ async def source_delete(request: Request, source_id: int, csrf_token: str = Form @app.get("/raw", response_class=HTMLResponse) +@app.get("/v2/raw", response_class=HTMLResponse) async def raw_posts( request: Request, status_filter: str = "", @@ -1894,7 +2428,46 @@ async def raw_posts( ORDER BY rpm.sort_order ASC, rpm.id ASC ) FILTER (WHERE rpm.id IS NOT NULL), '[]'::jsonb - ) AS media_items + ) AS media_items, + COALESCE( + ( + SELECT jsonb_agg( + jsonb_build_object( + 'platform', pp.platform, + 'status', pp.status, + 'target_id', pp.target_id, + 'external_id', pp.external_id, + 'url', pp.url, + 'error', pp.error, + 'published_at', pp.published_at + ) + ORDER BY pp.platform + ) + FROM post_publications pp + WHERE pp.raw_post_id=rp.id + ), + '[]'::jsonb + ) AS platform_publications + , + COALESCE( + ( + SELECT jsonb_agg( + jsonb_build_object( + 'platform', pp.platform, + 'status', pp.status, + 'target_id', pp.target_id, + 'external_id', pp.external_id, + 'url', pp.url, + 'error', pp.error, + 'published_at', pp.published_at + ) + ORDER BY pp.platform + ) + FROM post_publications pp + WHERE pp.raw_post_id=rp.id + ), + '[]'::jsonb + ) AS platform_publications FROM raw_posts rp JOIN sources s ON s.id=rp.source_id LEFT JOIN raw_post_media rpm ON rpm.raw_post_id=rp.id AND COALESCE(rpm.editor_added, FALSE)=FALSE @@ -1908,8 +2481,15 @@ async def raw_posts( pager["offset"], ) posts = [prepare_raw_post(row) for row in rows] + + is_v2 = request.url.path.startswith("/v2") + if is_v2: + template_name = "v2/raw_posts_content.html" if request.headers.get("hx-request") else "v2/raw_posts.html" + else: + template_name = "raw_posts.html" + return templates.TemplateResponse( - "raw_posts.html", + template_name, base_context( request, user, @@ -1932,6 +2512,61 @@ async def raw_posts( ) +@app.post("/raw/send-to-rewrite") +async def raw_send_to_rewrite( + request: Request, + csrf_token: str = Form(...), + raw_post_id: list[int] | None = Form(default=None), + next_url: str = Form("/raw"), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + post_ids = sorted({int(post_id) for post_id in (raw_post_id or []) if int(post_id) > 0}) + safe_next = next_url if next_url.startswith("/raw") and not next_url.startswith("//") else "/raw" + if not post_ids: + return redirect(safe_next) + pool = await get_pool() + updated_rows = await pool.fetch( + """ + WITH selected AS ( + SELECT id + FROM raw_posts + WHERE id=ANY($1::bigint[]) + AND status='storage_ready' + AND COALESCE(rewrite_status, 'pending') IN ('pending', 'failed') + FOR UPDATE + ) + UPDATE raw_posts rp + SET qualification_status='accepted', + qualification_decision='accepted', + qualification_model_decision='manual', + qualification_reason='Ручная отправка в рерайт из Raw', + qualification_reject_tag=NULL, + qualified_at=NOW(), + rewrite_status='pending', + rewrite_notes=NULL, + rewrite_batch_id=NULL, + editorial_status=NULL, + updated_at=NOW() + FROM selected + WHERE rp.id=selected.id + RETURNING rp.id + """, + post_ids, + ) + updated_ids = [int(row["id"]) for row in updated_rows] + await audit( + user["id"], + "raw.manual_send_to_rewrite", + "raw_post", + None, + {"selected_ids": post_ids, "updated_ids": updated_ids, "ignored_count": len(post_ids) - len(updated_ids)}, + ) + return redirect(safe_next) + + @app.get("/editor", response_class=HTMLResponse) async def editor_feed( request: Request, @@ -2002,6 +2637,7 @@ async def editor_feed( f""" SELECT rp.*, s.name AS source_name, s.tag AS source_tag, s.platform AS source_platform, s.url AS source_url, u.login AS reviewed_by_login, + COALESCE(c_id.tag, c_name.tag) AS resolved_category_tag, COUNT(rpm.id) AS media_count, COALESCE( jsonb_agg( @@ -2016,13 +2652,34 @@ async def editor_feed( ORDER BY rpm.sort_order ASC, rpm.id ASC ) FILTER (WHERE rpm.id IS NOT NULL), '[]'::jsonb - ) AS media_items + ) AS media_items, + COALESCE( + ( + SELECT jsonb_agg( + jsonb_build_object( + 'platform', pp.platform, + 'status', pp.status, + 'target_id', pp.target_id, + 'external_id', pp.external_id, + 'url', pp.url, + 'error', pp.error, + 'published_at', pp.published_at + ) + ORDER BY pp.platform + ) + FROM post_publications pp + WHERE pp.raw_post_id=rp.id + ), + '[]'::jsonb + ) AS platform_publications FROM raw_posts rp JOIN sources s ON s.id=rp.source_id LEFT JOIN admin_users u ON u.id=rp.reviewed_by + LEFT JOIN content_categories c_id ON c_id.sort_order=COALESCE(rp.final_category_id, rp.rewrite_category_id) + LEFT JOIN content_categories c_name ON lower(c_name.name)=lower(COALESCE(rp.final_category, rp.rewrite_category, '')) LEFT JOIN raw_post_media rpm ON rpm.raw_post_id=rp.id AND COALESCE(rpm.editor_hidden, FALSE)=FALSE WHERE {where_sql} - GROUP BY rp.id, s.name, s.tag, s.platform, s.url, u.login + GROUP BY rp.id, s.name, s.tag, s.platform, s.url, u.login, c_id.tag, c_name.tag ORDER BY CASE COALESCE(rp.editorial_status, 'review') WHEN 'review' THEN 0 @@ -2030,7 +2687,9 @@ async def editor_feed( WHEN 'regenerating' THEN 2 WHEN 'rejected' THEN 3 WHEN 'accepted' THEN 4 - ELSE 5 + WHEN 'publish_failed' THEN 5 + WHEN 'published' THEN 6 + ELSE 7 END, {sort_map.get(sort, sort_map["rewritten_desc"])} LIMIT ${limit_index} OFFSET ${offset_index} @@ -2040,7 +2699,7 @@ async def editor_feed( pager["offset"], ) posts = [prepare_editor_post(row) for row in rows] - categories = await writer_categories() + categories = await writer_category_options() counts_rows = await pool.fetch( """ SELECT COALESCE(editorial_status, 'review') AS status, COUNT(*) AS count @@ -2082,20 +2741,44 @@ async def editor_accept(request: Request, post_id: int, csrf_token: str = Form(. return redirect("/login") require_csrf(user, csrf_token) pool = await get_pool() + row = await pool.fetchrow( + """ + SELECT rp.final_text, + rp.rewritten_text, + COALESCE(rp.final_category, rp.rewrite_category, '') AS category_name, + COALESCE(rp.final_category_id, rp.rewrite_category_id) AS category_id, + COALESCE(rp.final_source_tag, rp.rewrite_source_tag, s.tag, '') AS source_tag + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + WHERE rp.id=$1 AND rp.rewrite_status='ready' + """, + post_id, + ) + if not row: + return redirect("/editor") + category = await resolve_category_for_editor(row["category_name"], row["category_id"]) + source_tag = normalize_hash_tag(row["source_tag"] or "", "source") + final_text = row["final_text"] or row["rewritten_text"] or "" await pool.execute( """ UPDATE raw_posts SET editorial_status='accepted', - final_text=COALESCE(final_text, rewritten_text), - final_category=COALESCE(final_category, rewrite_category), - final_category_tag=COALESCE(final_category_tag, rewrite_category_tag), - final_source_tag=COALESCE(final_source_tag, rewrite_source_tag), - reviewed_by=$2, + final_text=$2, + final_category=$3, + final_category_tag=$4, + final_source_tag=$5, + final_category_id=$6, + reviewed_by=$7, reviewed_at=NOW(), updated_at=NOW() WHERE id=$1 AND rewrite_status='ready' """, post_id, + final_text, + category["name"], + category["tag"], + source_tag, + category["id"], user["id"], ) await audit(user["id"], "editor.accept", "raw_post", post_id) @@ -2145,7 +2828,9 @@ async def editor_save( return redirect("/login") require_csrf(user, csrf_token) final_category = final_category.strip() - final_category_tag = normalize_hash_tag(final_category, "category") + category = await resolve_category_for_editor(final_category) + final_category = category["name"] + final_category_tag = category["tag"] final_source_tag = normalize_hash_tag(final_source_tag, "source") status_value = "accepted" if action == "accept" else "edited" pool = await get_pool() @@ -2158,7 +2843,8 @@ async def editor_save( final_category_tag=$5, final_source_tag=$6, editor_notes=$7, - reviewed_by=CASE WHEN $2='accepted' THEN $8 ELSE reviewed_by END, + final_category_id=$8, + reviewed_by=CASE WHEN $2='accepted' THEN $9 ELSE reviewed_by END, reviewed_at=CASE WHEN $2='accepted' THEN NOW() ELSE reviewed_at END, edited_at=NOW(), updated_at=NOW() @@ -2171,6 +2857,7 @@ async def editor_save( final_category_tag, final_source_tag, editor_notes.strip()[:1000], + category["id"], user["id"], ) delete_media_ids = delete_media_ids or [] @@ -2508,7 +3195,7 @@ async def workers(request: Request): item["value_json"] = json.loads(item["value_json"]) except json.JSONDecodeError: pass - if item.get("key") == "ai_writer_categories": + if item.get("key") in {"ai_writer_categories", "tg_poster_schedule_json", "vk_poster_schedule_json", *AI_WORKER_ENABLED_SETTINGS.values()}: continue settings.append(item) settings.sort(key=setting_sort_key) @@ -2535,18 +3222,142 @@ async def workers(request: Request): writer_model=writer_model, writer_model_options=writer_model_options, categories=await category_rows(), + tg_schedule=await tg_poster_schedule_rows(), + vk_schedule=await vk_poster_schedule_rows(), category_titles=CATEGORY_TITLES, prompt_hints=PROMPT_HINTS, ), ) +@app.post("/tg-poster-schedule/create") +async def tg_poster_schedule_create( + request: Request, + csrf_token: str = Form(...), + time_value: str = Form(..., alias="time"), + count: int = Form(1), + enabled: str | None = Form(default=None), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + rows = await tg_poster_schedule_rows() + rows.append( + { + "id": f"slot-{secrets.token_hex(4)}", + "time": time_value, + "count": count, + "enabled": enabled == "on", + } + ) + rows = normalize_tg_schedule(rows) + await save_tg_poster_schedule(rows) + await audit(user["id"], "tg_schedule.create", "tg_poster_schedule", None, {"rows": rows}) + return redirect("/workers") + + +@app.post("/tg-poster-schedule/{schedule_id}/update") +async def tg_poster_schedule_update( + request: Request, + schedule_id: str, + csrf_token: str = Form(...), + time_value: str = Form(..., alias="time"), + count: int = Form(1), + enabled: str | None = Form(default=None), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + rows = await tg_poster_schedule_rows() + for row in rows: + if row["id"] == schedule_id: + row.update({"time": time_value, "count": count, "enabled": enabled == "on"}) + break + rows = normalize_tg_schedule(rows) + await save_tg_poster_schedule(rows) + await audit(user["id"], "tg_schedule.update", "tg_poster_schedule", None, {"id": schedule_id, "rows": rows}) + return redirect("/workers") + + +@app.post("/tg-poster-schedule/{schedule_id}/delete") +async def tg_poster_schedule_delete(request: Request, schedule_id: str, csrf_token: str = Form(...)): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + rows = [row for row in await tg_poster_schedule_rows() if row["id"] != schedule_id] + await save_tg_poster_schedule(rows) + await audit(user["id"], "tg_schedule.delete", "tg_poster_schedule", None, {"id": schedule_id, "rows": rows}) + return redirect("/workers") + + +@app.post("/vk-poster-schedule/create") +async def vk_poster_schedule_create( + request: Request, + csrf_token: str = Form(...), + time_value: str = Form(..., alias="time"), + count: int = Form(1), + enabled: str | None = Form(default=None), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + rows = await vk_poster_schedule_rows() + rows.append({"id": f"vk-slot-{secrets.token_hex(4)}", "time": time_value, "count": count, "enabled": enabled == "on"}) + rows = normalize_tg_schedule(rows) + await save_vk_poster_schedule(rows) + await audit(user["id"], "vk_schedule.create", "vk_poster_schedule", None, {"rows": rows}) + return redirect("/workers") + + +@app.post("/vk-poster-schedule/{schedule_id}/update") +async def vk_poster_schedule_update( + request: Request, + schedule_id: str, + csrf_token: str = Form(...), + time_value: str = Form(..., alias="time"), + count: int = Form(1), + enabled: str | None = Form(default=None), +): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + rows = await vk_poster_schedule_rows() + for row in rows: + if row["id"] == schedule_id: + row.update({"time": time_value, "count": count, "enabled": enabled == "on"}) + break + rows = normalize_tg_schedule(rows) + await save_vk_poster_schedule(rows) + await audit(user["id"], "vk_schedule.update", "vk_poster_schedule", None, {"id": schedule_id, "rows": rows}) + return redirect("/workers") + + +@app.post("/vk-poster-schedule/{schedule_id}/delete") +async def vk_poster_schedule_delete(request: Request, schedule_id: str, csrf_token: str = Form(...)): + user = await get_current_user(request) + if not user: + return redirect("/login") + require_csrf(user, csrf_token) + rows = [row for row in await vk_poster_schedule_rows() if row["id"] != schedule_id] + await save_vk_poster_schedule(rows) + await audit(user["id"], "vk_schedule.delete", "vk_poster_schedule", None, {"id": schedule_id, "rows": rows}) + return redirect("/workers") + + @app.post("/categories/create") async def category_create( request: Request, csrf_token: str = Form(...), name: str = Form(...), tag: str = Form(""), + site_name: str = Form(...), + site_slug: str = Form(...), + site_enabled: str | None = Form(None), ): user = await get_current_user(request) if not user: @@ -2556,22 +3367,32 @@ async def category_create( if not name: return redirect("/workers") tag = normalize_hash_tag(tag or name, "category") + site_name = site_name.strip() + site_slug = re.sub(r"[^a-z0-9-]+", "-", site_slug.strip().lower().replace("_", "-")).strip("-") + if not site_name or not site_slug: + return redirect("/workers") pool = await get_pool() sort_order = int(await pool.fetchval("SELECT COALESCE(MAX(sort_order), 0) + 1 FROM content_categories") or 1) await pool.execute( """ - INSERT INTO content_categories(name, tag, sort_order) - VALUES($1, $2, $3) + INSERT INTO content_categories(name, tag, sort_order, site_name, site_slug, site_enabled) + VALUES($1, $2, $3, $4, $5, $6) ON CONFLICT (name) DO UPDATE SET tag=$2, + site_name=$4, + site_slug=$5, + site_enabled=$6, is_active=TRUE, updated_at=NOW() """, name, tag, sort_order, + site_name, + site_slug, + site_enabled is not None, ) - await audit(user["id"], "category.create", "content_category", None, {"name": name, "tag": tag}) + await audit(user["id"], "category.create", "content_category", None, {"name": name, "tag": tag, "site_name": site_name, "site_slug": site_slug, "site_enabled": site_enabled is not None}) return redirect("/workers") @@ -2582,6 +3403,9 @@ async def category_update( csrf_token: str = Form(...), name: str = Form(...), tag: str = Form(""), + site_name: str = Form(...), + site_slug: str = Form(...), + site_enabled: str | None = Form(None), ): user = await get_current_user(request) if not user: @@ -2591,20 +3415,30 @@ async def category_update( if not name: return redirect("/workers") tag = normalize_hash_tag(tag or name, "category") + site_name = site_name.strip() + site_slug = re.sub(r"[^a-z0-9-]+", "-", site_slug.strip().lower().replace("_", "-")).strip("-") + if not site_name or not site_slug: + return redirect("/workers") pool = await get_pool() await pool.execute( """ UPDATE content_categories SET name=$2, tag=$3, + site_name=$4, + site_slug=$5, + site_enabled=$6, updated_at=NOW() WHERE id=$1 """, category_id, name, tag, + site_name, + site_slug, + site_enabled is not None, ) - await audit(user["id"], "category.update", "content_category", category_id, {"name": name, "tag": tag}) + await audit(user["id"], "category.update", "content_category", category_id, {"name": name, "tag": tag, "site_name": site_name, "site_slug": site_slug, "site_enabled": site_enabled is not None}) return redirect("/workers") @@ -2663,19 +3497,48 @@ async def worker_toggle(request: Request, worker_name: str, csrf_token: str = Fo return redirect("/login") require_csrf(user, csrf_token) pool = await get_pool() - row = await pool.fetchrow( - """ - UPDATE worker_controls - SET enabled=NOT enabled, - updated_by=$2, - updated_at=NOW() - WHERE name=$1 - RETURNING enabled - """, - worker_name, + async with pool.acquire() as conn: + async with conn.transaction(): + row = await conn.fetchrow( + """ + UPDATE worker_controls + SET enabled=NOT enabled, + updated_by=$2, + updated_at=NOW() + WHERE name=$1 + RETURNING enabled + """, + worker_name, + user["id"], + ) + if not row: + return redirect("/workers") + enabled = bool(row["enabled"]) + setting_key = AI_WORKER_ENABLED_SETTINGS.get(worker_name) + if setting_key: + await conn.execute( + """ + UPDATE app_settings + SET value_json=$2::jsonb, + updated_by=$3, + updated_at=NOW() + WHERE key=$1 + """, + setting_key, + json.dumps(enabled), + user["id"], + ) + await audit( user["id"], + "worker.toggle", + "worker", + None, + { + "name": worker_name, + "enabled": enabled, + "synced_setting": AI_WORKER_ENABLED_SETTINGS.get(worker_name), + }, ) - await audit(user["id"], "worker.toggle", "worker", None, {"name": worker_name, "enabled": bool(row["enabled"])}) return redirect("/workers") @@ -2700,6 +3563,8 @@ async def settings_save(request: Request, csrf_token: str = Form(...), key: str if not value.strip(): return redirect("/workers") parsed = value.strip() + elif value_type == "json": + parsed = json.loads(value) else: parsed = value if key in {"ai_qualifier_prompt", "ai_qualifier_contract", "ai_writer_prompt", "ai_writer_contract"}: @@ -2720,6 +3585,124 @@ async def settings_save(request: Request, csrf_token: str = Form(...), key: str return redirect("/workers") +@app.get("/logs", response_class=HTMLResponse) +async def logs_page( + request: Request, + q: str = "", + action: str = "", + actor_id: int = 0, + entity_type: str = "", + date_from: str = "", + date_to: str = "", + page: int = 1, + per_page: int = 50, +): + user = await get_current_user(request) + if not user: + return redirect("/login") + pool = await get_pool() + await prune_audit_log() + where_parts: list[str] = [] + args: list[Any] = [] + if q.strip(): + add_where( + where_parts, + args, + "(al.action ILIKE ${i} OR al.entity_type ILIKE ${i} OR COALESCE(al.after_json::text, '') ILIKE ${i})", + f"%{q.strip()}%", + ) + if action.strip(): + add_where(where_parts, args, "al.action=${i}", action.strip()) + if actor_id > 0: + add_where(where_parts, args, "al.actor_id=${i}", actor_id) + if entity_type.strip(): + add_where(where_parts, args, "al.entity_type=${i}", entity_type.strip()) + parsed_date_from = parse_date_filter(date_from) + parsed_date_to = parse_date_filter(date_to) + if parsed_date_from: + add_where(where_parts, args, "timezone('Asia/Yekaterinburg', al.created_at)::date >= ${i}::date", parsed_date_from) + if parsed_date_to: + add_where(where_parts, args, "timezone('Asia/Yekaterinburg', al.created_at)::date <= ${i}::date", parsed_date_to) + where_sql = " AND ".join(where_parts) if where_parts else "TRUE" + total = int( + await pool.fetchval( + f""" + SELECT COUNT(*) + FROM audit_log al + LEFT JOIN admin_users u ON u.id=al.actor_id + WHERE {where_sql} + """, + *args, + ) + or 0 + ) + pager = pagination(page, per_page, total) + limit_index = len(args) + 1 + offset_index = len(args) + 2 + rows = await pool.fetch( + f""" + SELECT al.*, u.login AS actor_login + FROM audit_log al + LEFT JOIN admin_users u ON u.id=al.actor_id + WHERE {where_sql} + ORDER BY al.created_at DESC, al.id DESC + LIMIT ${limit_index} OFFSET ${offset_index} + """, + *args, + pager["per_page"], + pager["offset"], + ) + action_rows = await pool.fetch( + """ + SELECT action, COUNT(*) AS count + FROM audit_log + WHERE created_at >= NOW() - INTERVAL '30 days' + GROUP BY action + ORDER BY COUNT(*) DESC, action + LIMIT 80 + """ + ) + actor_rows = await pool.fetch( + """ + SELECT u.id, u.login, COUNT(al.id) AS count + FROM audit_log al + JOIN admin_users u ON u.id=al.actor_id + WHERE al.created_at >= NOW() - INTERVAL '30 days' + GROUP BY u.id, u.login + ORDER BY COUNT(al.id) DESC, u.login + """ + ) + entity_rows = await pool.fetch( + """ + SELECT entity_type, COUNT(*) AS count + FROM audit_log + WHERE created_at >= NOW() - INTERVAL '30 days' + GROUP BY entity_type + ORDER BY COUNT(*) DESC, entity_type + LIMIT 50 + """ + ) + return templates.TemplateResponse( + "logs.html", + base_context( + request, + user, + logs=[prepare_audit_row(row) for row in rows], + q=q, + action=action, + actor_id=actor_id, + entity_type=entity_type, + date_from=date_from, + date_to=date_to, + actions=[dict(row) for row in action_rows], + actors=[dict(row) for row in actor_rows], + entity_types=[dict(row) for row in entity_rows], + preserved_query=preserved_query(request, {"page", "per_page"}), + pagination=pager, + ), + ) + + @app.get("/users", response_class=HTMLResponse) async def users(request: Request, page: int = 1, per_page: int = 50): user = await get_current_user(request) diff --git a/src/vk_parser_app/constants.py b/src/vk_parser_app/constants.py index dca6219..35d313f 100644 --- a/src/vk_parser_app/constants.py +++ b/src/vk_parser_app/constants.py @@ -11,6 +11,10 @@ POST_STATUS_STORAGE_READY = "storage_ready" POST_STATUS_SKIPPED = "skipped" POST_STATUS_FAILED = "failed" +PUBLICATION_STATUS_PENDING = "pending" +PUBLICATION_STATUS_PUBLISHED = "published" +PUBLICATION_STATUS_FAILED = "publish_failed" + MEDIA_STATUS_PENDING = "pending" MEDIA_STATUS_UPLOADED = "uploaded" MEDIA_STATUS_LINK_ONLY = "link_only" @@ -28,3 +32,9 @@ WORKER_PARSER = "vk-parser" WORKER_STORAGE_UPLOADER = "vk-storage-uploader" WORKER_AI_QUALIFIER = "ai-qualifier" WORKER_AI_WRITER = "ai-writer" +WORKER_TG_POSTER = "tg-poster" +WORKER_TG_REACTOR = "tg-reactor" +WORKER_VK_POSTER = "vk-poster" +WORKER_MAX_POSTER = "max-poster" +WORKER_SITE_POSTER = "site-poster" +WORKER_DAILY_REPORT = "daily-report" diff --git a/src/vk_parser_app/templates/base.html b/src/vk_parser_app/templates/base.html index 3bdd5eb..5e5a6f0 100644 --- a/src/vk_parser_app/templates/base.html +++ b/src/vk_parser_app/templates/base.html @@ -3,6 +3,7 @@ + {{ title or "VK Parser Admin" }} + + + +
+
+ Parser V2 + +
+ +
+ +
+ {% block body %}{% endblock %} +
+ + + + diff --git a/src/vk_parser_app/templates/v2/raw_posts.html b/src/vk_parser_app/templates/v2/raw_posts.html new file mode 100644 index 0000000..49a7427 --- /dev/null +++ b/src/vk_parser_app/templates/v2/raw_posts.html @@ -0,0 +1,14 @@ +{% extends "v2/base.html" %} + +{% block body %} +
+
+

Raw-посты

+
Исходный пост, этап обработки и AI-оценка.
+
+
+ +
+ {% include "v2/raw_posts_content.html" %} +
+{% endblock %} diff --git a/src/vk_parser_app/templates/v2/raw_posts_content.html b/src/vk_parser_app/templates/v2/raw_posts_content.html new file mode 100644 index 0000000..0736434 --- /dev/null +++ b/src/vk_parser_app/templates/v2/raw_posts_content.html @@ -0,0 +1,152 @@ +
+ + + + +
+
+
+ + +
+
+ Всего: {{ pagination.total }} +
+ +
+ На странице: + +
+
+ +
+ + + + + + + + + + + + {% for p in posts %} + + + + + + + + {% endfor %} + +
#ИсточникЭтапПостAI
+ {{ p.id }} + + {{ p.source_name }} +
{{ p.source_platform }} · #{{ p.source_tag or "—" }}
+ {% if p.posted_at_fmt %}
{{ p.posted_at_fmt }}
{% endif %} + 		+
+ {{ p.stage.label }} +
+ {% if p.error_reason %}
{{ p.error_reason }}
{% endif %} + 		+
+ {{ p.raw_text[:200] }}{% if p.raw_text|length > 200 %}...{% endif %} +
+ 		+
+ + {{ p.ai_badge.label }} + + {{ p.ai_badge.sublabel }} +
+
+
+ + +
+
+ {% if pagination.has_prev %} + + {% else %} + + {% endif %} + + + + {% if pagination.has_next %} + + {% else %} + + {% endif %} +
+
+ +
+
+
+
diff --git a/src/vk_parser_app/templates/vk_oauth_callback.html b/src/vk_parser_app/templates/vk_oauth_callback.html index 64977cc..8d96bee 100644 --- a/src/vk_parser_app/templates/vk_oauth_callback.html +++ b/src/vk_parser_app/templates/vk_oauth_callback.html @@ -1,17 +1,37 @@ {% extends "base.html" %} {% block body %}
-

VK OAuth callback

- {% if code %} -

Скопируйте code и code_verifier для обмена на access token.

+

VK OAuth

+ {% if result %} + {% if result.ok %} +

Токен VK-постера сохранён

+
+
Сообщество
{{ result.owner_id }}
+
User ID
{{ result.user_id or "—" }}
+
Живёт, сек
{{ result.expires_in or "—" }}
+
Истекает
{{ result.expires_at or "—" }}
+
Refresh token
{% if result.has_refresh_token %}OK{% else %}Не получен{% endif %}
+
wall.get
{% if result.wall_get_ok %}OK{% else %}Ошибка: {{ result.wall_get_error }}{% endif %}
+
Фото upload
+
+ {% if result.photo_upload_server_ok %} + OK + {% else %} + Недоступно: {{ result.photo_upload_server_error }} +
VK выдаёт право photos для upload API отдельно; постер всё равно опубликует текст и существующие VK-вложения.
+ {% endif %} +
+
+

Вернуться к воркерам

+ {% else %} +

VK OAuth не завершился

+

{{ result.error_description or result.error or "Неизвестная ошибка" }}

+

Вернуться к воркерам Повторить

+ {% endif %} + {% elif code %} +

VK вернул code. Новый обработчик должен обменивать его автоматически через /vk-oauth/callback.

- - - {% if device_id %} - - - {% endif %} {% if state and expected_state and state != expected_state %}

state не совпал, такой code лучше не использовать.

{% endif %} @@ -19,34 +39,7 @@

{{ error }}

{{ error_description }}

{% else %} -

VK вернул callback без code.

+

VK вернул callback без данных.

{% endif %} - -
- {% endblock %} diff --git a/src/vk_parser_app/templates/workers.html b/src/vk_parser_app/templates/workers.html index 28a54cd..d78a971 100644 --- a/src/vk_parser_app/templates/workers.html +++ b/src/vk_parser_app/templates/workers.html @@ -19,15 +19,18 @@
Категории публикаций -

Активные категории попадают в редакторскую и передаются AI-райтеру как ID, название и тэг. ID стабилен: модель возвращает его в JSON, а тэг подставляется из БД.

+

Название описывает категорию для AI, тэг используется в соцсетях. Название и URL сайта хранятся отдельно и не меняют рабочие хэштеги.

+ + +
- + {% for c in categories %} + + - +
НазваниеТэгIDСтатус
Название AIТэгНазвание сайтаURL сайтаIDСтатус
@@ -37,8 +40,10 @@ {% if c.is_active %}активна{% else %}выключена{% endif %}{% if c.is_active %}активна{% else %}выключена{% endif %}
@@ -55,6 +60,75 @@
+
+ Расписание TG-постера +

Время публикации задаётся по Екатеринбургу. Для запуска нужны оба флага: worker tg-poster и настройка tg_poster_enabled.

+ + + + + + + + + + {% for row in tg_schedule %} + + + + + + + {% endfor %} +
Время ЕкбПостовСтатус
+
+ + +
+ 		+ +
+ + +
+
+
+ +
+ Расписание VK-постера +

Время публикации задаётся по Екатеринбургу. Для запуска нужны оба флага: worker vk-poster и настройка vk_poster_enabled.

+

Получить VK user token

+
+ + + + + +
+ + + {% for row in vk_schedule %} + + + + + + + {% endfor %} +
Время ЕкбПостовСтатус
+
+ + +
+ 		+ +
+ + +
+
+
+

Настройки

{% for category, rows in settings|groupby("category") %}
@@ -126,6 +200,8 @@ {% endif %} + {% elif s.value_type == "json" %} + {% else %} {% endif %} diff --git a/src/vk_parser_app/text_utils.py b/src/vk_parser_app/text_utils.py index 0066630..868a8d2 100644 --- a/src/vk_parser_app/text_utils.py +++ b/src/vk_parser_app/text_utils.py @@ -11,6 +11,34 @@ def normalize_hash_tag(value: str, fallback: str = "source") -> str: return tag or fallback +def strip_trailing_hashtag_line(text: str) -> str: + lines = str(text or "").rstrip().splitlines() + while lines and not lines[-1].strip(): + lines.pop() + if not lines: + return "" + parts = lines[-1].split() + if parts and all(part.startswith("#") and normalize_hash_tag(part, "") for part in parts): + lines.pop() + return "\n".join(lines).rstrip() + + +def publication_hashtags(category_tag: str, source_tag: str) -> str: + tags = [ + normalize_hash_tag(category_tag, "category"), + normalize_hash_tag(source_tag, "source"), + ] + return " ".join(f"#{tag}" for tag in tags if tag) + + +def build_publication_text(text: str, category_tag: str, source_tag: str) -> str: + base = strip_trailing_hashtag_line(text) + hashtags = publication_hashtags(category_tag, source_tag) + if not hashtags: + return base + return f"{base}\n\n{hashtags}" if base else hashtags + + def parse_categories(value: object) -> list[str]: if isinstance(value, list): raw_items = [str(item) for item in value] diff --git a/src/vk_parser_app/vk_api.py b/src/vk_parser_app/vk_api.py index 4569e54..8718add 100644 --- a/src/vk_parser_app/vk_api.py +++ b/src/vk_parser_app/vk_api.py @@ -155,6 +155,20 @@ class VKAPIClient: raise VKAPIError(None, f"photos.saveWallPhoto returned invalid response: {json.dumps(saved)[:300]}") return saved[0] + async def upload_wall_photo_url(self, group_id: int, url: str) -> dict: + if not self.session: + raise RuntimeError("VKAPIClient is not initialized") + async with self.session.get(url) as resp: + resp.raise_for_status() + content_type = resp.headers.get("content-type") or "image/jpeg" + data = await resp.read() + suffix = ".jpg" + if "png" in content_type: + suffix = ".png" + elif "webp" in content_type: + suffix = ".webp" + return await self.upload_wall_photo_bytes(group_id, data, filename=f"photo{suffix}") + async def create_wall_post( self, owner_id: int, diff --git a/src/vk_parser_app/workers/ai_qualifier.py b/src/vk_parser_app/workers/ai_qualifier.py index a7f34a0..e904f0d 100644 --- a/src/vk_parser_app/workers/ai_qualifier.py +++ b/src/vk_parser_app/workers/ai_qualifier.py @@ -151,6 +151,22 @@ class AIQualifierWorker: FROM raw_posts WHERE status='storage_ready' AND COALESCE(qualification_status, 'pending') IN ('pending', 'failed') + AND EXISTS ( + SELECT 1 + FROM raw_post_media m + WHERE m.raw_post_id=raw_posts.id + AND COALESCE(m.editor_hidden, FALSE)=FALSE + AND m.media_type IN ('photo', 'video') + AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') <> '' + ) + AND NOT EXISTS ( + SELECT 1 + FROM raw_post_media m + WHERE m.raw_post_id=raw_posts.id + AND COALESCE(m.editor_hidden, FALSE)=FALSE + AND m.media_type IN ('photo', 'video') + AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = '' + ) """ ) if int(count or 0) < batch_size: @@ -163,6 +179,22 @@ class AIQualifierWorker: FROM raw_posts WHERE status='storage_ready' AND COALESCE(qualification_status, 'pending') IN ('pending', 'failed') + AND EXISTS ( + SELECT 1 + FROM raw_post_media m + WHERE m.raw_post_id=raw_posts.id + AND COALESCE(m.editor_hidden, FALSE)=FALSE + AND m.media_type IN ('photo', 'video') + AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') <> '' + ) + AND NOT EXISTS ( + SELECT 1 + FROM raw_post_media m + WHERE m.raw_post_id=raw_posts.id + AND COALESCE(m.editor_hidden, FALSE)=FALSE + AND m.media_type IN ('photo', 'video') + AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = '' + ) ORDER BY created_at ASC, id ASC FOR UPDATE SKIP LOCKED LIMIT $1 diff --git a/src/vk_parser_app/workers/ai_writer.py b/src/vk_parser_app/workers/ai_writer.py index 264185c..a54ae59 100644 --- a/src/vk_parser_app/workers/ai_writer.py +++ b/src/vk_parser_app/workers/ai_writer.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import hashlib import json +import re from datetime import datetime, timezone from decimal import Decimal from typing import Any @@ -15,7 +16,14 @@ from ..constants import WORKER_AI_WRITER from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool from ..heartbeat import HeartbeatReporter from ..jobs import is_worker_enabled -from ..text_utils import normalize_hash_tag, parse_categories +from ..text_utils import build_publication_text, normalize_hash_tag, parse_categories + + +NON_TARGET_CATEGORY_ID = 18 +NON_TARGET_CATEGORY_TAGS = {"нцк", "nck"} +EMOJI_LEADING_MARKER_RE = re.compile( + r"^[ \t]*(?:🧵|🎯|🛡|🧤|📦|💵|⚙\ufe0f?|🎨|🎒|🔧|👕|💪)\s+" +) def now_utc() -> datetime: @@ -30,6 +38,14 @@ def normalize_prompt(prompt: str) -> str: return (prompt or "").replace("\\r\\n", "\n").replace("\\n", "\n").strip() +def strip_leading_emoji_markers(text: str) -> str: + lines = str(text or "").splitlines() + cleaned: list[str] = [] + for line in lines: + cleaned.append(EMOJI_LEADING_MARKER_RE.sub("", line)) + return "\n".join(cleaned).strip() + + DEFAULT_CONTRACT_PROMPT = """ OUTPUT SCHEMA (return array matching input order): {"rewrites":[{"id":123,"category_id":2,"text":"готовый текст без хэштегов","notes":"короткая заметка для редактора или null"}]} @@ -38,7 +54,9 @@ Rules: - Input is a JSON object with key "posts" containing accepted posts. - Input has key "categories" with objects: id, name, tag. - Each post includes producer_name and producer_tag. producer_name can be a manufacturer, shop, or publishing source. +- The "text" field for every rewrite must explicitly mention producer_name at least once. Do not satisfy this only through producer_tag, hashtags, notes, or metadata. - Pick exactly one category from the categories list in input and return its numeric id as category_id. +- If the post is non-target content, choose category_id 18 ("Не целевой контент", tag "НЦК"); the app will reject it automatically. - Do not add links or hashtags to rewritten text. - Return JSON only. No markdown. No text outside JSON. - Never return an empty object. Never omit the "rewrites" key. @@ -127,7 +145,7 @@ def validate_rewrites( post_id = int(item.get("id")) if post_id not in expected_ids: raise ValueError(f"AI returned unexpected id={post_id}") - text = str(item.get("text") or "").strip() + text = strip_leading_emoji_markers(str(item.get("text") or "").strip()) if len(text) < 40: raise ValueError(f"AI returned too short rewrite for id={post_id}") by_id = {int(c["id"]): c for c in categories if c.get("id") is not None} @@ -144,15 +162,18 @@ def validate_rewrites( raise ValueError(f"AI returned unknown category={category_name!r} for id={post_id}") notes = str(item.get("notes") or "").strip()[:1000] seen.add(post_id) + category_tag = normalize_hash_tag(category_row.get("tag") or category_row.get("name") or "", "category") + reject_by_category = int(category_row["id"]) == NON_TARGET_CATEGORY_ID or category_tag.lower() in NON_TARGET_CATEGORY_TAGS out.append( { "id": post_id, "category_id": int(category_row["id"]), "category": str(category_row["name"]), - "category_tag": normalize_hash_tag(category_row.get("tag") or category_row.get("name") or "", "category"), + "category_tag": category_tag, "source_tag": normalize_hash_tag(source_tags_by_id.get(post_id) or "", "source"), "text": text, "notes": notes, + "reject_by_category": reject_by_category, } ) missing = expected_ids - seen @@ -209,6 +230,22 @@ class AIWriterWorker: WHERE status='storage_ready' AND qualification_status='accepted' AND COALESCE(rewrite_status, 'pending') IN ('pending', 'failed') + AND EXISTS ( + SELECT 1 + FROM raw_post_media m + WHERE m.raw_post_id=raw_posts.id + AND COALESCE(m.editor_hidden, FALSE)=FALSE + AND m.media_type IN ('photo', 'video') + AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') <> '' + ) + AND NOT EXISTS ( + SELECT 1 + FROM raw_post_media m + WHERE m.raw_post_id=raw_posts.id + AND COALESCE(m.editor_hidden, FALSE)=FALSE + AND m.media_type IN ('photo', 'video') + AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = '' + ) """ ) if int(count or 0) < batch_size: @@ -222,6 +259,22 @@ class AIWriterWorker: WHERE status='storage_ready' AND qualification_status='accepted' AND COALESCE(rewrite_status, 'pending') IN ('pending', 'failed') + AND EXISTS ( + SELECT 1 + FROM raw_post_media m + WHERE m.raw_post_id=raw_posts.id + AND COALESCE(m.editor_hidden, FALSE)=FALSE + AND m.media_type IN ('photo', 'video') + AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') <> '' + ) + AND NOT EXISTS ( + SELECT 1 + FROM raw_post_media m + WHERE m.raw_post_id=raw_posts.id + AND COALESCE(m.editor_hidden, FALSE)=FALSE + AND m.media_type IN ('photo', 'video') + AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = '' + ) ORDER BY qualified_at ASC NULLS LAST, id ASC FOR UPDATE SKIP LOCKED LIMIT $1 @@ -316,6 +369,9 @@ class AIWriterWorker: async def apply_rewrites(self, batch_id: int, rewrites: list[dict], model: str, prompt: str) -> None: for item in rewrites: + final_text = build_publication_text(item["text"], item["category_tag"], item["source_tag"]) + editorial_status = "rejected" if item.get("reject_by_category") else "review" + editor_notes = "Отклонено AI: Не целевой контент" if item.get("reject_by_category") else None await self.pool.execute( """ UPDATE raw_posts @@ -329,12 +385,19 @@ class AIWriterWorker: rewrite_category_tag=$8, rewrite_source_tag=$9, rewrite_category_id=$10, - editorial_status='review', - final_text=$2, + editorial_status=$11, + final_text=$12, final_category=$7, final_category_tag=$8, final_source_tag=$9, final_category_id=$10, + editor_notes=$13, + qualification_status=CASE WHEN $11='rejected' THEN 'rejected' ELSE qualification_status END, + qualification_decision=CASE WHEN $11='rejected' THEN 'rejected' ELSE qualification_decision END, + qualification_model_decision=CASE WHEN $11='rejected' THEN 'rejected' ELSE qualification_model_decision END, + qualification_reason=CASE WHEN $11='rejected' THEN 'AI writer selected non-target category' ELSE qualification_reason END, + qualification_reject_tag=CASE WHEN $11='rejected' THEN 'non_target_content' ELSE qualification_reject_tag END, + reviewed_at=CASE WHEN $11='rejected' THEN NOW() ELSE reviewed_at END, rewritten_at=NOW(), updated_at=NOW() WHERE id=$1 @@ -349,6 +412,9 @@ class AIWriterWorker: item["category_tag"], item["source_tag"], item["category_id"], + editorial_status, + final_text, + editor_notes, ) async def call_ai( diff --git a/src/vk_parser_app/workers/daily_report.py b/src/vk_parser_app/workers/daily_report.py new file mode 100644 index 0000000..f17b80d --- /dev/null +++ b/src/vk_parser_app/workers/daily_report.py @@ -0,0 +1,457 @@ +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import re +import sys +from datetime import date, datetime, time, timedelta, timezone +from typing import Any +from zoneinfo import ZoneInfo + +from aiogram import Bot +from aiogram.exceptions import TelegramRetryAfter +from loguru import logger + +from ..config import settings +from ..constants import WORKER_DAILY_REPORT +from ..db import fetch_bool_setting, fetch_int_setting, fetch_setting, get_pool +from ..heartbeat import HeartbeatReporter +from ..jobs import is_worker_enabled + +LOCAL_TZ = ZoneInfo("Asia/Yekaterinburg") +MESSAGE_LIMIT = 4096 + + +def valid_time(value: str) -> bool: + return bool(re.fullmatch(r"(?:[01]\d|2[0-3]):[0-5]\d", str(value or "").strip())) + + +def parse_recipients(value: Any) -> list[int]: + raw_items: list[Any] + if isinstance(value, list): + raw_items = value + elif isinstance(value, str): + raw = value.strip() + if not raw: + return [] + try: + parsed = json.loads(raw) + raw_items = parsed if isinstance(parsed, list) else [parsed] + except json.JSONDecodeError: + raw_items = re.split(r"[\s,;]+", raw) + else: + raw_items = [value] + out: list[int] = [] + for item in raw_items: + try: + recipient_id = int(str(item).strip()) + except Exception: + continue + if recipient_id not in out: + out.append(recipient_id) + return out + + +def day_bounds(report_date: date) -> tuple[datetime, datetime]: + start_local = datetime.combine(report_date, time.min, tzinfo=LOCAL_TZ) + end_local = start_local + timedelta(days=1) + return start_local.astimezone(timezone.utc), end_local.astimezone(timezone.utc) + + +def fmt_int(value: Any) -> str: + try: + return f"{int(value):,}".replace(",", " ") + except Exception: + return "0" + + +def split_chunks(text: str, limit: int = MESSAGE_LIMIT) -> list[str]: + text = str(text or "").strip() + if not text: + return [] + chunks: list[str] = [] + while len(text) > limit: + split_at = text.rfind("\n", 0, limit) + if split_at < limit // 2: + split_at = limit + chunks.append(text[:split_at].strip()) + text = text[split_at:].strip() + if text: + chunks.append(text) + return chunks + + +class DailyReportWorker: + def __init__(self) -> None: + self.pool = None + self.bot: Bot | None = None + self.bot_token = "" + self.recipients: list[int] = [] + self.report_time = "00:04" + self.interval_sec = 60 + self.heartbeat = HeartbeatReporter(WORKER_DAILY_REPORT, 30) + + async def init(self) -> None: + self.pool = await get_pool() + await self.reload_settings() + + async def close(self) -> None: + if self.bot: + await self.bot.session.close() + + async def reload_settings(self) -> None: + token = ( + str(await fetch_setting("daily_report_bot_token", "") or "").strip() + or str(await fetch_setting("tg_poster_bot_token", "") or "").strip() + or settings.tg_bot_token + ) + if not token: + raise RuntimeError("daily_report_bot_token, tg_poster_bot_token and TG_BOT_TOKEN are empty") + if token != self.bot_token: + if self.bot: + await self.bot.session.close() + self.bot = Bot(token=token) + self.bot_token = token + self.recipients = parse_recipients(await fetch_setting("daily_report_recipient_ids", [442509142])) + if not self.recipients: + raise RuntimeError("daily_report_recipient_ids is empty") + report_time = str(await fetch_setting("daily_report_time", "00:04") or "00:04").strip() + self.report_time = report_time if valid_time(report_time) else "00:04" + self.interval_sec = max(10, await fetch_int_setting("daily_report_interval_sec", 60)) + logger.info("Daily report config: time={} recipients={}", self.report_time, self.recipients) + + async def save_last_sent_date(self, report_date: date) -> None: + await self.pool.execute( + """ + UPDATE app_settings + SET value_json=$2::jsonb, + updated_at=NOW() + WHERE key=$1 + """, + "daily_report_last_sent_date", + json.dumps(report_date.isoformat()), + ) + + async def countval(self, sql: str, *args: Any) -> int: + return int(await self.pool.fetchval(sql, *args) or 0) + + async def collect(self, report_date: date) -> dict[str, Any]: + start_utc, end_utc = day_bounds(report_date) + now_utc = datetime.now(timezone.utc) + async with self.pool.acquire() as conn: + raw_collected = await conn.fetchval( + "SELECT COUNT(*) FROM raw_posts WHERE created_at >= $1 AND created_at < $2", + start_utc, + end_utc, + ) + storage_ready = await conn.fetchval( + "SELECT COUNT(*) FROM raw_posts WHERE status='storage_ready' AND updated_at >= $1 AND updated_at < $2", + start_utc, + end_utc, + ) + media_failed = await conn.fetchval( + "SELECT COUNT(*) FROM raw_post_media WHERE status IN ('failed','link_only') AND updated_at >= $1 AND updated_at < $2", + start_utc, + end_utc, + ) + qualification_rows = await conn.fetch( + """ + SELECT COALESCE(qualification_status, 'pending') AS status, COUNT(*) AS count + FROM raw_posts + WHERE qualified_at >= $1 AND qualified_at < $2 + GROUP BY COALESCE(qualification_status, 'pending') + """, + start_utc, + end_utc, + ) + writer_ready = await conn.fetchval( + "SELECT COUNT(*) FROM raw_posts WHERE rewrite_status='ready' AND rewritten_at >= $1 AND rewritten_at < $2", + start_utc, + end_utc, + ) + review_now = await conn.fetchval( + """ + SELECT COUNT(*) + FROM raw_posts + WHERE rewrite_status='ready' + AND COALESCE(editorial_status, 'review')='review' + """ + ) + accepted_today = await conn.fetchval( + """ + SELECT COUNT(*) + FROM raw_posts + WHERE reviewed_at >= $1 AND reviewed_at < $2 + AND editorial_status='accepted' + """, + start_utc, + end_utc, + ) + rejected_today = await conn.fetchval( + """ + SELECT COUNT(*) + FROM raw_posts + WHERE reviewed_at >= $1 AND reviewed_at < $2 + AND editorial_status='rejected' + """, + start_utc, + end_utc, + ) + publication_rows = await conn.fetch( + """ + SELECT platform, status, COUNT(*) AS count + FROM post_publications + WHERE updated_at >= $1 AND updated_at < $2 + GROUP BY platform, status + ORDER BY platform, status + """, + start_utc, + end_utc, + ) + tg_queue = await conn.fetchval( + """ + SELECT COUNT(*) + FROM raw_posts rp + WHERE rp.status='storage_ready' + AND rp.rewrite_status='ready' + AND rp.editorial_status='accepted' + AND COALESCE(rp.publication_status, 'pending')='pending' + AND EXISTS ( + SELECT 1 FROM raw_post_media m + WHERE m.raw_post_id=rp.id + AND COALESCE(m.editor_hidden,FALSE)=FALSE + AND m.media_type IN ('photo','video') + AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') <> '' + ) + AND NOT EXISTS ( + SELECT 1 FROM raw_post_media m + WHERE m.raw_post_id=rp.id + AND COALESCE(m.editor_hidden,FALSE)=FALSE + AND m.media_type IN ('photo','video') + AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = '' + ) + """ + ) + vk_queue = await conn.fetchval( + """ + SELECT COUNT(*) + FROM raw_posts rp + LEFT JOIN post_publications pp ON pp.raw_post_id=rp.id AND pp.platform='vk' + WHERE rp.status='storage_ready' + AND rp.rewrite_status='ready' + AND COALESCE(rp.editorial_status, 'review') IN ('accepted', 'published', 'publish_failed') + AND COALESCE(pp.status, 'pending') IN ('pending', 'publish_failed') + AND EXISTS ( + SELECT 1 FROM raw_post_media rpm + WHERE rpm.raw_post_id=rp.id + AND COALESCE(rpm.editor_hidden, FALSE)=FALSE + AND COALESCE(rpm.editor_added, FALSE)=FALSE + AND ( + (rpm.media_type='photo' AND rpm.original_attachment_id LIKE 'photo%') + OR (rpm.media_type='video' AND rpm.original_attachment_id LIKE 'video%') + ) + ) + """ + ) + publish_failed_now = await conn.fetchval( + """ + SELECT COUNT(*) + FROM raw_posts + WHERE rewrite_status='ready' + AND COALESCE(editorial_status, '')='publish_failed' + """ + ) + workers = await conn.fetch( + """ + SELECT wc.name, wc.enabled, wh.heartbeat_at, wh.status + FROM worker_controls wc + LEFT JOIN worker_heartbeats wh ON wh.name=wc.name + ORDER BY wc.name + """ + ) + top_errors = await conn.fetch( + """ + SELECT platform, error, COUNT(*) AS count + FROM post_publications + WHERE status='publish_failed' + AND updated_at >= $1 AND updated_at < $2 + AND COALESCE(error, '') <> '' + GROUP BY platform, error + ORDER BY COUNT(*) DESC + LIMIT 5 + """, + start_utc, + end_utc, + ) + return { + "date": report_date, + "raw_collected": raw_collected, + "storage_ready": storage_ready, + "media_failed": media_failed, + "qualification": {row["status"]: int(row["count"]) for row in qualification_rows}, + "writer_ready": writer_ready, + "review_now": review_now, + "accepted_today": accepted_today, + "rejected_today": rejected_today, + "publications": {(row["platform"], row["status"]): int(row["count"]) for row in publication_rows}, + "tg_queue": tg_queue, + "vk_queue": vk_queue, + "publish_failed_now": publish_failed_now, + "workers": [dict(row) for row in workers], + "top_errors": [dict(row) for row in top_errors], + "now_utc": now_utc, + } + + def render(self, data: dict[str, Any]) -> str: + q = data["qualification"] + pubs = data["publications"] + workers_lines: list[str] = [] + for row in data["workers"]: + heartbeat_at = row.get("heartbeat_at") + is_stale = True + if heartbeat_at: + is_stale = (data["now_utc"] - heartbeat_at).total_seconds() > 600 + marker = "OK" if row.get("enabled") and heartbeat_at and not is_stale else ("OFF" if not row.get("enabled") else "STALE") + workers_lines.append(f"{marker} {row['name']}: {row.get('status') or '-'}") + errors_lines = [ + f"- {row['platform']}: {row['count']} x {str(row['error'])[:120]}" + for row in data["top_errors"] + ] + if not errors_lines: + errors_lines = ["- нет"] + report_date = data["date"].strftime("%d.%m.%Y") + return "\n".join( + [ + f"Ежедневный отчет N8 Parser за {report_date}", + "", + "Сбор:", + f"raw собрано: {fmt_int(data['raw_collected'])}", + f"storage ready за день: {fmt_int(data['storage_ready'])}", + f"media failed/link_only: {fmt_int(data['media_failed'])}", + "", + "AI:", + f"принято категоризатором: {fmt_int(q.get('accepted', 0))}", + f"отклонено категоризатором: {fmt_int(q.get('rejected', 0))}", + f"ошибки категоризатора: {fmt_int(q.get('failed', 0))}", + f"рерайтов готово: {fmt_int(data['writer_ready'])}", + "", + "Редакторская:", + f"принято редактором за день: {fmt_int(data['accepted_today'])}", + f"отклонено редактором за день: {fmt_int(data['rejected_today'])}", + f"на проверке сейчас: {fmt_int(data['review_now'])}", + f"ошибка публикации сейчас: {fmt_int(data['publish_failed_now'])}", + "", + "Публикации за день:", + f"TG опубликовано: {fmt_int(pubs.get(('tg', 'published'), 0))}", + f"VK опубликовано: {fmt_int(pubs.get(('vk', 'published'), 0))}", + f"TG ошибки: {fmt_int(pubs.get(('tg', 'publish_failed'), 0))}", + f"VK ошибки: {fmt_int(pubs.get(('vk', 'publish_failed'), 0))}", + "", + "Очередь сейчас:", + f"TG кандидатов: {fmt_int(data['tg_queue'])}", + f"VK кандидатов: {fmt_int(data['vk_queue'])}", + "", + "Ошибки дня:", + *errors_lines, + "", + "Воркеры:", + *workers_lines, + ] + ) + + async def send_text(self, text: str) -> None: + if not self.bot: + raise RuntimeError("bot is not initialized") + for recipient_id in self.recipients: + for chunk in split_chunks(text): + while True: + try: + await self.bot.send_message(recipient_id, chunk, disable_web_page_preview=True) + break + except TelegramRetryAfter as exc: + await asyncio.sleep(float(exc.retry_after) + 1) + + async def send_report(self, report_date: date, save_last_sent: bool = False) -> str: + await self.reload_settings() + data = await self.collect(report_date) + text = self.render(data) + await self.send_text(text) + if save_last_sent: + await self.save_last_sent_date(report_date) + await self.heartbeat.beat( + self.pool, + status="sent", + meta={"date": report_date.isoformat(), "recipients": self.recipients}, + force=True, + ) + return text + + async def due_report_date(self) -> date | None: + now = datetime.now(LOCAL_TZ) + hour, minute = [int(part) for part in self.report_time.split(":", 1)] + slot_start = datetime.combine(now.date(), time(hour=hour, minute=minute), tzinfo=LOCAL_TZ) + slot_end = slot_start + timedelta(minutes=10) + if not (slot_start <= now < slot_end): + return None + report_date = now.date() - timedelta(days=1) + last_sent = str(await fetch_setting("daily_report_last_sent_date", "") or "").strip() + if last_sent == report_date.isoformat(): + return None + return report_date + + async def run_once(self) -> bool: + await self.reload_settings() + enabled = await is_worker_enabled(self.pool, WORKER_DAILY_REPORT) + app_enabled = await fetch_bool_setting("daily_report_enabled", True) + if not enabled or not app_enabled: + await self.heartbeat.beat(self.pool, status="disabled", force=True) + return False + report_date = await self.due_report_date() + if not report_date: + await self.heartbeat.beat(self.pool, status="idle") + return False + await self.send_report(report_date, save_last_sent=True) + return True + + async def run_loop(self) -> None: + await self.init() + logger.info("{} started pid={}", WORKER_DAILY_REPORT, os.getpid()) + while True: + try: + await self.run_once() + except Exception as exc: + logger.exception("Daily report loop error: {}", exc) + if self.pool: + await self.heartbeat.beat(self.pool, status="error", meta={"error": str(exc)}, force=True) + await asyncio.sleep(self.interval_sec) + + +async def run_send_now(report_date: date | None = None) -> str: + worker = DailyReportWorker() + await worker.init() + try: + return await worker.send_report(report_date or datetime.now(LOCAL_TZ).date(), save_last_sent=False) + finally: + await worker.close() + + +async def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--send-now", action="store_true") + parser.add_argument("--date", help="Report date YYYY-MM-DD, default today in EKB for --send-now") + args = parser.parse_args() + logger.remove() + logger.add(sys.stdout, level=settings.log_level) + if args.send_now: + report_date = date.fromisoformat(args.date) if args.date else None + text = await run_send_now(report_date) + print(text) + return + worker = DailyReportWorker() + await worker.run_loop() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/vk_parser_app/workers/max_poster.py b/src/vk_parser_app/workers/max_poster.py new file mode 100644 index 0000000..1b4cace --- /dev/null +++ b/src/vk_parser_app/workers/max_poster.py @@ -0,0 +1,713 @@ +from __future__ import annotations + +import asyncio +import json +import mimetypes +import os +import re +import shutil +import sys +import tempfile +from datetime import datetime, time, timedelta +from pathlib import Path +from typing import Any +from urllib.parse import quote +from zoneinfo import ZoneInfo + +import aiohttp +from loguru import logger + +from ..config import settings +from ..constants import ( + PUBLICATION_STATUS_FAILED, + PUBLICATION_STATUS_PENDING, + PUBLICATION_STATUS_PUBLISHED, + WORKER_MAX_POSTER, +) +from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool +from ..heartbeat import HeartbeatReporter +from ..jobs import is_worker_enabled +from ..text_utils import build_publication_text + +LOCAL_TZ = ZoneInfo("Asia/Yekaterinburg") +MAX_MESSAGE_LIMIT = 4000 +MAX_MEDIA_ITEMS = 10 +MAX_VIDEO_BYTES = 250 * 1024 * 1024 +PROJECT_ROOT = Path(__file__).resolve().parents[3] +UPLOAD_ROOT = PROJECT_ROOT / "uploads" + + +class MAXAPIError(RuntimeError): + def __init__(self, message: str, status: int | None = None, code: str = "") -> None: + super().__init__(message) + self.status = status + self.code = code + + +def valid_time(value: str) -> bool: + return bool(re.fullmatch(r"(?:[01]\d|2[0-3]):[0-5]\d", str(value or "").strip())) + + +def normalize_schedule(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list): + value = [] + rows: list[dict[str, Any]] = [] + for idx, item in enumerate(value): + if not isinstance(item, dict): + continue + time_value = str(item.get("time") or "").strip() + if not valid_time(time_value): + continue + count = max(0, min(20, int(item.get("count") or 0))) + if count < 1: + continue + rows.append( + { + "id": str(item.get("id") or f"slot-{idx + 1}").strip()[:80], + "time": time_value, + "count": count, + "enabled": bool(item.get("enabled", True)), + } + ) + return sorted(rows, key=lambda row: row["time"]) + + +def split_message_chunks(text: str, limit: int) -> list[str]: + text = str(text or "").strip() + if not text: + return [] + chunks: list[str] = [] + while len(text) > limit: + split_at = text.rfind("\n", 0, limit) + if split_at < limit // 2: + split_at = text.rfind(" ", 0, limit) + if split_at < limit // 2: + split_at = limit + chunks.append(text[:split_at].strip()) + text = text[split_at:].strip() + if text: + chunks.append(text) + return chunks + + +def local_upload_path(url: str) -> Path | None: + raw = str(url or "").strip() + prefix = "/uploads/" + if not raw.startswith(prefix): + return None + path = (UPLOAD_ROOT / raw.removeprefix(prefix)).resolve() + try: + path.relative_to(UPLOAD_ROOT.resolve()) + except ValueError: + return None + return path if path.is_file() else None + + +class MAXAPIClient: + def __init__(self, token: str, base_url: str, timeout_sec: int, max_attempts: int, retry_backoff_max_sec: int) -> None: + self.token = token + self.base_url = base_url.rstrip("/") + self.timeout_sec = timeout_sec + self.max_attempts = max_attempts + self.retry_backoff_max_sec = retry_backoff_max_sec + self.session: aiohttp.ClientSession | None = None + + async def __aenter__(self) -> "MAXAPIClient": + timeout = aiohttp.ClientTimeout(total=self.timeout_sec) + self.session = aiohttp.ClientSession(timeout=timeout, headers={"Authorization": self.token}) + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + if self.session: + await self.session.close() + + async def request(self, method: str, path: str, **kwargs) -> dict[str, Any]: + if not self.session: + raise RuntimeError("MAX session is not initialized") + url = f"{self.base_url}{path}" + last_error = "" + for attempt in range(1, self.max_attempts + 1): + try: + async with self.session.request(method, url, **kwargs) as resp: + text = await resp.text() + try: + data = json.loads(text) if text else {} + except json.JSONDecodeError: + data = {"raw": text} + if 200 <= resp.status < 300: + return data + retry_after = resp.headers.get("Retry-After") + message = data.get("message") if isinstance(data, dict) else text + code = data.get("code") if isinstance(data, dict) else "" + last_error = f"MAX API {resp.status} {code}: {message or text}" + if resp.status == 429 and retry_after: + await asyncio.sleep(float(retry_after) + 0.5) + continue + if code == "attachment.not.ready" and attempt < self.max_attempts: + await asyncio.sleep(min(2 ** attempt, self.retry_backoff_max_sec)) + continue + if resp.status < 500: + raise MAXAPIError(last_error, resp.status, str(code or "")) + except Exception as exc: + last_error = str(exc) + if attempt >= self.max_attempts: + raise + await asyncio.sleep(min(2 ** attempt, self.retry_backoff_max_sec)) + raise RuntimeError(last_error or "MAX API request failed") + + async def send_message(self, chat_id: int, text: str, attachments: list[dict[str, Any]] | None = None) -> dict[str, Any]: + payload: dict[str, Any] = {"text": text, "notify": True} + if attachments: + payload["attachments"] = attachments + return await self.request("POST", f"/messages?chat_id={chat_id}", json=payload) + + async def get_video_info(self, token: str) -> dict[str, Any]: + return await self.request("GET", f"/videos/{quote(token, safe='')}") + + async def get_upload_url(self, media_type: str) -> dict[str, Any]: + upload_type = "video" if media_type == "video" else "image" + return await self.request("POST", f"/uploads?type={upload_type}") + + async def upload_file(self, upload_url: str, path: Path) -> dict[str, Any]: + if not self.session: + raise RuntimeError("MAX session is not initialized") + form = aiohttp.FormData() + content_type = mimetypes.guess_type(str(path))[0] or "application/octet-stream" + with path.open("rb") as fh: + form.add_field("data", fh, filename=path.name, content_type=content_type) + async with self.session.post(upload_url, data=form) as resp: + text = await resp.text() + try: + data = json.loads(text) if text else {} + except json.JSONDecodeError: + data = {"raw": text} + if resp.status < 200 or resp.status >= 300: + raise RuntimeError(f"MAX upload {resp.status}: {data}") + return data + + async def react(self, path_template: str, message_id: str, reaction: str) -> None: + path = path_template.format(message_id=message_id) + payload = {"reaction": reaction} + await self.request("POST", path, json=payload) + + +class MAXPoster: + def __init__(self) -> None: + self.pool = None + self.heartbeat = HeartbeatReporter(WORKER_MAX_POSTER, 30) + self.token = "" + self.chat_id = 0 + self.api_base_url = "https://platform-api2.max.ru" + self.schedule: list[dict[str, Any]] = [] + self.interval_sec = 60 + self.message_limit = MAX_MESSAGE_LIMIT + self.media_max_items = MAX_MEDIA_ITEMS + self.media_process_delay_sec = 3.0 + self.video_ready_attempts = 5 + self.video_ready_delay_sec = 10.0 + self.max_attempts = 3 + self.retry_backoff_max_sec = 30 + self.send_delay_sec = 1.0 + self.recent_window = 20 + self.category_repeat_penalty = 3.0 + self.source_repeat_penalty = 4.0 + self.auto_reaction_enabled = True + self.auto_reaction = "👍" + self.reaction_path_template = "/messages/{message_id}/reactions" + self.dry_run = False + + async def init(self) -> None: + self.pool = await get_pool() + + async def reload_settings(self) -> None: + self.token = str(await fetch_setting("max_poster_bot_token", "") or "").strip() + self.chat_id = int(str(await fetch_setting("max_poster_chat_id", "0") or "0").strip()) + self.api_base_url = str(await fetch_setting("max_poster_api_base_url", self.api_base_url) or self.api_base_url).strip() + self.schedule = normalize_schedule(await fetch_setting("max_poster_schedule_json", [])) + self.interval_sec = max(10, await fetch_int_setting("max_poster_interval_sec", 60)) + self.message_limit = max(512, min(MAX_MESSAGE_LIMIT, await fetch_int_setting("max_poster_message_limit", MAX_MESSAGE_LIMIT))) + self.media_max_items = max(0, min(MAX_MEDIA_ITEMS, await fetch_int_setting("max_poster_media_max_items", MAX_MEDIA_ITEMS))) + self.media_process_delay_sec = max(0.0, await fetch_float_setting("max_poster_media_process_delay_sec", 3.0)) + self.video_ready_attempts = max(1, await fetch_int_setting("max_poster_video_ready_attempts", 5)) + self.video_ready_delay_sec = max(1.0, await fetch_float_setting("max_poster_video_ready_delay_sec", 10.0)) + self.max_attempts = max(1, await fetch_int_setting("max_poster_max_attempts", 3)) + self.retry_backoff_max_sec = max(1, await fetch_int_setting("max_poster_retry_backoff_max_sec", 30)) + self.send_delay_sec = max(0.0, await fetch_float_setting("max_poster_send_delay_sec", 1.0)) + self.recent_window = max(1, await fetch_int_setting("max_poster_recent_window", 20)) + self.category_repeat_penalty = max(0.0, await fetch_float_setting("max_poster_category_repeat_penalty", 3.0)) + self.source_repeat_penalty = max(0.0, await fetch_float_setting("max_poster_source_repeat_penalty", 4.0)) + self.auto_reaction_enabled = await fetch_bool_setting("max_poster_auto_reaction_enabled", True) + self.auto_reaction = str(await fetch_setting("max_poster_auto_reaction", "👍") or "👍").strip() or "👍" + self.reaction_path_template = str(await fetch_setting("max_poster_reaction_path_template", self.reaction_path_template) or "").strip() + self.dry_run = await fetch_bool_setting("max_poster_dry_run", False) + if not self.token: + raise RuntimeError("max_poster_bot_token is empty") + if not self.chat_id: + raise RuntimeError("max_poster_chat_id is empty") + logger.info("MAX poster config: chat={} schedule={} dry_run={}", self.chat_id, self.schedule, self.dry_run) + + async def due_slots(self) -> list[dict[str, Any]]: + now = datetime.now(LOCAL_TZ) + today = now.date() + due: list[dict[str, Any]] = [] + for row in self.schedule: + if not row.get("enabled"): + continue + hour, minute = [int(part) for part in str(row["time"]).split(":", 1)] + slot_start = datetime.combine(today, time(hour=hour, minute=minute), tzinfo=LOCAL_TZ) + slot_end = slot_start + timedelta(minutes=10) + if not (slot_start <= now < slot_end): + continue + inserted = await self.pool.fetchrow( + """ + INSERT INTO publication_runs(poster, schedule_id, scheduled_for, scheduled_time, planned_count, status) + VALUES($1, $2, $3, $4, $5, 'started') + ON CONFLICT (poster, schedule_id, scheduled_for) DO NOTHING + RETURNING id + """, + "max", + row["id"], + today, + row["time"], + int(row["count"]), + ) + if inserted: + due.append({**row, "run_id": int(inserted["id"]), "date": today}) + return due + + async def load_recent(self) -> list[dict[str, Any]]: + rows = await self.pool.fetch( + """ + SELECT rp.final_category_tag, rp.rewrite_category_tag, rp.final_source_tag, rp.rewrite_source_tag + FROM post_publications pp + JOIN raw_posts rp ON rp.id=pp.raw_post_id + WHERE pp.platform='max' + AND pp.status=$1 + AND pp.target_id=$2 + ORDER BY pp.published_at DESC NULLS LAST, pp.id DESC + LIMIT $3::int + """, + PUBLICATION_STATUS_PUBLISHED, + str(self.chat_id), + self.recent_window, + ) + return [dict(row) for row in rows] + + async def load_candidates(self, limit: int) -> list[dict[str, Any]]: + rows = await self.pool.fetch( + """ + SELECT rp.*, s.name AS source_name, s.tag AS source_tag + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + LEFT JOIN post_publications pp ON pp.raw_post_id=rp.id AND pp.platform='max' + WHERE rp.status='storage_ready' + AND rp.rewrite_status='ready' + AND COALESCE(rp.editorial_status, 'review') IN ('accepted', 'published', 'publish_failed') + AND COALESCE(pp.status, $1) = $1 + AND EXISTS ( + SELECT 1 + FROM raw_post_media rpm + WHERE rpm.raw_post_id=rp.id + AND COALESCE(rpm.editor_hidden, FALSE)=FALSE + AND rpm.media_type IN ('photo', 'video') + AND COALESCE(rpm.original_url, '') <> '' + ) + ORDER BY COALESCE(rp.reviewed_at, rp.edited_at, rp.rewritten_at, rp.created_at) ASC, + rp.id ASC + LIMIT $2::int + """, + PUBLICATION_STATUS_PENDING, + max(limit, 50), + ) + return [dict(row) for row in rows] + + def choose_posts(self, candidates: list[dict[str, Any]], recent: list[dict[str, Any]], count: int) -> list[dict[str, Any]]: + selected: list[dict[str, Any]] = [] + recent_categories = [str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "") for row in recent] + recent_sources = [str(row.get("final_source_tag") or row.get("rewrite_source_tag") or "") for row in recent] + pool = list(candidates) + while pool and len(selected) < count: + best_idx = 0 + selected_categories = [str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "") for row in selected] + selected_sources = [str(row.get("final_source_tag") or row.get("rewrite_source_tag") or row.get("source_tag") or "") for row in selected] + previous_category = selected_categories[-1] if selected_categories else (recent_categories[0] if recent_categories else "") + previous_source = selected_sources[-1] if selected_sources else (recent_sources[0] if recent_sources else "") + best_rank: tuple[int, float, int] | None = None + for idx, post in enumerate(pool): + category = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or "") + source = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "") + immediate_repeats = int(bool(category) and category == previous_category) + int(bool(source) and source == previous_source) + repeat_penalty = ( + (recent_categories.count(category) + selected_categories.count(category)) * self.category_repeat_penalty + + (recent_sources.count(source) + selected_sources.count(source)) * self.source_repeat_penalty + ) + rank = (immediate_repeats, repeat_penalty, idx) + if best_rank is None or rank < best_rank: + best_idx = idx + best_rank = rank + selected.append(pool.pop(best_idx)) + return selected + + async def load_media(self, raw_post_id: int) -> list[dict[str, Any]]: + rows = await self.pool.fetch( + """ + SELECT * + FROM raw_post_media + WHERE raw_post_id=$1 + AND COALESCE(editor_hidden, FALSE)=FALSE + AND media_type IN ('photo', 'video') + AND COALESCE(original_url, '') <> '' + ORDER BY sort_order ASC, id ASC + LIMIT $2::int + """, + raw_post_id, + self.media_max_items, + ) + return [dict(row) for row in rows] + + def build_text(self, post: dict[str, Any]) -> str: + category_tag = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or post.get("final_category") or post.get("rewrite_category") or "") + source_tag = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "") + return build_publication_text(post.get("final_text") or post.get("rewritten_text") or "", category_tag, source_tag) + + async def download_media_to_temp(self, session: aiohttp.ClientSession, item: dict[str, Any]) -> Path | None: + media_type = str(item.get("media_type") or "") + file_id = str(item.get("tg_file_id") or item.get("storage_attachment_id") or "").strip() + if media_type == "video" and file_id: + return await self.download_telegram_file_to_temp(session, file_id, ".mp4") + url = str(item.get("original_url") or "").strip() + local_path = local_upload_path(url) + if local_path: + return local_path + if not url.startswith(("http://", "https://")): + return None + suffix = Path(url.split("?", 1)[0]).suffix[:12] or (".mp4" if item.get("media_type") == "video" else ".jpg") + tmp = tempfile.NamedTemporaryFile(prefix="max-poster-", suffix=suffix, delete=False) + path = Path(tmp.name) + tmp.close() + total = 0 + try: + async with session.get(url) as resp: + resp.raise_for_status() + with path.open("wb") as fh: + async for chunk in resp.content.iter_chunked(1024 * 512): + total += len(chunk) + if total > MAX_VIDEO_BYTES: + raise RuntimeError("media file is larger than MAX video limit") + fh.write(chunk) + return path + except Exception: + path.unlink(missing_ok=True) + raise + + async def download_telegram_file_to_temp(self, session: aiohttp.ClientSession, file_id: str, suffix: str) -> Path: + token = settings.tg_bot_token + if not token: + raise RuntimeError("TG_BOT_TOKEN is empty; cannot download stored video") + api_base = str(await fetch_setting("local_bot_api_url", settings.local_bot_api_url) or "").strip().rstrip("/") + if not api_base: + api_base = "https://api.telegram.org" + get_file_url = f"{api_base}/bot{token}/getFile" + async with session.post(get_file_url, json={"file_id": file_id}) as resp: + data = await resp.json(content_type=None) + if not data.get("ok") or not isinstance(data.get("result"), dict): + raise RuntimeError(f"Telegram getFile failed: {data}") + file_path = str(data["result"].get("file_path") or "").strip() + if not file_path: + raise RuntimeError(f"Telegram getFile did not return file_path: {data}") + local_file = Path(file_path) + if local_file.is_absolute() and local_file.is_file(): + if local_file.stat().st_size > MAX_VIDEO_BYTES: + raise RuntimeError("telegram local video file is larger than MAX video limit") + tmp = tempfile.NamedTemporaryFile(prefix="max-poster-tg-", suffix=suffix, delete=False) + path = Path(tmp.name) + tmp.close() + try: + shutil.copyfile(local_file, path) + return path + except Exception: + path.unlink(missing_ok=True) + raise + download_url = f"{api_base}/file/bot{token}/{file_path}" + tmp = tempfile.NamedTemporaryFile(prefix="max-poster-tg-", suffix=suffix, delete=False) + path = Path(tmp.name) + tmp.close() + total = 0 + try: + async with session.get(download_url) as resp: + resp.raise_for_status() + with path.open("wb") as fh: + async for chunk in resp.content.iter_chunked(1024 * 1024): + total += len(chunk) + if total > MAX_VIDEO_BYTES: + raise RuntimeError("telegram video file is larger than MAX video limit") + fh.write(chunk) + return path + except Exception: + path.unlink(missing_ok=True) + raise + + async def upload_media(self, client: MAXAPIClient, item: dict[str, Any]) -> dict[str, Any] | None: + media_type = str(item.get("media_type") or "") + if media_type not in {"photo", "video"}: + return None + temp_path: Path | None = None + downloaded = False + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=180)) as session: + temp_path = await self.download_media_to_temp(session, item) + downloaded = bool(temp_path and not local_upload_path(str(item.get("original_url") or ""))) + if not temp_path: + return None + try: + upload_info = await client.get_upload_url("video" if media_type == "video" else "image") + upload_url = str(upload_info.get("url") or "").strip() + if not upload_url: + raise RuntimeError(f"MAX upload URL is empty: {upload_info}") + payload = await client.upload_file(upload_url, temp_path) + if media_type == "video" and upload_info.get("token") and not payload.get("token"): + payload["token"] = upload_info["token"] + has_image_token = bool(payload.get("photos")) if isinstance(payload.get("photos"), dict) else False + if not payload.get("token") and not has_image_token: + raise RuntimeError(f"MAX upload did not return token: {payload}") + return {"type": "video" if media_type == "video" else "image", "payload": payload} + finally: + if downloaded and temp_path: + temp_path.unlink(missing_ok=True) + + def message_id_from_response(self, data: dict[str, Any]) -> str: + candidates = [ + data.get("id"), + data.get("mid"), + (data.get("message") or {}).get("id") if isinstance(data.get("message"), dict) else None, + (data.get("message") or {}).get("mid") if isinstance(data.get("message"), dict) else None, + ] + message = data.get("message") if isinstance(data.get("message"), dict) else {} + body = message.get("body") if isinstance(message.get("body"), dict) else {} + candidates.extend([body.get("id"), body.get("mid"), body.get("message_id")]) + return next((str(value) for value in candidates if value), "") + + async def send_message_waiting_for_media( + self, + client: MAXAPIClient, + text: str, + attachments: list[dict[str, Any]], + ) -> dict[str, Any]: + has_video = any(item.get("type") == "video" for item in attachments) + attempts = self.video_ready_attempts if has_video else 1 + delay = self.video_ready_delay_sec + for attempt in range(1, attempts + 1): + try: + return await client.send_message(self.chat_id, text, attachments=attachments or None) + except MAXAPIError as exc: + if exc.code != "attachment.not.ready" or attempt >= attempts: + raise + logger.warning( + "MAX attachment is not ready, retry send in {}s attempt={}/{}", + delay, + attempt + 1, + attempts, + ) + await asyncio.sleep(delay) + delay = min(delay * 2, self.retry_backoff_max_sec * 4) + raise RuntimeError("MAX attachment readiness retry exhausted") + + def video_tokens(self, attachments: list[dict[str, Any]]) -> list[str]: + tokens: list[str] = [] + for item in attachments: + if item.get("type") != "video": + continue + payload = item.get("payload") if isinstance(item.get("payload"), dict) else {} + token = str(payload.get("token") or "").strip() + if token: + tokens.append(token) + return tokens + + async def wait_for_videos(self, client: MAXAPIClient, attachments: list[dict[str, Any]]) -> None: + tokens = self.video_tokens(attachments) + if not tokens: + return + delay = self.video_ready_delay_sec + for attempt in range(1, self.video_ready_attempts + 1): + pending: list[str] = [] + for token in tokens: + info = await client.get_video_info(token) + if not info.get("urls"): + pending.append(token) + if not pending: + return + if attempt >= self.video_ready_attempts: + raise RuntimeError(f"MAX video is not ready after {attempt} attempts") + logger.warning( + "MAX video is not ready, retry info in {}s attempt={}/{} pending={}", + delay, + attempt + 1, + self.video_ready_attempts, + len(pending), + ) + await asyncio.sleep(delay) + delay = min(delay * 2, self.retry_backoff_max_sec * 4) + + async def send_post(self, post: dict[str, Any]) -> tuple[list[str], str | None, list[dict[str, Any]], str | None]: + text = self.build_text(post) + chunks = split_message_chunks(text, self.message_limit) + media_rows = await self.load_media(int(post["id"])) if self.media_max_items else [] + message_ids: list[str] = [] + attachments: list[dict[str, Any]] = [] + first_url: str | None = None + reaction_error: str | None = None + async with MAXAPIClient( + self.token, + self.api_base_url, + timeout_sec=max(30, await fetch_int_setting("max_poster_timeout_sec", 120)), + max_attempts=self.max_attempts, + retry_backoff_max_sec=self.retry_backoff_max_sec, + ) as client: + for item in media_rows: + try: + attachment = await self.upload_media(client, item) + if attachment: + attachments.append(attachment) + except Exception as exc: + logger.warning("MAX media skipped raw_post={} media_id={}: {}", post["id"], item.get("id"), exc) + if len(attachments) >= self.media_max_items: + break + if self.media_process_delay_sec and attachments: + await asyncio.sleep(self.media_process_delay_sec) + if self.dry_run: + return ["dry-run"], None, attachments, None + await self.wait_for_videos(client, attachments) + first_text = chunks[0] if chunks else "" + first = await self.send_message_waiting_for_media(client, first_text, attachments) + message = first.get("message") if isinstance(first.get("message"), dict) else first + first_url = message.get("url") if isinstance(message, dict) else None + first_message_id = self.message_id_from_response(first) + if first_message_id: + message_ids.append(first_message_id) + if self.auto_reaction_enabled and self.reaction_path_template: + try: + await client.react(self.reaction_path_template, first_message_id, self.auto_reaction) + except Exception as exc: + reaction_error = str(exc) + logger.warning("MAX auto reaction failed message={}: {}", first_message_id, exc) + for chunk in chunks[1:]: + if self.send_delay_sec: + await asyncio.sleep(self.send_delay_sec) + result = await client.send_message(self.chat_id, chunk) + message_id = self.message_id_from_response(result) + if message_id: + message_ids.append(message_id) + return message_ids, first_url, attachments, reaction_error + + async def upsert_publication( + self, + post_id: int, + status: str, + external_id: str | None = None, + url: str | None = None, + error: str | None = None, + attachments: list[dict[str, Any]] | None = None, + ) -> None: + await self.pool.execute( + """ + INSERT INTO post_publications( + raw_post_id, platform, status, target_id, external_id, url, error, attachments_json, published_at, updated_at + ) + VALUES($1, 'max', $2, $3, $4, $5, $6, $7::jsonb, CASE WHEN $2=$8 THEN NOW() ELSE NULL END, NOW()) + ON CONFLICT (raw_post_id, platform) DO UPDATE + SET status=EXCLUDED.status, + target_id=EXCLUDED.target_id, + external_id=EXCLUDED.external_id, + url=EXCLUDED.url, + error=EXCLUDED.error, + attachments_json=EXCLUDED.attachments_json, + published_at=CASE WHEN EXCLUDED.status=$8 THEN COALESCE(EXCLUDED.published_at, NOW()) ELSE post_publications.published_at END, + updated_at=NOW() + """, + post_id, + status, + str(self.chat_id), + external_id, + url, + error[:1000] if error else None, + json.dumps(attachments or [], ensure_ascii=False), + PUBLICATION_STATUS_PUBLISHED, + ) + + async def process_slot(self, slot: dict[str, Any]) -> int: + count = int(slot["count"]) + recent = await self.load_recent() + candidates = await self.load_candidates(max(50, count * 10)) + posts = self.choose_posts(candidates, recent, count) + published = 0 + for post in posts: + await self.heartbeat.beat(self.pool, status="publishing", meta={"post_id": int(post["id"]), "slot": slot["id"]}, force=True) + try: + message_ids, url, attachments, reaction_error = await self.send_post(post) + await self.upsert_publication( + int(post["id"]), + PUBLICATION_STATUS_PUBLISHED, + ",".join(message_ids), + url, + error=None, + attachments=attachments, + ) + published += 1 + logger.info("MAX published raw_post={} messages={} attachments={}", post["id"], message_ids, len(attachments)) + except Exception as exc: + await self.upsert_publication(int(post["id"]), PUBLICATION_STATUS_FAILED, error=str(exc)) + logger.exception("MAX publish failed raw_post={}: {}", post["id"], exc) + if self.send_delay_sec: + await asyncio.sleep(self.send_delay_sec) + await self.pool.execute( + """ + UPDATE publication_runs + SET published_count=$2, + status=$3, + error=$4, + updated_at=NOW() + WHERE id=$1 + """, + int(slot["run_id"]), + published, + "done" if published == count else "partial", + None if published == count else f"published {published} of {count}", + ) + return published + + async def run_once(self) -> bool: + await self.reload_settings() + enabled = await is_worker_enabled(self.pool, WORKER_MAX_POSTER) + app_enabled = await fetch_bool_setting("max_poster_enabled", False) + if not enabled or not app_enabled: + await self.heartbeat.beat(self.pool, status="disabled", force=True) + return False + slots = await self.due_slots() + if not slots: + await self.heartbeat.beat(self.pool, status="idle") + return False + total = 0 + for slot in slots: + total += await self.process_slot(slot) + await self.heartbeat.beat(self.pool, status="idle", meta={"published": total}, force=True) + return bool(total) + + async def run_loop(self) -> None: + await self.init() + logger.info("{} started pid={}", WORKER_MAX_POSTER, os.getpid()) + while True: + try: + await self.run_once() + except Exception as exc: + logger.exception("MAX poster loop error: {}", exc) + await asyncio.sleep(self.interval_sec) + + +async def main() -> None: + logger.remove() + logger.add(sys.stdout, level=settings.log_level) + worker = MAXPoster() + await worker.run_loop() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/vk_parser_app/workers/site_poster.py b/src/vk_parser_app/workers/site_poster.py new file mode 100644 index 0000000..927e269 --- /dev/null +++ b/src/vk_parser_app/workers/site_poster.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import asyncio +import os +import sys +from pathlib import Path + +from loguru import logger + +from ..config import settings +from ..constants import WORKER_SITE_POSTER +from ..db import fetch_bool_setting, fetch_int_setting, fetch_setting, get_pool +from ..heartbeat import HeartbeatReporter +from ..jobs import is_worker_enabled + + +class SitePoster: + def __init__(self) -> None: + self.pool = None + self.heartbeat = HeartbeatReporter(WORKER_SITE_POSTER, 30) + self.project_root = Path(__file__).resolve().parents[3] + self.interval_sec = 300 + self.photo_batch_size = 10 + self.video_batch_size = 1 + self.ghost_url = "https://f-n8.ru" + self.key_file = "/opt/vk-parser/.site-poster-key" + + async def init(self) -> None: + self.pool = await get_pool() + + async def reload_settings(self) -> None: + self.interval_sec = max(30, await fetch_int_setting("site_poster_interval_sec", 300)) + self.photo_batch_size = max(0, await fetch_int_setting("site_poster_photo_batch_size", 10)) + self.video_batch_size = max(0, await fetch_int_setting("site_poster_video_batch_size", 1)) + self.ghost_url = str(await fetch_setting("site_poster_ghost_url", "https://f-n8.ru") or "https://f-n8.ru").rstrip("/") + self.key_file = str(await fetch_setting("site_poster_key_file", "/opt/vk-parser/.site-poster-key") or "").strip() + if not self.key_file: + raise RuntimeError("site_poster_key_file is empty") + + async def publish_mode(self, flag: str, limit: int) -> tuple[int, int]: + if limit < 1: + return 0, 0 + command = [ + sys.executable, + str(self.project_root / "scripts" / "publish_site_batch.py"), + flag, + "--limit", + str(limit), + "--ghost-url", + self.ghost_url, + "--key-file", + self.key_file, + ] + env = os.environ.copy() + env["PYTHONPATH"] = str(self.project_root / "src") + env["PYTHONUNBUFFERED"] = "1" + process = await asyncio.create_subprocess_exec( + *command, + cwd=str(self.project_root), + env=env, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + ) + published = 0 + failed = 0 + assert process.stdout is not None + while True: + try: + raw_line = await asyncio.wait_for(process.stdout.readline(), timeout=20) + except TimeoutError: + await self.heartbeat.beat( + self.pool, + status="publishing", + meta={"mode": flag, "published": published, "failed": failed}, + force=True, + ) + continue + if not raw_line: + break + line = raw_line.decode("utf-8", errors="replace").strip() + if not line: + continue + if line.startswith("PUBLISHED "): + published += 1 + logger.info("Site poster: {}", line) + elif line.startswith("FAILED "): + failed += 1 + logger.error("Site poster: {}", line) + elif line not in {"NO_READY_POSTS", "ALREADY_RUNNING"}: + logger.debug("Site poster child: {}", line) + await self.heartbeat.beat( + self.pool, + status="publishing", + meta={"mode": flag, "published": published, "failed": failed}, + ) + return_code = await process.wait() + if return_code != 0: + raise RuntimeError(f"site publisher exited with code {return_code} for {flag}") + return published, failed + + async def run_once(self) -> bool: + await self.reload_settings() + worker_enabled = await is_worker_enabled(self.pool, WORKER_SITE_POSTER) + app_enabled = await fetch_bool_setting("site_poster_enabled", False) + if not worker_enabled or not app_enabled: + await self.heartbeat.beat(self.pool, status="disabled", force=True) + return False + await self.heartbeat.beat(self.pool, status="checking", force=True) + photo_published, photo_failed = await self.publish_mode("--all-ready", self.photo_batch_size) + video_published, video_failed = await self.publish_mode("--all-ready-videos", self.video_batch_size) + published = photo_published + video_published + failed = photo_failed + video_failed + await self.heartbeat.beat( + self.pool, + status="idle" if not failed else "error", + meta={"published": published, "failed": failed}, + force=True, + ) + return bool(published) + + async def run_loop(self) -> None: + await self.init() + logger.info("{} started pid={}", WORKER_SITE_POSTER, os.getpid()) + while True: + try: + await self.run_once() + except Exception as exc: + logger.exception("Site poster loop error: {}", exc) + await self.heartbeat.beat(self.pool, status="error", meta={"error": str(exc)[:500]}, force=True) + await asyncio.sleep(self.interval_sec) + + +async def main() -> None: + logger.remove() + logger.add(sys.stdout, level=settings.log_level) + worker = SitePoster() + await worker.run_loop() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/vk_parser_app/workers/tg_poster.py b/src/vk_parser_app/workers/tg_poster.py new file mode 100644 index 0000000..ab89c66 --- /dev/null +++ b/src/vk_parser_app/workers/tg_poster.py @@ -0,0 +1,535 @@ +from __future__ import annotations + +import asyncio +import json +import os +import re +import sys +from datetime import datetime, time, timedelta +from pathlib import Path +from typing import Any +from zoneinfo import ZoneInfo + +from aiogram import Bot +from aiogram.exceptions import TelegramRetryAfter +from aiogram.types import FSInputFile, InputMediaPhoto, InputMediaVideo +from loguru import logger + +from ..config import settings +from ..constants import ( + PUBLICATION_STATUS_FAILED, + PUBLICATION_STATUS_PENDING, + PUBLICATION_STATUS_PUBLISHED, + WORKER_TG_POSTER, +) +from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool +from ..heartbeat import HeartbeatReporter +from ..jobs import is_worker_enabled +from ..text_utils import build_publication_text + +LOCAL_TZ = ZoneInfo("Asia/Yekaterinburg") +MAX_MEDIA_GROUP = 10 +PROJECT_ROOT = Path(__file__).resolve().parents[3] +UPLOAD_ROOT = PROJECT_ROOT / "uploads" + + +def parse_topic(value: str) -> tuple[int, int | None]: + raw = str(value or "").strip() + if not raw: + raise RuntimeError("tg_poster_chat_id is empty") + if ":" in raw: + chat_id, thread_id = raw.split(":", 1) + return int(chat_id), int(thread_id) + return int(raw), None + + +def valid_time(value: str) -> bool: + return bool(re.fullmatch(r"(?:[01]\d|2[0-3]):[0-5]\d", str(value or "").strip())) + + +def split_message_chunks(text: str, limit: int) -> list[str]: + text = str(text or "").strip() + if not text: + return [] + chunks: list[str] = [] + while len(text) > limit: + split_at = text.rfind("\n", 0, limit) + if split_at < limit // 2: + split_at = text.rfind(" ", 0, limit) + if split_at < limit // 2: + split_at = limit + chunks.append(text[:split_at].strip()) + text = text[split_at:].strip() + if text: + chunks.append(text) + return chunks + + +def tg_message_url(chat_id: int, message_id: int) -> str: + chat = str(abs(int(chat_id))) + if chat.startswith("100"): + chat = chat[3:] + return f"https://t.me/c/{chat}/{message_id}" + + +def local_upload_path(url: str) -> Path | None: + raw = str(url or "").strip() + prefix = "/uploads/" + if not raw.startswith(prefix): + return None + path = (UPLOAD_ROOT / raw.removeprefix(prefix)).resolve() + try: + path.relative_to(UPLOAD_ROOT.resolve()) + except ValueError: + return None + return path if path.is_file() else None + + +class TelegramPoster: + def __init__(self) -> None: + self.pool = None + self.bot: Bot | None = None + self.bot_token = "" + self.chat_id: int | None = None + self.thread_id: int | None = None + self.heartbeat = HeartbeatReporter(WORKER_TG_POSTER, 30) + self.schedule: list[dict[str, Any]] = [] + self.media_group_max_items = MAX_MEDIA_GROUP + self.caption_limit = 1024 + self.message_limit = 4096 + self.max_attempts = 3 + self.retry_backoff_max_sec = 30 + self.send_delay_sec = 1.0 + self.overflow_caption = "⬇️ Описание" + self.recent_window = 20 + self.category_repeat_penalty = 3.0 + self.source_repeat_penalty = 4.0 + + async def init(self) -> None: + self.pool = await get_pool() + await self.reload_settings() + + async def reload_settings(self) -> None: + token = str(await fetch_setting("tg_poster_bot_token", "") or "").strip() or settings.tg_bot_token + if not token: + raise RuntimeError("tg_poster_bot_token and TG_BOT_TOKEN are empty") + if token != self.bot_token: + if self.bot: + await self.bot.session.close() + self.bot = Bot(token=token) + self.bot_token = token + self.chat_id, self.thread_id = parse_topic(str(await fetch_setting("tg_poster_chat_id", "-1003712724327") or "")) + self.schedule = self.normalize_schedule(await fetch_setting("tg_poster_schedule_json", [])) + self.media_group_max_items = max(1, min(MAX_MEDIA_GROUP, await fetch_int_setting("tg_poster_media_group_max_items", 10))) + self.caption_limit = max(128, min(1024, await fetch_int_setting("tg_poster_caption_limit", 1024))) + self.message_limit = max(512, min(4096, await fetch_int_setting("tg_poster_message_limit", 4096))) + self.max_attempts = max(1, await fetch_int_setting("tg_poster_max_attempts", 3)) + self.retry_backoff_max_sec = max(1, await fetch_int_setting("tg_poster_retry_backoff_max_sec", 30)) + self.send_delay_sec = max(0.0, await fetch_float_setting("tg_poster_send_delay_sec", 1.0)) + self.overflow_caption = str(await fetch_setting("tg_poster_text_overflow_caption", self.overflow_caption) or self.overflow_caption) + self.recent_window = max(1, await fetch_int_setting("tg_poster_recent_window", 20)) + self.category_repeat_penalty = max(0.0, await fetch_float_setting("tg_poster_category_repeat_penalty", 3.0)) + self.source_repeat_penalty = max(0.0, await fetch_float_setting("tg_poster_source_repeat_penalty", 4.0)) + logger.info("TG poster config: chat={} schedule={} caption_limit={}", self.chat_id, self.schedule, self.caption_limit) + + async def close(self) -> None: + if self.bot: + await self.bot.session.close() + + def normalize_schedule(self, value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list): + value = [] + rows: list[dict[str, Any]] = [] + for idx, item in enumerate(value): + if not isinstance(item, dict): + continue + time_value = str(item.get("time") or "").strip() + if not valid_time(time_value): + continue + count = max(0, min(20, int(item.get("count") or 0))) + if count < 1: + continue + rows.append( + { + "id": str(item.get("id") or f"slot-{idx + 1}").strip()[:80], + "time": time_value, + "count": count, + "enabled": bool(item.get("enabled", True)), + } + ) + return sorted(rows, key=lambda row: row["time"]) + + def chat_kwargs(self) -> dict[str, int]: + kwargs = {"chat_id": int(self.chat_id)} + if self.thread_id: + kwargs["message_thread_id"] = int(self.thread_id) + return kwargs + + async def tg_retry(self, fn): + for attempt in range(1, self.max_attempts + 1): + try: + return await fn() + except TelegramRetryAfter as exc: + delay = float(exc.retry_after) + 0.5 + logger.warning("Telegram flood control, sleep {}s", delay) + await asyncio.sleep(delay) + except Exception as exc: + logger.warning("Telegram post error without retry to avoid duplicate send: {}", exc) + raise + raise RuntimeError("telegram retry exhausted") + + async def due_slots(self) -> list[dict[str, Any]]: + now = datetime.now(LOCAL_TZ) + today = now.date() + due: list[dict[str, Any]] = [] + for row in self.schedule: + if not row.get("enabled"): + continue + hour, minute = [int(part) for part in str(row["time"]).split(":", 1)] + slot_start = datetime.combine(today, time(hour=hour, minute=minute), tzinfo=LOCAL_TZ) + slot_end = slot_start + timedelta(minutes=10) + if not (slot_start <= now < slot_end): + continue + inserted = await self.pool.fetchrow( + """ + INSERT INTO publication_runs(poster, schedule_id, scheduled_for, scheduled_time, planned_count, status) + VALUES($1, $2, $3, $4, $5, 'started') + ON CONFLICT (poster, schedule_id, scheduled_for) DO NOTHING + RETURNING id + """, + "tg", + row["id"], + today, + row["time"], + int(row["count"]), + ) + if inserted: + due.append({**row, "run_id": int(inserted["id"]), "date": today}) + return due + + async def load_recent(self) -> list[dict[str, Any]]: + rows = await self.pool.fetch( + """ + SELECT final_category_tag, rewrite_category_tag, final_source_tag, rewrite_source_tag + FROM raw_posts + WHERE publication_status=$1 + AND tg_publication_chat_id=$2 + ORDER BY published_at DESC NULLS LAST, id DESC + LIMIT $3::int + """, + PUBLICATION_STATUS_PUBLISHED, + self.chat_id, + self.recent_window, + ) + return [dict(row) for row in rows] + + async def load_candidates(self, limit: int) -> list[dict[str, Any]]: + rows = await self.pool.fetch( + """ + SELECT rp.*, s.name AS source_name, s.tag AS source_tag + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + WHERE rp.status='storage_ready' + AND rp.rewrite_status='ready' + AND rp.editorial_status='accepted' + AND COALESCE(rp.publication_status, 'pending') = $1 + AND EXISTS ( + SELECT 1 + FROM raw_post_media rpm + WHERE rpm.raw_post_id=rp.id + AND COALESCE(rpm.editor_hidden, FALSE)=FALSE + AND rpm.media_type IN ('photo', 'video') + AND COALESCE(rpm.tg_file_id, rpm.storage_attachment_id, '') <> '' + ) + AND NOT EXISTS ( + SELECT 1 + FROM raw_post_media rpm + WHERE rpm.raw_post_id=rp.id + AND COALESCE(rpm.editor_hidden, FALSE)=FALSE + AND rpm.media_type IN ('photo', 'video') + AND COALESCE(rpm.tg_file_id, rpm.storage_attachment_id, '') = '' + ) + ORDER BY COALESCE(rp.reviewed_at, rp.edited_at, rp.rewritten_at, rp.created_at) ASC, + rp.id ASC + LIMIT $2::int + """, + PUBLICATION_STATUS_PENDING, + max(limit, 50), + ) + return [dict(row) for row in rows] + + def choose_posts(self, candidates: list[dict[str, Any]], recent: list[dict[str, Any]], count: int) -> list[dict[str, Any]]: + selected: list[dict[str, Any]] = [] + recent_categories = [ + str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "") + for row in recent + ] + recent_sources = [ + str(row.get("final_source_tag") or row.get("rewrite_source_tag") or "") + for row in recent + ] + pool = list(candidates) + while pool and len(selected) < count: + best_idx = 0 + selected_categories = [str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "") for row in selected] + selected_sources = [str(row.get("final_source_tag") or row.get("rewrite_source_tag") or row.get("source_tag") or "") for row in selected] + previous_category = selected_categories[-1] if selected_categories else (recent_categories[0] if recent_categories else "") + previous_source = selected_sources[-1] if selected_sources else (recent_sources[0] if recent_sources else "") + best_rank: tuple[int, float, int] | None = None + for idx, post in enumerate(pool): + category = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or "") + source = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "") + immediate_repeats = int(bool(category) and category == previous_category) + int(bool(source) and source == previous_source) + repeat_penalty = ( + (recent_categories.count(category) + selected_categories.count(category)) * self.category_repeat_penalty + + (recent_sources.count(source) + selected_sources.count(source)) * self.source_repeat_penalty + ) + rank = (immediate_repeats, repeat_penalty, idx) + if best_rank is None or rank < best_rank: + best_idx = idx + best_rank = rank + selected.append(pool.pop(best_idx)) + return selected + + async def load_media(self, raw_post_id: int) -> list[dict[str, Any]]: + rows = await self.pool.fetch( + """ + SELECT * + FROM raw_post_media + WHERE raw_post_id=$1 + AND COALESCE(editor_hidden, FALSE)=FALSE + AND media_type IN ('photo', 'video') + ORDER BY sort_order ASC, id ASC + """, + raw_post_id, + ) + return [dict(row) for row in rows] + + def media_value(self, item: dict[str, Any]) -> Any | None: + stored = item.get("tg_file_id") or item.get("storage_attachment_id") + if stored: + return str(stored) + url = str(item.get("original_url") or "").strip() + path = local_upload_path(url) + if path: + return FSInputFile(path) + return None + + def build_text(self, post: dict[str, Any]) -> str: + category_tag = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or post.get("final_category") or post.get("rewrite_category") or "") + source_tag = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "") + return build_publication_text(post.get("final_text") or post.get("rewritten_text") or "", category_tag, source_tag) + + async def send_text(self, text: str) -> list[int]: + message_ids: list[int] = [] + for chunk in split_message_chunks(text, self.message_limit): + message = await self.tg_retry(lambda text=chunk: self.bot.send_message(text=text, **self.chat_kwargs())) + message_ids.append(int(message.message_id)) + if self.send_delay_sec: + await asyncio.sleep(self.send_delay_sec) + return message_ids + + async def send_media_items(self, items: list[dict[str, Any]], caption: str | None = None) -> list[int]: + if not items: + return [] + if len(items) == 1: + item = items[0] + if item["type"] == "photo": + message = await self.tg_retry( + lambda: self.bot.send_photo(photo=item["media"], caption=caption, **self.chat_kwargs()) + ) + else: + message = await self.tg_retry( + lambda: self.bot.send_video(video=item["media"], caption=caption, **self.chat_kwargs()) + ) + return [int(message.message_id)] + group = [] + for idx, item in enumerate(items): + media_caption = caption if idx == 0 else None + if item["type"] == "photo": + group.append(InputMediaPhoto(media=item["media"], caption=media_caption)) + else: + group.append(InputMediaVideo(media=item["media"], caption=media_caption)) + messages = await self.tg_retry(lambda: self.bot.send_media_group(media=group, **self.chat_kwargs())) + return [int(message.message_id) for message in messages] + + async def send_post(self, post: dict[str, Any]) -> list[int]: + text = self.build_text(post) + media_rows = await self.load_media(int(post["id"])) + media_items = [] + for item in media_rows: + media = self.media_value(item) + if media is None: + continue + media_type = str(item.get("media_type") or "") + if media_type == "photo": + media_items.append({"type": "photo", "media": media}) + elif media_type == "video": + media_items.append({"type": "video", "media": media}) + message_ids: list[int] = [] + if not media_items: + return await self.send_text(text) + + first_caption = text if len(text) <= self.caption_limit else self.overflow_caption[: self.caption_limit] + first = media_items[: self.media_group_max_items] + message_ids.extend(await self.send_media_items(first, first_caption)) + if self.send_delay_sec: + await asyncio.sleep(self.send_delay_sec) + + rest = media_items[self.media_group_max_items :] + for start in range(0, len(rest), self.media_group_max_items): + chunk = rest[start : start + self.media_group_max_items] + message_ids.extend(await self.send_media_items(chunk)) + if self.send_delay_sec: + await asyncio.sleep(self.send_delay_sec) + + if len(text) > self.caption_limit: + message_ids.extend(await self.send_text(text)) + return message_ids + + async def mark_published(self, post_id: int, message_ids: list[int]) -> None: + async with self.pool.acquire() as conn: + async with conn.transaction(): + await conn.execute( + """ + UPDATE raw_posts + SET publication_status=$2, + editorial_status='published', + publication_error=NULL, + published_at=NOW(), + tg_publication_chat_id=$3, + tg_publication_thread_id=$4, + tg_publication_message_ids=$5, + tg_publication_url=$6, + updated_at=NOW() + WHERE id=$1 + """, + post_id, + PUBLICATION_STATUS_PUBLISHED, + self.chat_id, + self.thread_id, + message_ids, + tg_message_url(int(self.chat_id), message_ids[0]) if message_ids else None, + ) + await conn.execute( + """ + INSERT INTO post_publications(raw_post_id, platform, status, target_id, external_id, url, attachments_json, published_at, updated_at) + VALUES($1, 'tg', $2, $3, $4, $5, $6::jsonb, NOW(), NOW()) + ON CONFLICT (raw_post_id, platform) DO UPDATE + SET status=EXCLUDED.status, + target_id=EXCLUDED.target_id, + external_id=EXCLUDED.external_id, + url=EXCLUDED.url, + error=NULL, + attachments_json=EXCLUDED.attachments_json, + published_at=NOW(), + updated_at=NOW() + """, + post_id, + PUBLICATION_STATUS_PUBLISHED, + str(self.chat_id), + ",".join(str(mid) for mid in message_ids), + tg_message_url(int(self.chat_id), message_ids[0]) if message_ids else None, + json.dumps(message_ids, ensure_ascii=False), + ) + + async def mark_failed(self, post_id: int, error: str) -> None: + async with self.pool.acquire() as conn: + async with conn.transaction(): + await conn.execute( + """ + UPDATE raw_posts + SET publication_status=$2, + editorial_status='publish_failed', + publication_error=$3, + updated_at=NOW() + WHERE id=$1 + """, + post_id, + PUBLICATION_STATUS_FAILED, + error[:1000], + ) + await conn.execute( + """ + INSERT INTO post_publications(raw_post_id, platform, status, target_id, error, updated_at) + VALUES($1, 'tg', $2, $3, $4, NOW()) + ON CONFLICT (raw_post_id, platform) DO UPDATE + SET status=EXCLUDED.status, + target_id=EXCLUDED.target_id, + error=EXCLUDED.error, + updated_at=NOW() + """, + post_id, + PUBLICATION_STATUS_FAILED, + str(self.chat_id), + error[:1000], + ) + async def process_slot(self, slot: dict[str, Any]) -> int: + count = int(slot["count"]) + recent = await self.load_recent() + candidates = await self.load_candidates(max(50, count * 10)) + posts = self.choose_posts(candidates, recent, count) + published = 0 + for post in posts: + await self.heartbeat.beat(self.pool, status="publishing", meta={"post_id": int(post["id"]), "slot": slot["id"]}, force=True) + try: + message_ids = await self.send_post(post) + await self.mark_published(int(post["id"]), message_ids) + published += 1 + logger.info("TG published raw_post={} messages={}", post["id"], message_ids) + except Exception as exc: + await self.mark_failed(int(post["id"]), str(exc)) + logger.exception("TG publish failed raw_post={}: {}", post["id"], exc) + await self.pool.execute( + """ + UPDATE publication_runs + SET published_count=$2, + status=$3, + error=$4, + updated_at=NOW() + WHERE id=$1 + """, + int(slot["run_id"]), + published, + "done" if published == count else "partial", + None if published == count else f"published {published} of {count}", + ) + return published + + async def run_once(self) -> bool: + await self.reload_settings() + enabled = await is_worker_enabled(self.pool, WORKER_TG_POSTER) + app_enabled = await fetch_bool_setting("tg_poster_enabled", False) + if not enabled or not app_enabled: + await self.heartbeat.beat(self.pool, status="disabled", force=True) + return False + slots = await self.due_slots() + if not slots: + await self.heartbeat.beat(self.pool, status="idle") + return False + total = 0 + for slot in slots: + total += await self.process_slot(slot) + await self.heartbeat.beat(self.pool, status="idle", meta={"published": total}, force=True) + return bool(total) + + async def run_loop(self) -> None: + await self.init() + logger.info("{} started pid={}", WORKER_TG_POSTER, os.getpid()) + try: + while True: + await self.run_once() + await asyncio.sleep(max(10, await fetch_int_setting("tg_poster_interval_sec", 60))) + finally: + await self.close() + + +async def main() -> None: + logger.remove() + logger.add(sys.stdout, level=settings.log_level) + worker = TelegramPoster() + await worker.run_loop() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/vk_parser_app/workers/tg_reactor.py b/src/vk_parser_app/workers/tg_reactor.py new file mode 100644 index 0000000..7ff094e --- /dev/null +++ b/src/vk_parser_app/workers/tg_reactor.py @@ -0,0 +1,326 @@ +from __future__ import annotations + +import asyncio +import json +import os +import re +import sys +from typing import Any + +import aiohttp +from loguru import logger + +from ..config import settings +from ..constants import PUBLICATION_STATUS_PUBLISHED, WORKER_TG_REACTOR +from ..db import fetch_bool_setting, fetch_int_setting, fetch_setting, get_pool +from ..heartbeat import HeartbeatReporter +from ..jobs import is_worker_enabled + +PLATFORM_TG = "tg" +REACTION_STATUS_PENDING = "pending" +REACTION_STATUS_IN_PROGRESS = "in_progress" +REACTION_STATUS_REACTED = "reacted" +REACTION_STATUS_FAILED = "failed" + + +def split_tokens(value: Any) -> list[str]: + if isinstance(value, list): + candidates = [str(item or "") for item in value] + else: + candidates = re.split(r"[\s,;]+", str(value or "")) + return [token.strip() for token in candidates if token.strip()] + + +def normalize_reactions(value: Any) -> list[str]: + if not isinstance(value, list): + value = [] + reactions = [] + for item in value: + emoji = str(item or "").strip() + if emoji: + reactions.append(emoji) + return reactions or ["👍"] + + +def parse_message_ids(row: dict[str, Any]) -> list[str]: + raw = row.get("attachments_json") + values: list[Any] = [] + if isinstance(raw, list): + values = raw + elif isinstance(raw, str): + try: + parsed = json.loads(raw) + if isinstance(parsed, list): + values = parsed + except json.JSONDecodeError: + pass + if not values and row.get("external_id"): + values = str(row["external_id"]).split(",") + message_ids = [] + for value in values: + text = str(value or "").strip() + if text: + message_ids.append(text) + return message_ids + + +class TelegramReactor: + def __init__(self) -> None: + self.pool = None + self.heartbeat = HeartbeatReporter(WORKER_TG_REACTOR, 30) + self.bot_tokens: list[str] = [] + self.reactions: list[str] = ["👍"] + self.delay_sec = 60 + self.interval_sec = 60 + self.reaction_pause_sec = 2 + self.react_all_messages = False + self.max_attempts = 3 + self.retry_backoff_max_sec = 30 + self.since = "" + + async def init(self) -> None: + self.pool = await get_pool() + await self.reload_settings() + + async def reload_settings(self) -> None: + tokens = split_tokens(await fetch_setting("tg_reactor_bot_tokens", "")) + if not tokens: + fallback = str(await fetch_setting("tg_poster_bot_token", "") or "").strip() or settings.tg_bot_token + tokens = split_tokens(fallback) + if not tokens: + raise RuntimeError("tg_reactor_bot_tokens, tg_poster_bot_token and TG_BOT_TOKEN are empty") + self.bot_tokens = tokens + self.reactions = normalize_reactions(await fetch_setting("tg_reactor_reactions_json", ["👍"])) + self.since = str(await fetch_setting("tg_reactor_since", "") or "").strip() + self.delay_sec = max(0, await fetch_int_setting("tg_reactor_delay_sec", 60)) + self.interval_sec = max(10, await fetch_int_setting("tg_reactor_interval_sec", 60)) + self.reaction_pause_sec = max(0, await fetch_int_setting("tg_reactor_reaction_pause_sec", 2)) + self.react_all_messages = await fetch_bool_setting("tg_reactor_react_all_messages", False) + self.max_attempts = max(1, await fetch_int_setting("tg_reactor_max_attempts", 3)) + self.retry_backoff_max_sec = max(1, await fetch_int_setting("tg_reactor_retry_backoff_max_sec", 30)) + logger.info( + "TG reactor config: bots={} reactions={} since={} delay={}s pause={}s all_messages={}", + len(self.bot_tokens), + self.reactions, + self.since or "-", + self.delay_sec, + self.reaction_pause_sec, + self.react_all_messages, + ) + + async def load_publications(self, limit: int = 50) -> list[dict[str, Any]]: + rows = await self.pool.fetch( + """ + SELECT pp.* + FROM post_publications pp + WHERE pp.platform=$1 + AND pp.status=$2 + AND pp.published_at IS NOT NULL + AND pp.published_at <= NOW() - ($3::int * INTERVAL '1 second') + AND ($4::text = '' OR pp.published_at >= $4::timestamptz) + ORDER BY pp.published_at DESC, pp.id DESC + LIMIT $5::int + """, + PLATFORM_TG, + PUBLICATION_STATUS_PUBLISHED, + self.delay_sec, + self.since, + limit, + ) + return [dict(row) for row in rows] + + async def ensure_reaction_row( + self, + publication: dict[str, Any], + reactor_key: str, + reaction: str, + message_id: str, + ) -> dict[str, Any] | None: + row = await self.pool.fetchrow( + """ + INSERT INTO post_reactions( + publication_id, raw_post_id, platform, reactor_key, reaction, target_id, message_id, status, updated_at + ) + VALUES($1, $2, $3, $4, $5, $6, $7, $8, NOW()) + ON CONFLICT (publication_id, reactor_key, message_id) DO NOTHING + RETURNING * + """, + int(publication["id"]), + int(publication["raw_post_id"]), + PLATFORM_TG, + reactor_key, + reaction, + str(publication["target_id"]), + message_id, + REACTION_STATUS_PENDING, + ) + if row: + return dict(row) + row = await self.pool.fetchrow( + """ + SELECT * + FROM post_reactions + WHERE publication_id=$1 AND reactor_key=$2 AND message_id=$3 + """, + int(publication["id"]), + reactor_key, + message_id, + ) + if not row or row["status"] == REACTION_STATUS_REACTED or int(row["attempts"] or 0) >= self.max_attempts: + return None + return dict(row) + + async def claim_reaction(self, reaction_id: int) -> dict[str, Any] | None: + row = await self.pool.fetchrow( + """ + UPDATE post_reactions + SET status=$2, + attempts=attempts + 1, + error=NULL, + updated_at=NOW() + WHERE id=$1 + AND status IN ($3, $4) + AND attempts < $5 + RETURNING * + """, + reaction_id, + REACTION_STATUS_IN_PROGRESS, + REACTION_STATUS_PENDING, + REACTION_STATUS_FAILED, + self.max_attempts, + ) + return dict(row) if row else None + + async def set_reaction(self, token: str, chat_id: str, message_id: str, reaction: str) -> None: + url = f"https://api.telegram.org/bot{token}/setMessageReaction" + payload = { + "chat_id": int(chat_id), + "message_id": int(message_id), + "reaction": [{"type": "emoji", "emoji": reaction}], + "is_big": False, + } + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=30)) as session: + async with session.post(url, json=payload) as resp: + data = await resp.json(content_type=None) + if not data.get("ok"): + description = data.get("description") or data + retry_after = ((data.get("parameters") or {}).get("retry_after")) if isinstance(data, dict) else None + if retry_after: + raise RuntimeError(f"Telegram retry_after={retry_after}: {description}") + raise RuntimeError(str(description)) + + async def mark_reacted(self, reaction_id: int) -> None: + await self.pool.execute( + """ + UPDATE post_reactions + SET status=$2, + error=NULL, + reacted_at=NOW(), + updated_at=NOW() + WHERE id=$1 + """, + reaction_id, + REACTION_STATUS_REACTED, + ) + + async def mark_failed(self, reaction_id: int, error: str) -> None: + await self.pool.execute( + """ + UPDATE post_reactions + SET status=$2, + error=$3, + updated_at=NOW() + WHERE id=$1 + """, + reaction_id, + REACTION_STATUS_FAILED, + error[:1000], + ) + + async def process_publication(self, publication: dict[str, Any]) -> int: + message_ids = parse_message_ids(publication) + if not message_ids: + return 0 + if not self.react_all_messages: + message_ids = message_ids[:1] + + done = 0 + for message_id in message_ids: + for idx, token in enumerate(self.bot_tokens): + reaction = self.reactions[idx % len(self.reactions)] + reactor_key = f"bot-{idx + 1}:{reaction}" + row = await self.ensure_reaction_row(publication, reactor_key, reaction, message_id) + if not row: + continue + claimed = await self.claim_reaction(int(row["id"])) + if not claimed: + continue + try: + await self.set_reaction(token, str(publication["target_id"]), message_id, reaction) + await self.mark_reacted(int(claimed["id"])) + done += 1 + logger.info( + "TG reacted raw_post={} publication={} message={} reactor={}", + publication["raw_post_id"], + publication["id"], + message_id, + reactor_key, + ) + if self.reaction_pause_sec: + await asyncio.sleep(self.reaction_pause_sec) + except Exception as exc: + await self.mark_failed(int(claimed["id"]), str(exc)) + logger.warning( + "TG reaction failed raw_post={} publication={} message={} reactor={}: {}", + publication["raw_post_id"], + publication["id"], + message_id, + reactor_key, + exc, + ) + await asyncio.sleep(min(2 ** int(claimed["attempts"]), self.retry_backoff_max_sec)) + return done + + async def run_once(self) -> bool: + await self.reload_settings() + enabled = await is_worker_enabled(self.pool, WORKER_TG_REACTOR) + app_enabled = await fetch_bool_setting("tg_reactor_enabled", False) + if not enabled or not app_enabled: + await self.heartbeat.beat(self.pool, status="disabled", force=True) + return False + publications = await self.load_publications() + if not publications: + await self.heartbeat.beat(self.pool, status="idle") + return False + total = 0 + for publication in publications: + await self.heartbeat.beat( + self.pool, + status="reacting", + meta={"publication_id": int(publication["id"]), "raw_post_id": int(publication["raw_post_id"])}, + force=True, + ) + total += await self.process_publication(publication) + await self.heartbeat.beat(self.pool, status="idle", meta={"reactions": total}, force=True) + return bool(total) + + async def run_loop(self) -> None: + await self.init() + logger.info("{} started pid={}", WORKER_TG_REACTOR, os.getpid()) + while True: + try: + await self.run_once() + except Exception as exc: + logger.exception("TG reactor loop error: {}", exc) + await asyncio.sleep(self.interval_sec) + + +async def main() -> None: + logger.remove() + logger.add(sys.stdout, level=settings.log_level) + worker = TelegramReactor() + await worker.run_loop() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/vk_parser_app/workers/vk_poster.py b/src/vk_parser_app/workers/vk_poster.py new file mode 100644 index 0000000..9ffa960 --- /dev/null +++ b/src/vk_parser_app/workers/vk_poster.py @@ -0,0 +1,442 @@ +from __future__ import annotations + +import asyncio +import json +import os +import re +import sys +from datetime import datetime, time, timedelta, timezone +from typing import Any +from zoneinfo import ZoneInfo + +import aiohttp +from loguru import logger + +from ..config import settings +from ..constants import ( + PUBLICATION_STATUS_FAILED, + PUBLICATION_STATUS_PENDING, + PUBLICATION_STATUS_PUBLISHED, + WORKER_VK_POSTER, +) +from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool +from ..heartbeat import HeartbeatReporter +from ..jobs import is_worker_enabled +from ..text_utils import build_publication_text +from ..vk_api import VKAPIClient, post_vk_url + +LOCAL_TZ = ZoneInfo("Asia/Yekaterinburg") +VK_MESSAGE_LIMIT = 16384 +VK_MAX_ATTACHMENTS = 10 + + +def valid_time(value: str) -> bool: + return bool(re.fullmatch(r"(?:[01]\d|2[0-3]):[0-5]\d", str(value or "").strip())) + + +def normalize_schedule(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list): + value = [] + rows: list[dict[str, Any]] = [] + for idx, item in enumerate(value): + if not isinstance(item, dict): + continue + time_value = str(item.get("time") or "").strip() + if not valid_time(time_value): + continue + count = max(0, min(20, int(item.get("count") or 0))) + if count < 1: + continue + rows.append( + { + "id": str(item.get("id") or f"slot-{idx + 1}").strip()[:80], + "time": time_value, + "count": count, + "enabled": bool(item.get("enabled", True)), + } + ) + return sorted(rows, key=lambda row: row["time"]) + + +class VKPoster: + def __init__(self) -> None: + self.pool = None + self.heartbeat = HeartbeatReporter(WORKER_VK_POSTER, 30) + self.token = "" + self.owner_id = 0 + self.from_group = True + self.schedule: list[dict[str, Any]] = [] + self.interval_sec = 60 + self.send_delay_sec = 1.0 + self.recent_window = 20 + self.category_repeat_penalty = 3.0 + self.source_repeat_penalty = 4.0 + self.dry_run = False + + async def init(self) -> None: + self.pool = await get_pool() + + def token_expires_soon(self, value: Any) -> bool: + raw = str(value or "").strip() + if not raw: + return False + try: + expires_at = datetime.fromisoformat(raw) + if expires_at.tzinfo is None: + expires_at = expires_at.replace(tzinfo=timezone.utc) + except ValueError: + return False + return expires_at <= datetime.now(timezone.utc) + timedelta(minutes=15) + + async def save_setting(self, key: str, value: Any) -> None: + await self.pool.execute( + """ + UPDATE app_settings + SET value_json=$2::jsonb, + updated_at=NOW() + WHERE key=$1 + """, + key, + json.dumps(value, ensure_ascii=False), + ) + + async def refresh_access_token_if_needed(self) -> None: + expires_at = await fetch_setting("vk_poster_token_expires_at", "") + if not self.token_expires_soon(expires_at): + return + refresh_token = str(await fetch_setting("vk_poster_refresh_token", "") or "").strip() + client_id = str(await fetch_setting("vk_poster_app_id", "") or "").strip() + client_secret = str(await fetch_setting("vk_poster_client_secret", "") or "").strip() + device_id = str(await fetch_setting("vk_poster_token_device_id", "") or "").strip() + if not refresh_token or not client_id: + return + data = { + "grant_type": "refresh_token", + "refresh_token": refresh_token, + "client_id": client_id, + "device_id": device_id, + "state": "vk-poster-refresh", + } + if client_secret: + data["client_secret"] = client_secret + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=30)) as session: + async with session.post("https://id.vk.ru/oauth2/auth", data=data) as resp: + payload = await resp.json(content_type=None) + if payload.get("error"): + raise RuntimeError(f"VK token refresh failed: {payload.get('error')}: {payload.get('error_description')}") + access_token = str(payload.get("access_token") or "").strip() + if not access_token: + raise RuntimeError(f"VK token refresh did not return access_token: {payload}") + new_refresh = str(payload.get("refresh_token") or "").strip() or refresh_token + new_device_id = str(payload.get("device_id") or "").strip() or device_id + expires_in = payload.get("expires_in") + expires_at_value = "" + try: + expires_at_value = (datetime.now(timezone.utc) + timedelta(seconds=int(expires_in))).isoformat() + except Exception: + pass + await self.save_setting("vk_poster_access_token", access_token) + await self.save_setting("vk_poster_refresh_token", new_refresh) + await self.save_setting("vk_poster_token_device_id", new_device_id) + await self.save_setting("vk_poster_token_expires_at", expires_at_value) + self.token = access_token + logger.info("VK access token refreshed; expires_at={}", expires_at_value) + + async def reload_settings(self) -> None: + self.token = ( + str(await fetch_setting("vk_poster_access_token", "") or "").strip() + or settings.vk_group_access_token + or settings.vk_access_token + ) + await self.refresh_access_token_if_needed() + self.owner_id = int(await fetch_int_setting("vk_poster_owner_id", settings.vk_storage_owner_id if settings.vk_storage_group_id else 0)) + self.from_group = await fetch_bool_setting("vk_poster_from_group", True) + self.schedule = normalize_schedule(await fetch_setting("vk_poster_schedule_json", [])) + self.interval_sec = max(10, await fetch_int_setting("vk_poster_interval_sec", 60)) + self.send_delay_sec = max(0.0, await fetch_float_setting("vk_poster_send_delay_sec", 1.0)) + self.recent_window = max(1, await fetch_int_setting("vk_poster_recent_window", 20)) + self.category_repeat_penalty = max(0.0, await fetch_float_setting("vk_poster_category_repeat_penalty", 3.0)) + self.source_repeat_penalty = max(0.0, await fetch_float_setting("vk_poster_source_repeat_penalty", 4.0)) + self.dry_run = await fetch_bool_setting("vk_poster_dry_run", False) + if not self.token: + raise RuntimeError("vk_poster_access_token, VK_GROUP_ACCESS_TOKEN and VK_ACCESS_TOKEN are empty") + if not self.owner_id: + raise RuntimeError("vk_poster_owner_id is empty") + logger.info("VK poster config: owner={} schedule={} dry_run={}", self.owner_id, self.schedule, self.dry_run) + + async def due_slots(self) -> list[dict[str, Any]]: + now = datetime.now(LOCAL_TZ) + today = now.date() + due: list[dict[str, Any]] = [] + for row in self.schedule: + if not row.get("enabled"): + continue + hour, minute = [int(part) for part in str(row["time"]).split(":", 1)] + slot_start = datetime.combine(today, time(hour=hour, minute=minute), tzinfo=LOCAL_TZ) + slot_end = slot_start + timedelta(minutes=10) + if not (slot_start <= now < slot_end): + continue + inserted = await self.pool.fetchrow( + """ + INSERT INTO publication_runs(poster, schedule_id, scheduled_for, scheduled_time, planned_count, status) + VALUES($1, $2, $3, $4, $5, 'started') + ON CONFLICT (poster, schedule_id, scheduled_for) DO NOTHING + RETURNING id + """, + "vk", + row["id"], + today, + row["time"], + int(row["count"]), + ) + if inserted: + due.append({**row, "run_id": int(inserted["id"]), "date": today}) + return due + + async def load_recent(self) -> list[dict[str, Any]]: + rows = await self.pool.fetch( + """ + SELECT rp.final_category_tag, rp.rewrite_category_tag, rp.final_source_tag, rp.rewrite_source_tag + FROM post_publications pp + JOIN raw_posts rp ON rp.id=pp.raw_post_id + WHERE pp.platform='vk' + AND pp.status=$1 + AND pp.target_id=$2 + ORDER BY pp.published_at DESC NULLS LAST, pp.id DESC + LIMIT $3::int + """, + PUBLICATION_STATUS_PUBLISHED, + str(self.owner_id), + self.recent_window, + ) + return [dict(row) for row in rows] + + async def load_candidates(self, limit: int) -> list[dict[str, Any]]: + rows = await self.pool.fetch( + """ + SELECT rp.*, s.name AS source_name, s.tag AS source_tag + FROM raw_posts rp + JOIN sources s ON s.id=rp.source_id + LEFT JOIN post_publications pp ON pp.raw_post_id=rp.id AND pp.platform='vk' + WHERE rp.status='storage_ready' + AND rp.rewrite_status='ready' + AND COALESCE(rp.editorial_status, 'review') IN ('accepted', 'published', 'publish_failed') + AND COALESCE(pp.status, $1) = $1 + AND EXISTS ( + SELECT 1 + FROM raw_post_media rpm + WHERE rpm.raw_post_id=rp.id + AND COALESCE(rpm.editor_hidden, FALSE)=FALSE + AND COALESCE(rpm.editor_added, FALSE)=FALSE + AND ( + (rpm.media_type='photo' AND rpm.original_attachment_id LIKE 'photo%') + OR ( + rpm.media_type='video' + AND rpm.original_attachment_id LIKE 'video%' + AND COALESCE(rpm.status, '') <> 'link_only' + ) + ) + ) + ORDER BY COALESCE(rp.reviewed_at, rp.edited_at, rp.rewritten_at, rp.created_at) ASC, + rp.id ASC + LIMIT $2::int + """, + PUBLICATION_STATUS_PENDING, + max(limit, 50), + ) + return [dict(row) for row in rows] + + def choose_posts(self, candidates: list[dict[str, Any]], recent: list[dict[str, Any]], count: int) -> list[dict[str, Any]]: + selected: list[dict[str, Any]] = [] + recent_categories = [str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "") for row in recent] + recent_sources = [str(row.get("final_source_tag") or row.get("rewrite_source_tag") or "") for row in recent] + pool = list(candidates) + while pool and len(selected) < count: + best_idx = 0 + selected_categories = [str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "") for row in selected] + selected_sources = [str(row.get("final_source_tag") or row.get("rewrite_source_tag") or row.get("source_tag") or "") for row in selected] + previous_category = selected_categories[-1] if selected_categories else (recent_categories[0] if recent_categories else "") + previous_source = selected_sources[-1] if selected_sources else (recent_sources[0] if recent_sources else "") + best_rank: tuple[int, float, int] | None = None + for idx, post in enumerate(pool): + category = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or "") + source = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "") + immediate_repeats = int(bool(category) and category == previous_category) + int(bool(source) and source == previous_source) + repeat_penalty = ( + (recent_categories.count(category) + selected_categories.count(category)) * self.category_repeat_penalty + + (recent_sources.count(source) + selected_sources.count(source)) * self.source_repeat_penalty + ) + rank = (immediate_repeats, repeat_penalty, idx) + if best_rank is None or rank < best_rank: + best_idx = idx + best_rank = rank + selected.append(pool.pop(best_idx)) + return selected + + async def load_media(self, raw_post_id: int) -> list[dict[str, Any]]: + rows = await self.pool.fetch( + """ + SELECT * + FROM raw_post_media + WHERE raw_post_id=$1 + AND COALESCE(editor_hidden, FALSE)=FALSE + AND media_type IN ('photo', 'video') + ORDER BY sort_order ASC, id ASC + LIMIT $2::int + """, + raw_post_id, + VK_MAX_ATTACHMENTS, + ) + return [dict(row) for row in rows] + + def build_text(self, post: dict[str, Any]) -> str: + category_tag = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or post.get("final_category") or post.get("rewrite_category") or "") + source_tag = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "") + return build_publication_text(post.get("final_text") or post.get("rewritten_text") or "", category_tag, source_tag)[:VK_MESSAGE_LIMIT] + + async def media_attachment(self, client: VKAPIClient, item: dict[str, Any]) -> str | None: + attachment_id = str(item.get("original_attachment_id") or "").strip() + media_type = str(item.get("media_type") or "").strip() + if media_type == "video": + if str(item.get("status") or "").strip() == "link_only": + return None + return attachment_id if attachment_id.startswith("video") else None + if media_type != "photo": + return None + if attachment_id.startswith("photo") and not item.get("editor_added"): + return attachment_id + return None + + async def send_post(self, post: dict[str, Any]) -> tuple[int, str, list[str]]: + text = self.build_text(post) + media_rows = await self.load_media(int(post["id"])) + async with VKAPIClient( + token=self.token, + rps=2, + timeout_total_sec=max(30, await fetch_int_setting("vk_poster_timeout_sec", 90)), + retry_attempts=max(1, await fetch_int_setting("vk_poster_max_attempts", 3)), + retry_min_delay_sec=2, + retry_max_delay_sec=max(2, await fetch_int_setting("vk_poster_retry_backoff_max_sec", 30)), + ) as client: + attachments: list[str] = [] + for item in media_rows: + try: + attachment = await self.media_attachment(client, item) + except Exception as exc: + logger.warning("VK media skipped raw_post={} media_id={}: {}", post["id"], item.get("id"), exc) + continue + if attachment: + attachments.append(attachment) + if len(attachments) >= VK_MAX_ATTACHMENTS: + break + if self.dry_run: + return 0, "", attachments + post_id = await client.create_wall_post(self.owner_id, text, attachments, from_group=self.from_group) + return post_id, post_vk_url(self.owner_id, post_id), attachments + + async def upsert_publication( + self, + post_id: int, + status: str, + external_id: str | None = None, + url: str | None = None, + error: str | None = None, + attachments: list[str] | None = None, + ) -> None: + await self.pool.execute( + """ + INSERT INTO post_publications( + raw_post_id, platform, status, target_id, external_id, url, error, attachments_json, published_at, updated_at + ) + VALUES($1, 'vk', $2, $3, $4, $5, $6, $7::jsonb, CASE WHEN $2=$8 THEN NOW() ELSE NULL END, NOW()) + ON CONFLICT (raw_post_id, platform) DO UPDATE + SET status=EXCLUDED.status, + target_id=EXCLUDED.target_id, + external_id=EXCLUDED.external_id, + url=EXCLUDED.url, + error=EXCLUDED.error, + attachments_json=EXCLUDED.attachments_json, + published_at=CASE WHEN EXCLUDED.status=$8 THEN COALESCE(EXCLUDED.published_at, NOW()) ELSE post_publications.published_at END, + updated_at=NOW() + """, + post_id, + status, + str(self.owner_id), + external_id, + url, + error[:1000] if error else None, + json.dumps(attachments or [], ensure_ascii=False), + PUBLICATION_STATUS_PUBLISHED, + ) + + async def process_slot(self, slot: dict[str, Any]) -> int: + count = int(slot["count"]) + recent = await self.load_recent() + candidates = await self.load_candidates(max(50, count * 10)) + posts = self.choose_posts(candidates, recent, count) + published = 0 + for post in posts: + await self.heartbeat.beat(self.pool, status="publishing", meta={"post_id": int(post["id"]), "slot": slot["id"]}, force=True) + try: + vk_post_id, url, attachments = await self.send_post(post) + await self.upsert_publication(int(post["id"]), PUBLICATION_STATUS_PUBLISHED, str(vk_post_id), url, attachments=attachments) + published += 1 + logger.info("VK published raw_post={} post_id={} attachments={}", post["id"], vk_post_id, len(attachments)) + except Exception as exc: + await self.upsert_publication(int(post["id"]), PUBLICATION_STATUS_FAILED, error=str(exc)) + logger.exception("VK publish failed raw_post={}: {}", post["id"], exc) + if self.send_delay_sec: + await asyncio.sleep(self.send_delay_sec) + await self.pool.execute( + """ + UPDATE publication_runs + SET published_count=$2, + status=$3, + error=$4, + updated_at=NOW() + WHERE id=$1 + """, + int(slot["run_id"]), + published, + "done" if published == count else "partial", + None if published == count else f"published {published} of {count}", + ) + return published + + async def run_once(self) -> bool: + await self.reload_settings() + enabled = await is_worker_enabled(self.pool, WORKER_VK_POSTER) + app_enabled = await fetch_bool_setting("vk_poster_enabled", False) + if not enabled or not app_enabled: + await self.heartbeat.beat(self.pool, status="disabled", force=True) + return False + slots = await self.due_slots() + if not slots: + await self.heartbeat.beat(self.pool, status="idle") + return False + total = 0 + for slot in slots: + total += await self.process_slot(slot) + await self.heartbeat.beat(self.pool, status="idle", meta={"published": total}, force=True) + return bool(total) + + async def run_loop(self) -> None: + await self.init() + logger.info("{} started pid={}", WORKER_VK_POSTER, os.getpid()) + while True: + try: + await self.run_once() + except Exception as exc: + logger.exception("VK poster loop error: {}", exc) + await asyncio.sleep(self.interval_sec) + + +async def main() -> None: + logger.remove() + logger.add(sys.stdout, level=settings.log_level) + worker = VKPoster() + await worker.run_loop() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/vk_parser_app/workers/vk_storage_uploader.py b/src/vk_parser_app/workers/vk_storage_uploader.py index c064e88..4f151eb 100644 --- a/src/vk_parser_app/workers/vk_storage_uploader.py +++ b/src/vk_parser_app/workers/vk_storage_uploader.py @@ -91,6 +91,7 @@ class TelegramStorageUploader: self.text_overflow_marker = "Продолжение следующим сообщением." self.download_timeout_sec = 45 self.video_max_size_mb = 49 + self.video_max_height = 720 self.video_max_duration_sec = 300 self.yt_dlp_timeout_sec = 300 self.max_media_attempts = 3 @@ -126,6 +127,7 @@ class TelegramStorageUploader: self.text_overflow_marker = str(await fetch_setting("telegram_text_overflow_marker", self.text_overflow_marker)) self.download_timeout_sec = max(5, await fetch_int_setting("uploader_download_timeout_sec", 45)) self.video_max_size_mb = max(1, await fetch_int_setting("video_max_size_mb", 49)) + self.video_max_height = max(144, await fetch_int_setting("video_max_height", 720)) self.video_max_duration_sec = max(1, await fetch_int_setting("video_max_duration_sec", 300)) self.yt_dlp_timeout_sec = max(30, await fetch_int_setting("uploader_yt_dlp_timeout_sec", 300)) self.max_media_attempts = max(1, await fetch_int_setting("uploader_max_media_attempts", 3)) @@ -192,6 +194,64 @@ class TelegramStorageUploader: ) return [dict(row) for row in rows] + def media_blocking_error(self, media: list[dict], prepared_media_ids: set[int] | None = None) -> str | None: + prepared_media_ids = prepared_media_ids or set() + visible_media = [ + item + for item in media + if str(item.get("media_type") or "") in ("photo", "video") + and not bool(item.get("editor_hidden")) + ] + if not visible_media: + return "no publishable media" + pending = [ + item + for item in visible_media + if str(item.get("status") or "") == "pending" + and int(item["id"]) not in prepared_media_ids + ] + if pending: + details = ", ".join( + f"{item.get('media_type')}#{item.get('id')}: {item.get('error') or 'pending'}" + for item in pending[:5] + ) + return f"media still pending: {details}" + failed = [ + item + for item in visible_media + if str(item.get("status") or "") == MEDIA_STATUS_FAILED + ] + if failed: + details = ", ".join( + f"{item.get('media_type')}#{item.get('id')}: {item.get('error') or 'failed'}" + for item in failed[:5] + ) + return f"media failed: {details}" + link_only = [ + item + for item in visible_media + if str(item.get("status") or "") == MEDIA_STATUS_LINK_ONLY + ] + if link_only: + details = ", ".join( + f"{item.get('media_type')}#{item.get('id')}: {item.get('error') or 'link only'}" + for item in link_only[:5] + ) + return f"media link only: {details}" + missing_upload = [ + item + for item in visible_media + if not (item.get("tg_file_id") or item.get("storage_attachment_id")) + and int(item["id"]) not in prepared_media_ids + ] + if missing_upload: + details = ", ".join( + f"{item.get('media_type')}#{item.get('id')}: missing tg_file_id" + for item in missing_upload[:5] + ) + return f"media not uploaded: {details}" + return None + async def download_bytes(self, session: aiohttp.ClientSession, url: str) -> bytes | None: try: async with session.get(url, timeout=aiohttp.ClientTimeout(total=self.download_timeout_sec)) as response: @@ -271,7 +331,13 @@ class TelegramStorageUploader: output_path, "--no-playlist", "-f", - f"best[filesize<{max_size}]/bestvideo[filesize<{max_size}]+bestaudio/best", + ( + f"best[height<={self.video_max_height}][filesize<{max_size}]" + f"/best[height<={self.video_max_height}]" + f"/bestvideo[height<={self.video_max_height}][filesize<{max_size}]+bestaudio/best" + f"/bestvideo[height<={self.video_max_height}]+bestaudio/best" + f"/best[filesize<{max_size}]" + ), "--quiet", "--no-warnings", ] @@ -498,6 +564,25 @@ class TelegramStorageUploader: media = await self.load_media(raw_post_id) prepared = await self.prepare_media(raw_post_id, media) media = await self.load_media(raw_post_id) + prepared_media_ids = {int(item["media_id"]) for item in prepared} + blocking_error = self.media_blocking_error(media, prepared_media_ids) + if blocking_error: + for item in prepared: + tmp_path = item.get("tmp_path") + if tmp_path: + try: + if os.path.exists(tmp_path): + os.remove(tmp_path) + except Exception: + pass + if not blocking_error.startswith("media still pending:"): + await self.mark_post_failed(raw_post_id, blocking_error) + await ack_done(self.pool, job_id) + logger.warning("Telegram storage blocked raw_post={}: {}", raw_post_id, blocking_error) + return + await ack_retry(self.pool, job, blocking_error, delay_sec=120) + logger.warning("Telegram storage retry raw_post={}: {}", raw_post_id, blocking_error) + return try: message_ids, meta_message_id = await self.send_storage_post(post, prepared, media) finally: