Update UI and migrations, fix dockerfile
This commit is contained in:
Your Name
+1714
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,110 @@
|
||||
# N8 Parser: current state
|
||||
|
||||
Last updated: 2026-06-17
|
||||
|
||||
This file is the short handoff state for future Codex threads. Read this first before touching the project, so the whole chat history does not need to be carried forward.
|
||||
|
||||
## Server
|
||||
|
||||
- Production host: `109.120.156.205`
|
||||
- Domain: `https://sw.exostring.xyz`
|
||||
- App directory on server: `/opt/vk-parser`
|
||||
- Timezone: `Asia/Yekaterinburg`
|
||||
- Main services:
|
||||
- `vk-parser-admin.service`
|
||||
- `vk-parser.service`
|
||||
- `vk-storage-uploader.service`
|
||||
- `ai-qualifier.service`
|
||||
- `ai-writer.service`
|
||||
- `site-poster.service`
|
||||
- Useful commands:
|
||||
- `cd /opt/vk-parser`
|
||||
- `systemctl status vk-parser-admin.service ai-qualifier.service ai-writer.service`
|
||||
- `journalctl -u ai-writer.service -n 100 --no-pager`
|
||||
- `journalctl -u ai-qualifier.service -n 100 --no-pager`
|
||||
- `PYTHONPATH=src .venv/bin/python -m py_compile src/vk_parser_app/admin.py src/vk_parser_app/workers/ai_writer.py`
|
||||
|
||||
## Local Project
|
||||
|
||||
- Local workspace: `D:\DEVELOPMENT\new_vk_parser`
|
||||
- Main app package: `src/vk_parser_app`
|
||||
- Admin routes: `src/vk_parser_app/admin.py`
|
||||
- Templates: `src/vk_parser_app/templates`
|
||||
- Workers:
|
||||
- VK parser: `src/vk_parser_app/workers/parser.py`
|
||||
- Telegram media/storage uploader: `src/vk_parser_app/workers/vk_storage_uploader.py`
|
||||
- AI qualifier: `src/vk_parser_app/workers/ai_qualifier.py`
|
||||
- AI writer: `src/vk_parser_app/workers/ai_writer.py`
|
||||
- DB migrations: `db/migrations`
|
||||
- Extended project documentation: `PROJECT.md`
|
||||
|
||||
## Pipeline
|
||||
|
||||
1. Sources are configured in admin.
|
||||
2. VK parser reads enabled sources and stores raw posts.
|
||||
3. Media/storage uploader sends raw post copies to Telegram storage and stores media metadata.
|
||||
4. AI qualifier scores raw posts and marks accepted/rejected/maybe.
|
||||
5. AI writer rewrites accepted posts into editor-ready drafts.
|
||||
6. Editor page allows review, editing, category/source tag selection, media upload/removal, accept/reject.
|
||||
7. Site poster automatically publishes accepted media-ready posts to Ghost.
|
||||
|
||||
## Important Text Rules
|
||||
|
||||
- AI writer output text must be clean: no physical hashtags at the end.
|
||||
- `raw_posts.final_text` must also stay clean.
|
||||
- Category and source hashtags are separate fields:
|
||||
- category tag: `final_category_tag` / `rewrite_category_tag`
|
||||
- source tag: `final_source_tag` / `rewrite_source_tag` / `sources.tag`
|
||||
- Publication preview may show hashtags by composing:
|
||||
- clean text
|
||||
- blank line
|
||||
- `#category #source`
|
||||
- The editor textarea must show only clean editable text.
|
||||
- The list preview in `/editor` should show the publication preview, including composed hashtags.
|
||||
- `build_publication_text()` in `src/vk_parser_app/text_utils.py` is display/composition helper only. Do not use it before writing `final_text` to DB.
|
||||
|
||||
## Recent Hotfix: Hashtags
|
||||
|
||||
On 2026-06-17 the hashtag flow was fixed:
|
||||
|
||||
- `ai_writer.py` now stores clean `final_text=$2`, not text with hashtags.
|
||||
- `admin.py` prepares:
|
||||
- `review_text`: clean text for textarea
|
||||
- `publication_preview_text`: composed display text with hashtags
|
||||
- editor accept/save store clean text.
|
||||
- existing DB rows with trailing `#category #source` in `final_text` were cleaned.
|
||||
- server check after fix: `remaining_hashtag_tail: 0`.
|
||||
|
||||
## Admin Pages
|
||||
|
||||
- `/sources`: source management.
|
||||
- `/raw`: raw post feed.
|
||||
- `/editor`: editor queue.
|
||||
- `/workers`: worker controls and settings.
|
||||
- `/prompt-test`: AI writer prompt test page.
|
||||
- `/users`: admin users.
|
||||
- `/logs`: audit/log page if enabled.
|
||||
|
||||
## Prompt Architecture
|
||||
|
||||
- Prompts are split into editable human part and technical contract part.
|
||||
- User should be able to edit human prompt safely without breaking JSON schema.
|
||||
- Contracts should contain JSON-only/output-schema requirements.
|
||||
- Writer should return category ID, not free-form tag, where possible.
|
||||
- Categories are managed in worker/admin settings and have stable IDs.
|
||||
|
||||
## Security Notes
|
||||
|
||||
- Secrets should not be committed.
|
||||
- `.env` should contain only secrets and deploy-specific values.
|
||||
- Runtime settings that are not secrets should live in DB/admin settings.
|
||||
- Admin has login protection and server has firewall/fail2ban hardening from previous iterations.
|
||||
|
||||
## Working Rules For Future Codex Runs
|
||||
|
||||
- Prefer small targeted file reads with `rg` and narrow ranges.
|
||||
- Avoid dumping long post texts from DB unless explicitly needed.
|
||||
- For server checks, return counts/statuses first.
|
||||
- Be careful: both local and server worktrees may be dirty.
|
||||
- Do not `git reset --hard` or revert user/server hotfixes.
|
||||
- When fixing production, patch server and then mirror the same change locally.
|
||||
@@ -21,8 +21,6 @@ SET value_json = to_jsonb($prompt$
|
||||
|
||||
Формат текста:
|
||||
- 3-6 коротких строк или 2-4 компактных абзаца;
|
||||
- можно использовать 1-3 тематических эмоджи как маркеры структуры, например: 🧵 материал/конструкция, 🎒 переноска, 🛡 защита, ⚙️ детали, 📦 комплект/наличие, 🎯 назначение;
|
||||
- эмоджи должны помогать читать текст, а не украшать каждую строку;
|
||||
- не добавляй ссылки и хэштеги в text.
|
||||
|
||||
Важное правило про источник:
|
||||
|
||||
@@ -30,8 +30,6 @@ SET value_json = to_jsonb($prompt$
|
||||
|
||||
ФОРМАТ ТЕКСТА
|
||||
- 3-6 коротких строк или 2-5 компактных абзацев;
|
||||
- можно использовать 1-3 тематических эмоджи как маркеры структуры, например: 🧵 материал/конструкция, 🎒 переноска, 🛡 защита, ⚙️ детали, 📦 комплект/наличие, 🎯 назначение;
|
||||
- эмоджи должны помогать читать текст, а не украшать каждую строку;
|
||||
- не добавляй ссылки и хэштеги в text.
|
||||
|
||||
СТИЛЬ
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
DELETE FROM audit_log
|
||||
WHERE created_at < NOW() - INTERVAL '30 days';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_log_created_at
|
||||
ON audit_log(created_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_log_action_time
|
||||
ON audit_log(action, created_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_log_actor_time
|
||||
ON audit_log(actor_id, created_at DESC)
|
||||
WHERE actor_id IS NOT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_log_entity_time
|
||||
ON audit_log(entity_type, entity_id, created_at DESC);
|
||||
@@ -0,0 +1,15 @@
|
||||
UPDATE app_settings s
|
||||
SET value_json = to_jsonb(wc.enabled),
|
||||
updated_at = NOW()
|
||||
FROM worker_controls wc
|
||||
WHERE (wc.name, s.key) IN (
|
||||
('ai-qualifier', 'ai_qualifier_enabled'),
|
||||
('ai-writer', 'ai_writer_enabled')
|
||||
);
|
||||
|
||||
UPDATE raw_posts
|
||||
SET rewrite_status='failed',
|
||||
rewrite_notes=COALESCE(rewrite_notes, 'recovered stale writer processing state'),
|
||||
updated_at=NOW()
|
||||
WHERE rewrite_status='processing'
|
||||
AND updated_at < NOW() - INTERVAL '30 minutes';
|
||||
@@ -0,0 +1,21 @@
|
||||
-- Keep the AI-writer divider deterministic.
|
||||
-- Target separator length: 27 characters.
|
||||
|
||||
UPDATE app_settings
|
||||
SET value_json = to_jsonb(
|
||||
replace(
|
||||
value_json #>> '{}',
|
||||
'2. Черта с точным количеством символов ━━━━━━━━━━━━━━━━━━━━━━━━━━━',
|
||||
'2. Черта-разделитель: отдельная строка ровно из 27 символов. Используй строго эту строку без изменений: ━━━━━━━━━━━━━━━━━━━━━━━━━━━'
|
||||
)
|
||||
),
|
||||
updated_at = NOW()
|
||||
WHERE key = 'ai_writer_prompt'
|
||||
AND (value_json #>> '{}') LIKE '%Черта с точным количеством символов%';
|
||||
|
||||
UPDATE raw_posts
|
||||
SET rewritten_text = regexp_replace(rewritten_text, E'(^━{3,}$)', '━━━━━━━━━━━━━━━━━━━━━━━━━━━', 'gm'),
|
||||
final_text = regexp_replace(final_text, E'(^━{3,}$)', '━━━━━━━━━━━━━━━━━━━━━━━━━━━', 'gm'),
|
||||
updated_at = NOW()
|
||||
WHERE COALESCE(rewritten_text, '') LIKE '%━%'
|
||||
OR COALESCE(final_text, '') LIKE '%━%';
|
||||
@@ -0,0 +1,93 @@
|
||||
ALTER TABLE raw_posts
|
||||
ADD COLUMN IF NOT EXISTS publication_status TEXT NOT NULL DEFAULT 'pending',
|
||||
ADD COLUMN IF NOT EXISTS publication_error TEXT,
|
||||
ADD COLUMN IF NOT EXISTS published_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS tg_publication_chat_id BIGINT,
|
||||
ADD COLUMN IF NOT EXISTS tg_publication_thread_id BIGINT,
|
||||
ADD COLUMN IF NOT EXISTS tg_publication_message_ids BIGINT[],
|
||||
ADD COLUMN IF NOT EXISTS tg_publication_url TEXT;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_posts_publication_queue
|
||||
ON raw_posts(publication_status, editorial_status, rewrite_status, published_at, reviewed_at, id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_posts_publication_rank
|
||||
ON raw_posts(final_category_tag, final_source_tag, published_at DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS publication_runs (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
poster TEXT NOT NULL,
|
||||
schedule_id TEXT NOT NULL,
|
||||
scheduled_for DATE NOT NULL,
|
||||
scheduled_time TEXT NOT NULL,
|
||||
planned_count INTEGER NOT NULL DEFAULT 0,
|
||||
published_count INTEGER NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL DEFAULT 'started',
|
||||
error TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE(poster, schedule_id, scheduled_for)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_publication_runs_poster_date
|
||||
ON publication_runs(poster, scheduled_for DESC, scheduled_time);
|
||||
|
||||
INSERT INTO worker_controls(name, enabled, settings_json)
|
||||
VALUES ('tg-poster', FALSE, '{}'::jsonb)
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
|
||||
INSERT INTO app_settings(key, value_json, value_type, title, description, category)
|
||||
VALUES
|
||||
('tg_poster_enabled', 'false'::jsonb, 'bool', 'TG poster enabled', 'Double safety flag for publishing accepted posts to Telegram.', 'TG Poster'),
|
||||
('tg_poster_bot_token', '""'::jsonb, 'secret', 'TG poster bot token', 'Bot token for final Telegram publication. For test can be the same as upload bot token.', 'TG Poster'),
|
||||
('tg_poster_chat_id', '"-1003712724327"'::jsonb, 'str', 'TG publication chat id', 'Target Telegram channel/group id. Supports chat_id:thread_id for topics.', 'TG Poster'),
|
||||
('tg_poster_schedule_json', '[{"id":"morning","time":"10:00","count":1,"enabled":true},{"id":"evening","time":"18:00","count":1,"enabled":true}]'::jsonb, 'json', 'TG publication schedule', 'EKB time schedule rows: id, time HH:MM, count, enabled.', 'TG Poster'),
|
||||
('tg_poster_interval_sec', '60'::jsonb, 'int', 'TG poster interval, sec', 'Pause between poster checks.', 'TG Poster'),
|
||||
('tg_poster_media_group_max_items', '10'::jsonb, 'int', 'TG media group max items', 'Telegram media group supports 2..10 media items.', 'TG Poster'),
|
||||
('tg_poster_caption_limit', '1024'::jsonb, 'int', 'TG caption limit', 'Telegram caption limit for media messages.', 'TG Poster'),
|
||||
('tg_poster_message_limit', '4096'::jsonb, 'int', 'TG message limit', 'Telegram text message limit.', 'TG Poster'),
|
||||
('tg_poster_max_attempts', '3'::jsonb, 'int', 'TG post retry attempts', 'Retries for Telegram API errors.', 'TG Poster'),
|
||||
('tg_poster_retry_backoff_max_sec', '30'::jsonb, 'int', 'TG retry backoff max, sec', 'Max retry sleep for Telegram API errors.', 'TG Poster'),
|
||||
('tg_poster_send_delay_sec', '1'::jsonb, 'float', 'TG send delay, sec', 'Delay between Telegram sends.', 'TG Poster'),
|
||||
('tg_poster_text_overflow_caption', '"⬇️ Описание под медиа"'::jsonb, 'str', 'TG overflow caption', 'Caption used when full text is longer than Telegram media caption limit.', 'TG Poster'),
|
||||
('tg_poster_recent_window', '20'::jsonb, 'int', 'Ranking recent window', 'How many published posts are checked to reduce repeated categories and sources.', 'TG Poster'),
|
||||
('tg_poster_category_repeat_penalty', '3'::jsonb, 'float', 'Category repeat penalty', 'Ranking penalty for recently published same category.', 'TG Poster'),
|
||||
('tg_poster_source_repeat_penalty', '4'::jsonb, 'float', 'Source repeat penalty', 'Ranking penalty for recently published same source.', 'TG Poster')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
UPDATE app_settings
|
||||
SET description='Обрезает исходный raw-текст перед отправкой в LLM. Итоговый rewrite должен быть компактным: Telegram caption 1024 символа минус будущая строка хэштегов.'
|
||||
WHERE key='ai_writer_max_text_chars';
|
||||
|
||||
UPDATE app_settings
|
||||
SET value_json = to_jsonb($contract$
|
||||
OUTPUT SCHEMA (return array matching input order):
|
||||
{"rewrites":[{"id":123,"category_id":2,"text":"готовый текст без хэштегов","notes":"короткая заметка для редактора или null"}]}
|
||||
|
||||
Rules:
|
||||
- Input is a JSON object with key "posts" containing accepted posts.
|
||||
- Input has key "categories" with objects: id, name, tag.
|
||||
- Each post includes producer_name and producer_tag. producer_name can be a manufacturer, shop, or publishing source.
|
||||
- Pick exactly one category from the categories list in input and return its numeric id as category_id.
|
||||
- Do not add links or hashtags to rewritten text.
|
||||
- Keep text compact enough for Telegram media caption: target 850 characters, hard maximum 900 characters.
|
||||
- Remember the system appends category and source hashtags after text, so leave space for them.
|
||||
- Return JSON only. No markdown. No text outside JSON.
|
||||
- Never return an empty object. Never omit the "rewrites" key.
|
||||
- Return one rewrite for every input post id.
|
||||
- The "text" field must be a ready-to-publish Russian post with normal paragraph line breaks encoded as JSON string newlines.
|
||||
- The "notes" field must be a short editor note in Russian or null.
|
||||
$contract$::text),
|
||||
updated_at = NOW()
|
||||
WHERE key='ai_writer_contract';
|
||||
|
||||
UPDATE app_settings
|
||||
SET value_json = to_jsonb(
|
||||
regexp_replace(
|
||||
value_json #>> '{}',
|
||||
'- 3-6 коротких строк или 2-5 компактных абзацев;',
|
||||
'- 3-6 коротких строк или 2-4 компактных абзаца; итоговый text должен целиться в 850 символов и не превышать 900 символов, потому что система добавит хэштеги и Telegram caption ограничен 1024 символами;',
|
||||
'g'
|
||||
)
|
||||
)
|
||||
WHERE key='ai_writer_prompt'
|
||||
AND (value_json #>> '{}') LIKE '%3-6 коротких строк или 2-5 компактных абзацев%';
|
||||
@@ -0,0 +1,46 @@
|
||||
UPDATE app_settings
|
||||
SET value_json = '"⬇️ Описание"'::jsonb,
|
||||
updated_at = NOW()
|
||||
WHERE key='tg_poster_text_overflow_caption';
|
||||
|
||||
UPDATE app_settings
|
||||
SET value_json = $schedule$[
|
||||
{"id":"hour-00","time":"00:00","count":4,"enabled":true},
|
||||
{"id":"hour-01","time":"01:00","count":4,"enabled":true},
|
||||
{"id":"hour-02","time":"02:00","count":4,"enabled":true},
|
||||
{"id":"hour-03","time":"03:00","count":4,"enabled":true},
|
||||
{"id":"hour-04","time":"04:00","count":4,"enabled":true},
|
||||
{"id":"hour-05","time":"05:00","count":4,"enabled":true},
|
||||
{"id":"hour-06","time":"06:00","count":4,"enabled":true},
|
||||
{"id":"hour-07","time":"07:00","count":4,"enabled":true},
|
||||
{"id":"hour-08","time":"08:00","count":4,"enabled":true},
|
||||
{"id":"hour-09","time":"09:00","count":4,"enabled":true},
|
||||
{"id":"hour-10","time":"10:00","count":4,"enabled":true},
|
||||
{"id":"hour-11","time":"11:00","count":4,"enabled":true},
|
||||
{"id":"hour-12","time":"12:00","count":4,"enabled":true},
|
||||
{"id":"hour-13","time":"13:00","count":4,"enabled":true},
|
||||
{"id":"hour-14","time":"14:00","count":4,"enabled":true},
|
||||
{"id":"hour-15","time":"15:00","count":4,"enabled":true},
|
||||
{"id":"hour-16","time":"16:00","count":4,"enabled":true},
|
||||
{"id":"hour-17","time":"17:00","count":4,"enabled":true},
|
||||
{"id":"hour-18","time":"18:00","count":4,"enabled":true},
|
||||
{"id":"hour-19","time":"19:00","count":4,"enabled":true},
|
||||
{"id":"hour-20","time":"20:00","count":4,"enabled":true},
|
||||
{"id":"hour-21","time":"21:00","count":4,"enabled":true},
|
||||
{"id":"hour-22","time":"22:00","count":4,"enabled":true},
|
||||
{"id":"hour-23","time":"23:00","count":4,"enabled":true}
|
||||
]$schedule$::jsonb,
|
||||
updated_at = NOW()
|
||||
WHERE key='tg_poster_schedule_json';
|
||||
|
||||
UPDATE raw_posts
|
||||
SET editorial_status='published',
|
||||
updated_at=NOW()
|
||||
WHERE publication_status='published'
|
||||
AND editorial_status='accepted';
|
||||
|
||||
UPDATE raw_posts
|
||||
SET editorial_status='publish_failed',
|
||||
updated_at=NOW()
|
||||
WHERE publication_status='publish_failed'
|
||||
AND editorial_status='accepted';
|
||||
@@ -0,0 +1,44 @@
|
||||
-- Keep the AI-writer divider deterministic.
|
||||
-- Target separator length: 17 characters.
|
||||
|
||||
UPDATE app_settings
|
||||
SET value_json = to_jsonb(
|
||||
regexp_replace(
|
||||
value_json #>> '{}',
|
||||
'Черта-разделитель:[^\n]*',
|
||||
'Черта-разделитель: отдельная строка ровно из 17 символов. Используй строго эту строку без изменений: ━━━━━━━━━━━━━━━━━',
|
||||
'g'
|
||||
)
|
||||
),
|
||||
updated_at = NOW()
|
||||
WHERE key = 'ai_writer_prompt'
|
||||
AND (value_json #>> '{}') LIKE '%Черта-разделитель%';
|
||||
|
||||
UPDATE app_settings
|
||||
SET value_json = to_jsonb(
|
||||
CASE
|
||||
WHEN (value_json #>> '{}') LIKE '%Separator line must be exactly%'
|
||||
THEN regexp_replace(
|
||||
value_json #>> '{}',
|
||||
'Separator line must be exactly[^\n]*',
|
||||
'Separator line must be exactly this 17-character line when used: ━━━━━━━━━━━━━━━━━',
|
||||
'g'
|
||||
)
|
||||
ELSE (value_json #>> '{}') || E'\n- Separator line must be exactly this 17-character line when used: ━━━━━━━━━━━━━━━━━'
|
||||
END
|
||||
),
|
||||
updated_at = NOW()
|
||||
WHERE key = 'ai_writer_contract';
|
||||
|
||||
UPDATE raw_posts
|
||||
SET rewritten_text = CASE
|
||||
WHEN rewritten_text IS NULL THEN NULL
|
||||
ELSE regexp_replace(rewritten_text, E'^[━─—-]{3,}\\s*$', '━━━━━━━━━━━━━━━━━', 'gm')
|
||||
END,
|
||||
final_text = CASE
|
||||
WHEN final_text IS NULL THEN NULL
|
||||
ELSE regexp_replace(final_text, E'^[━─—-]{3,}\\s*$', '━━━━━━━━━━━━━━━━━', 'gm')
|
||||
END,
|
||||
updated_at = NOW()
|
||||
WHERE COALESCE(rewritten_text, '') ~ E'(?m)^[━─—-]{3,}\\s*$'
|
||||
OR COALESCE(final_text, '') ~ E'(?m)^[━─—-]{3,}\\s*$';
|
||||
@@ -0,0 +1,34 @@
|
||||
-- Manual Raw-to-rewrite flow is implemented in admin.py.
|
||||
-- Non-target category (sort_order/category_id 18, tag НЦК) must not enter editorial review.
|
||||
|
||||
UPDATE app_settings
|
||||
SET value_json = to_jsonb(
|
||||
CASE
|
||||
WHEN (value_json #>> '{}') LIKE '%category_id 18%'
|
||||
THEN value_json #>> '{}'
|
||||
ELSE (value_json #>> '{}') || E'\n- If the post is non-target content, choose category_id 18 ("Не целевой контент", tag "НЦК"); the app will reject it automatically.'
|
||||
END
|
||||
),
|
||||
updated_at = NOW()
|
||||
WHERE key = 'ai_writer_contract';
|
||||
|
||||
UPDATE raw_posts
|
||||
SET editorial_status='rejected',
|
||||
qualification_status='rejected',
|
||||
qualification_decision='rejected',
|
||||
qualification_model_decision='rejected',
|
||||
qualification_reason=COALESCE(NULLIF(qualification_reason, ''), 'AI writer selected non-target category'),
|
||||
qualification_reject_tag=COALESCE(NULLIF(qualification_reject_tag, ''), 'non_target_content'),
|
||||
editor_notes=COALESCE(NULLIF(editor_notes, ''), 'Отклонено AI: Не целевой контент'),
|
||||
reviewed_at=COALESCE(reviewed_at, NOW()),
|
||||
updated_at=NOW()
|
||||
WHERE rewrite_status='ready'
|
||||
AND COALESCE(editorial_status, 'review')='review'
|
||||
AND (
|
||||
rewrite_category_id=18
|
||||
OR final_category_id=18
|
||||
OR lower(COALESCE(rewrite_category_tag, '')) IN ('нцк', 'nck')
|
||||
OR lower(COALESCE(final_category_tag, '')) IN ('нцк', 'nck')
|
||||
OR lower(COALESCE(rewrite_category, '')) = lower('Не целевой контент')
|
||||
OR lower(COALESCE(final_category, '')) = lower('Не целевой контент')
|
||||
);
|
||||
@@ -0,0 +1,67 @@
|
||||
CREATE TABLE IF NOT EXISTS post_publications (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
raw_post_id BIGINT NOT NULL REFERENCES raw_posts(id) ON DELETE CASCADE,
|
||||
platform TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
target_id TEXT,
|
||||
external_id TEXT,
|
||||
url TEXT,
|
||||
error TEXT,
|
||||
attachments_json JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
published_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE(raw_post_id, platform)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_post_publications_platform_status
|
||||
ON post_publications(platform, status, target_id, published_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_post_publications_raw_post
|
||||
ON post_publications(raw_post_id);
|
||||
|
||||
INSERT INTO post_publications(raw_post_id, platform, status, target_id, external_id, url, error, attachments_json, published_at, updated_at)
|
||||
SELECT
|
||||
id,
|
||||
'tg',
|
||||
publication_status,
|
||||
tg_publication_chat_id::text,
|
||||
CASE WHEN tg_publication_message_ids IS NULL THEN NULL ELSE array_to_string(tg_publication_message_ids, ',') END,
|
||||
tg_publication_url,
|
||||
publication_error,
|
||||
COALESCE(to_jsonb(tg_publication_message_ids), '[]'::jsonb),
|
||||
published_at,
|
||||
NOW()
|
||||
FROM raw_posts
|
||||
WHERE COALESCE(publication_status, 'pending') <> 'pending'
|
||||
ON CONFLICT (raw_post_id, platform) DO UPDATE
|
||||
SET status=EXCLUDED.status,
|
||||
target_id=EXCLUDED.target_id,
|
||||
external_id=EXCLUDED.external_id,
|
||||
url=EXCLUDED.url,
|
||||
error=EXCLUDED.error,
|
||||
attachments_json=EXCLUDED.attachments_json,
|
||||
published_at=EXCLUDED.published_at,
|
||||
updated_at=NOW();
|
||||
|
||||
INSERT INTO worker_controls(name, enabled, settings_json)
|
||||
VALUES ('vk-poster', FALSE, '{}'::jsonb)
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
|
||||
INSERT INTO app_settings(key, value_json, value_type, title, description, category)
|
||||
VALUES
|
||||
('vk_poster_enabled', 'false'::jsonb, 'bool', 'VK poster enabled', 'Double safety flag for publishing accepted posts to VK.', 'VK Poster'),
|
||||
('vk_poster_access_token', '""'::jsonb, 'secret', 'VK poster access token', 'Group/user token with wall.post and photos permissions. If empty, env VK_GROUP_ACCESS_TOKEN/VK_ACCESS_TOKEN is used.', 'VK Poster'),
|
||||
('vk_poster_owner_id', '0'::jsonb, 'int', 'VK target owner_id', 'Target VK group owner_id, usually negative: -123456789.', 'VK Poster'),
|
||||
('vk_poster_from_group', 'true'::jsonb, 'bool', 'Post from group', 'Use from_group=1 when publishing to a community wall.', 'VK Poster'),
|
||||
('vk_poster_schedule_json', '[{"id":"vk-morning","time":"10:05","count":1,"enabled":true},{"id":"vk-evening","time":"18:05","count":1,"enabled":true}]'::jsonb, 'json', 'VK publication schedule', 'EKB time schedule rows: id, time HH:MM, count, enabled.', 'VK Poster'),
|
||||
('vk_poster_interval_sec', '60'::jsonb, 'int', 'VK poster interval, sec', 'Pause between poster checks.', 'VK Poster'),
|
||||
('vk_poster_timeout_sec', '90'::jsonb, 'int', 'VK API timeout, sec', 'Total timeout for VK API calls and media uploads.', 'VK Poster'),
|
||||
('vk_poster_max_attempts', '3'::jsonb, 'int', 'VK post retry attempts', 'Retries for VK API/network errors.', 'VK Poster'),
|
||||
('vk_poster_retry_backoff_max_sec', '30'::jsonb, 'int', 'VK retry backoff max, sec', 'Max retry sleep for VK API/network errors.', 'VK Poster'),
|
||||
('vk_poster_send_delay_sec', '1'::jsonb, 'float', 'VK send delay, sec', 'Delay between VK posts.', 'VK Poster'),
|
||||
('vk_poster_recent_window', '20'::jsonb, 'int', 'Ranking recent window', 'How many VK-published posts are checked to reduce repeated categories and sources.', 'VK Poster'),
|
||||
('vk_poster_category_repeat_penalty', '3'::jsonb, 'float', 'Category repeat penalty', 'Ranking penalty for recently published same category.', 'VK Poster'),
|
||||
('vk_poster_source_repeat_penalty', '4'::jsonb, 'float', 'Source repeat penalty', 'Ranking penalty for recently published same source.', 'VK Poster'),
|
||||
('vk_poster_dry_run', 'false'::jsonb, 'bool', 'VK dry run', 'If true, prepares attachments but does not call wall.post. Keep false for production.', 'VK Poster')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
@@ -0,0 +1,5 @@
|
||||
INSERT INTO app_settings(key, value_json, value_type, title, description, category)
|
||||
VALUES
|
||||
('vk_poster_app_id', '""'::jsonb, 'str', 'VK OAuth app id', 'VK application id used for user OAuth code flow.', 'VK Poster'),
|
||||
('vk_poster_client_secret', '""'::jsonb, 'secret', 'VK OAuth client secret', 'Protected key from VK application settings. Used server-side to exchange code for user access token.', 'VK Poster')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
@@ -0,0 +1,6 @@
|
||||
INSERT INTO app_settings(key, value_json, value_type, title, description, category)
|
||||
VALUES
|
||||
('vk_poster_refresh_token', '""'::jsonb, 'secret', 'VK OAuth refresh token', 'Refresh token from VK ID code flow. Used by VK poster to renew access token automatically.', 'VK Poster'),
|
||||
('vk_poster_token_device_id', '""'::jsonb, 'str', 'VK OAuth device id', 'VK ID device_id returned with authorization code/token exchange.', 'VK Poster'),
|
||||
('vk_poster_token_expires_at', '""'::jsonb, 'str', 'VK access token expires at', 'UTC ISO timestamp used by VK poster to refresh the access token before expiry.', 'VK Poster')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
@@ -0,0 +1,18 @@
|
||||
-- Make producer mention mandatory in every AI writer result.
|
||||
|
||||
UPDATE app_settings
|
||||
SET value_json = to_jsonb(
|
||||
CASE
|
||||
WHEN (value_json #>> '{}') LIKE '%explicitly mention producer_name%'
|
||||
OR (value_json #>> '{}') LIKE '%явно упоминает producer_name%'
|
||||
THEN value_json #>> '{}'
|
||||
ELSE replace(
|
||||
value_json #>> '{}',
|
||||
'- Each post includes producer_name and producer_tag. producer_name can be a manufacturer, shop, or publishing source.',
|
||||
'- Each post includes producer_name and producer_tag. producer_name can be a manufacturer, shop, or publishing source.' || E'\n' ||
|
||||
'- The "text" field for every rewrite must explicitly mention producer_name at least once. Do not satisfy this only through producer_tag, hashtags, notes, or metadata.'
|
||||
)
|
||||
END
|
||||
),
|
||||
updated_at = NOW()
|
||||
WHERE key = 'ai_writer_contract';
|
||||
@@ -0,0 +1,39 @@
|
||||
CREATE TABLE IF NOT EXISTS post_reactions (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
publication_id BIGINT NOT NULL REFERENCES post_publications(id) ON DELETE CASCADE,
|
||||
raw_post_id BIGINT NOT NULL REFERENCES raw_posts(id) ON DELETE CASCADE,
|
||||
platform TEXT NOT NULL,
|
||||
reactor_key TEXT NOT NULL,
|
||||
reaction TEXT NOT NULL,
|
||||
target_id TEXT NOT NULL,
|
||||
message_id TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
error TEXT,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
reacted_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE(publication_id, reactor_key, message_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_post_reactions_status
|
||||
ON post_reactions(platform, status, updated_at);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_post_reactions_raw_post
|
||||
ON post_reactions(raw_post_id);
|
||||
|
||||
INSERT INTO worker_controls(name, enabled, settings_json)
|
||||
VALUES ('tg-reactor', FALSE, '{}'::jsonb)
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
|
||||
INSERT INTO app_settings(key, value_json, value_type, title, description, category)
|
||||
VALUES
|
||||
('tg_reactor_enabled', 'false'::jsonb, 'bool', 'TG reactor enabled', 'Double safety flag for Telegram reaction worker.', 'TG Reactor'),
|
||||
('tg_reactor_bot_tokens', '""'::jsonb, 'secret', 'TG reactor bot tokens', 'One or more Telegram bot tokens separated by comma/newline. If empty, uses tg_poster_bot_token/TG_BOT_TOKEN.', 'TG Reactor'),
|
||||
('tg_reactor_reactions_json', '["👍"]'::jsonb, 'json', 'TG reactions', 'Emoji reactions. One bot can set one reaction; multiple tokens can set multiple reactions.', 'TG Reactor'),
|
||||
('tg_reactor_delay_sec', '60'::jsonb, 'int', 'Delay after publication, sec', 'How long to wait after TG publication before reacting.', 'TG Reactor'),
|
||||
('tg_reactor_interval_sec', '60'::jsonb, 'int', 'TG reactor interval, sec', 'Pause between reaction worker checks.', 'TG Reactor'),
|
||||
('tg_reactor_react_all_messages', 'false'::jsonb, 'bool', 'React to all media messages', 'If false, reacts only to the first message of each published post.', 'TG Reactor'),
|
||||
('tg_reactor_max_attempts', '3'::jsonb, 'int', 'TG reaction retry attempts', 'Retries per publication/message/reactor.', 'TG Reactor'),
|
||||
('tg_reactor_retry_backoff_max_sec', '30'::jsonb, 'int', 'TG reaction retry backoff max, sec', 'Max sleep for Telegram reaction retry errors.', 'TG Reactor')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
@@ -0,0 +1,4 @@
|
||||
INSERT INTO app_settings(key, value_json, value_type, title, description, category)
|
||||
VALUES
|
||||
('tg_reactor_since', '""'::jsonb, 'str', 'React only since', 'UTC ISO timestamp. If set, TG reactor ignores publications older than this moment.', 'TG Reactor')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
@@ -0,0 +1,4 @@
|
||||
INSERT INTO app_settings(key, value_json, value_type, title, description, category)
|
||||
VALUES
|
||||
('tg_reactor_reaction_pause_sec', '2'::jsonb, 'int', 'Pause between TG reactions, sec', 'Small pause after each successful reaction to avoid a burst of Telegram API calls.', 'TG Reactor')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
@@ -0,0 +1,14 @@
|
||||
INSERT INTO app_settings(key, value_json, value_type, title, description, category)
|
||||
VALUES
|
||||
('video_max_height', '720'::jsonb, 'int', 'Video max height', 'Maximum VK video height selected by yt-dlp before uploading to Telegram.', 'Uploader')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
UPDATE app_settings
|
||||
SET value_json='600'::jsonb,
|
||||
updated_at=NOW()
|
||||
WHERE key='uploader_yt_dlp_timeout_sec'
|
||||
AND CASE
|
||||
WHEN jsonb_typeof(value_json) = 'number' THEN (value_json #>> '{}')::int
|
||||
WHEN jsonb_typeof(value_json) = 'string' AND (value_json #>> '{}') ~ '^\d+$' THEN (value_json #>> '{}')::int
|
||||
ELSE 0
|
||||
END < 600;
|
||||
@@ -0,0 +1,13 @@
|
||||
INSERT INTO worker_controls(name, enabled, settings_json)
|
||||
VALUES ('daily-report', TRUE, '{}'::jsonb)
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
|
||||
INSERT INTO app_settings(key, value_json, value_type, title, description, category)
|
||||
VALUES
|
||||
('daily_report_enabled', 'true'::jsonb, 'bool', 'Daily report enabled', 'Double safety flag for daily Telegram operational report.', 'Daily Report'),
|
||||
('daily_report_bot_token', '""'::jsonb, 'secret', 'Daily report bot token', 'Telegram bot token for reports. If empty, uses tg_poster_bot_token or TG_BOT_TOKEN.', 'Daily Report'),
|
||||
('daily_report_recipient_ids', '[442509142]'::jsonb, 'json', 'Daily report recipients', 'Telegram user/chat IDs that receive the daily report.', 'Daily Report'),
|
||||
('daily_report_time', '"00:04"'::jsonb, 'str', 'Daily report time', 'EKB time HH:MM when yesterday report is sent.', 'Daily Report'),
|
||||
('daily_report_interval_sec', '60'::jsonb, 'int', 'Daily report interval, sec', 'Pause between checks.', 'Daily Report'),
|
||||
('daily_report_last_sent_date', '""'::jsonb, 'str', 'Last sent report date', 'Internal YYYY-MM-DD report date already sent automatically.', 'Daily Report')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
@@ -0,0 +1,57 @@
|
||||
ALTER TABLE content_categories
|
||||
ADD COLUMN IF NOT EXISTS site_name TEXT,
|
||||
ADD COLUMN IF NOT EXISTS site_slug TEXT,
|
||||
ADD COLUMN IF NOT EXISTS site_enabled BOOLEAN NOT NULL DEFAULT TRUE;
|
||||
|
||||
UPDATE content_categories
|
||||
SET site_name = CASE sort_order
|
||||
WHEN 1 THEN 'Одежда'
|
||||
WHEN 2 THEN 'Снаряжение'
|
||||
WHEN 3 THEN 'Броня'
|
||||
WHEN 4 THEN 'Аксессуары'
|
||||
WHEN 5 THEN 'Связь'
|
||||
WHEN 6 THEN 'Патчи'
|
||||
WHEN 7 THEN 'Навигация'
|
||||
WHEN 8 THEN 'Прицелы'
|
||||
WHEN 9 THEN 'Ночные приборы'
|
||||
WHEN 10 THEN 'Медицина'
|
||||
WHEN 11 THEN 'Свет'
|
||||
WHEN 12 THEN 'Электроника'
|
||||
WHEN 13 THEN 'Ножи'
|
||||
WHEN 14 THEN 'Оружейный обвес'
|
||||
WHEN 15 THEN 'Практика'
|
||||
WHEN 16 THEN 'Теория'
|
||||
WHEN 17 THEN 'Пушки'
|
||||
WHEN 18 THEN 'НЦК'
|
||||
ELSE initcap(replace(tag, '_', ' '))
|
||||
END,
|
||||
site_slug = CASE sort_order
|
||||
WHEN 1 THEN 'odezhda'
|
||||
WHEN 2 THEN 'snaryazhenie'
|
||||
WHEN 3 THEN 'bronya'
|
||||
WHEN 4 THEN 'aksessuary'
|
||||
WHEN 5 THEN 'svyaz'
|
||||
WHEN 6 THEN 'patchi'
|
||||
WHEN 7 THEN 'navigatsiya'
|
||||
WHEN 8 THEN 'pricely'
|
||||
WHEN 9 THEN 'nochnye-pribory'
|
||||
WHEN 10 THEN 'medicina'
|
||||
WHEN 11 THEN 'svet'
|
||||
WHEN 12 THEN 'elektronika'
|
||||
WHEN 13 THEN 'nozhi'
|
||||
WHEN 14 THEN 'oruzheynyy-obves'
|
||||
WHEN 15 THEN 'praktika'
|
||||
WHEN 16 THEN 'teoriya'
|
||||
WHEN 17 THEN 'pushki'
|
||||
WHEN 18 THEN 'nck'
|
||||
ELSE site_slug
|
||||
END,
|
||||
site_enabled = sort_order <> 18,
|
||||
updated_at = NOW();
|
||||
|
||||
ALTER TABLE content_categories
|
||||
ALTER COLUMN site_name SET NOT NULL,
|
||||
ALTER COLUMN site_slug SET NOT NULL;
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_content_categories_site_slug
|
||||
ON content_categories(site_slug);
|
||||
@@ -0,0 +1,13 @@
|
||||
INSERT INTO worker_controls(name, enabled, settings_json)
|
||||
VALUES ('site-poster', TRUE, '{}'::jsonb)
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
|
||||
INSERT INTO app_settings(key, value_json, value_type, title, description, category)
|
||||
VALUES
|
||||
('site_poster_enabled', 'true'::jsonb, 'bool', 'Site poster enabled', 'Double safety flag for automatic Ghost publishing.', 'Site Poster'),
|
||||
('site_poster_interval_sec', '300'::jsonb, 'int', 'Site poster interval, sec', 'Pause between automatic queue checks.', 'Site Poster'),
|
||||
('site_poster_photo_batch_size', '10'::jsonb, 'int', 'Photo batch size', 'Maximum photo-ready posts per automatic run.', 'Site Poster'),
|
||||
('site_poster_video_batch_size', '1'::jsonb, 'int', 'Video batch size', 'Maximum video-only posts per automatic run.', 'Site Poster'),
|
||||
('site_poster_ghost_url', '"https://f-n8.ru"'::jsonb, 'str', 'Ghost URL', 'Target Ghost public URL.', 'Site Poster'),
|
||||
('site_poster_key_file', '"/opt/vk-parser/.site-poster-key"'::jsonb, 'str', 'Ghost key file', 'Server path to the Ghost Admin API key.', 'Site Poster')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
@@ -0,0 +1,28 @@
|
||||
INSERT INTO worker_controls(name, enabled, settings_json)
|
||||
VALUES ('max-poster', FALSE, '{}'::jsonb)
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
|
||||
INSERT INTO app_settings(key, value_json, value_type, title, description, category)
|
||||
VALUES
|
||||
('max_poster_enabled', 'false'::jsonb, 'bool', 'MAX poster enabled', 'Double safety flag for publishing accepted posts to MAX.', 'MAX Poster'),
|
||||
('max_poster_bot_token', '""'::jsonb, 'secret', 'MAX poster bot token', 'Bot access token for MAX API. Keep empty in git and set on server.', 'MAX Poster'),
|
||||
('max_poster_chat_id', '""'::jsonb, 'str', 'MAX target chat id', 'Target MAX channel/group chat_id.', 'MAX Poster'),
|
||||
('max_poster_api_base_url', '"https://platform-api2.max.ru"'::jsonb, 'str', 'MAX API base URL', 'MAX Bot API base URL. Since 2026 use platform-api2.max.ru.', 'MAX Poster'),
|
||||
('max_poster_schedule_json', '[{"id":"hour-00","time":"00:00","count":4,"enabled":true},{"id":"hour-01","time":"01:00","count":4,"enabled":true},{"id":"hour-02","time":"02:00","count":4,"enabled":true},{"id":"hour-03","time":"03:00","count":4,"enabled":true},{"id":"hour-04","time":"04:00","count":4,"enabled":true},{"id":"hour-05","time":"05:00","count":4,"enabled":true},{"id":"hour-06","time":"06:00","count":4,"enabled":true},{"id":"hour-07","time":"07:00","count":4,"enabled":true},{"id":"hour-08","time":"08:00","count":4,"enabled":true},{"id":"hour-09","time":"09:00","count":4,"enabled":true},{"id":"hour-10","time":"10:00","count":4,"enabled":true},{"id":"hour-11","time":"11:00","count":4,"enabled":true},{"id":"hour-12","time":"12:00","count":4,"enabled":true},{"id":"hour-13","time":"13:00","count":4,"enabled":true},{"id":"hour-14","time":"14:00","count":4,"enabled":true},{"id":"hour-15","time":"15:00","count":4,"enabled":true},{"id":"hour-16","time":"16:00","count":4,"enabled":true},{"id":"hour-17","time":"17:00","count":4,"enabled":true},{"id":"hour-18","time":"18:00","count":4,"enabled":true},{"id":"hour-19","time":"19:00","count":4,"enabled":true},{"id":"hour-20","time":"20:00","count":4,"enabled":true},{"id":"hour-21","time":"21:00","count":4,"enabled":true},{"id":"hour-22","time":"22:00","count":4,"enabled":true},{"id":"hour-23","time":"23:00","count":4,"enabled":true}]'::jsonb, 'json', 'MAX publication schedule', 'EKB time schedule rows: id, time HH:MM, count, enabled. Defaults to the TG hourly schedule.', 'MAX Poster'),
|
||||
('max_poster_interval_sec', '60'::jsonb, 'int', 'MAX poster interval, sec', 'Pause between poster checks.', 'MAX Poster'),
|
||||
('max_poster_message_limit', '4000'::jsonb, 'int', 'MAX message limit', 'MAX text message limit.', 'MAX Poster'),
|
||||
('max_poster_media_max_items', '10'::jsonb, 'int', 'MAX media max items', 'Maximum media attachments per MAX message.', 'MAX Poster'),
|
||||
('max_poster_media_process_delay_sec', '3'::jsonb, 'float', 'MAX media process delay, sec', 'Pause after media upload before sending the message.', 'MAX Poster'),
|
||||
('max_poster_video_ready_attempts', '5'::jsonb, 'int', 'MAX video ready attempts', 'Retries for sending a message while MAX is still processing uploaded video.', 'MAX Poster'),
|
||||
('max_poster_video_ready_delay_sec', '10'::jsonb, 'float', 'MAX video ready delay, sec', 'Initial retry delay while MAX is processing uploaded video. Delay doubles after each attempt.', 'MAX Poster'),
|
||||
('max_poster_max_attempts', '3'::jsonb, 'int', 'MAX post retry attempts', 'Retries for MAX API/network errors and attachment processing.', 'MAX Poster'),
|
||||
('max_poster_retry_backoff_max_sec', '30'::jsonb, 'int', 'MAX retry backoff max, sec', 'Max retry sleep for MAX API/network errors.', 'MAX Poster'),
|
||||
('max_poster_send_delay_sec', '1'::jsonb, 'float', 'MAX send delay, sec', 'Delay between MAX sends.', 'MAX Poster'),
|
||||
('max_poster_recent_window', '20'::jsonb, 'int', 'Ranking recent window', 'How many MAX-published posts are checked to reduce repeated categories and sources.', 'MAX Poster'),
|
||||
('max_poster_category_repeat_penalty', '3'::jsonb, 'float', 'Category repeat penalty', 'Ranking penalty for recently published same category.', 'MAX Poster'),
|
||||
('max_poster_source_repeat_penalty', '4'::jsonb, 'float', 'Source repeat penalty', 'Ranking penalty for recently published same source.', 'MAX Poster'),
|
||||
('max_poster_auto_reaction_enabled', 'true'::jsonb, 'bool', 'MAX auto reaction enabled', 'Try to set a reaction after publishing. If MAX API rejects it, publication remains successful.', 'MAX Poster'),
|
||||
('max_poster_auto_reaction', '"👍"'::jsonb, 'str', 'MAX auto reaction', 'Emoji reaction to try after publishing.', 'MAX Poster'),
|
||||
('max_poster_reaction_path_template', '"/messages/{message_id}/reactions"'::jsonb, 'str', 'MAX reaction path template', 'Relative API path for reaction call if supported by MAX API.', 'MAX Poster'),
|
||||
('max_poster_dry_run', 'false'::jsonb, 'bool', 'MAX dry run', 'If true, prepares publication but does not call MAX send message.', 'MAX Poster')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
@@ -0,0 +1,15 @@
|
||||
[Unit]
|
||||
Description=VK Parser MAX Poster
|
||||
After=network.target postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/vk-parser
|
||||
Environment=PYTHONPATH=/opt/vk-parser/src
|
||||
ExecStart=/opt/vk-parser/.venv/bin/python -m vk_parser_app.workers.max_poster
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
User=root
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,14 @@
|
||||
[Unit]
|
||||
Description=VK Parser Site Poster
|
||||
After=network.target postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/vk-parser
|
||||
Environment=PYTHONPATH=/opt/vk-parser/src
|
||||
ExecStart=/opt/vk-parser/.venv/bin/python -m vk_parser_app.workers.site_poster
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 148 KiB |
@@ -0,0 +1,8 @@
|
||||
DATABASE_URL=postgresql://forma_n8:change-me@127.0.0.1:5432/forma_n8
|
||||
SITE_BASE_URL=http://127.0.0.1:8090
|
||||
SITE_NAME=Forma N8
|
||||
SITE_TAGLINE=Новости снаряжения
|
||||
SITE_INDEXING_ENABLED=false
|
||||
PUBLISH_API_TOKEN=change-me
|
||||
UPLOAD_DIR=/var/lib/forma-n8/uploads
|
||||
PAGE_SIZE=12
|
||||
@@ -0,0 +1,55 @@
|
||||
# Forma N8 Site
|
||||
|
||||
Отдельное SSR-приложение публичного сайта Forma N8. Контент и постоянные URL не зависят от будущего дизайна.
|
||||
|
||||
## Возможности
|
||||
|
||||
- лента новостей, страницы статей, категорий и производителей;
|
||||
- PostgreSQL со стабильными `external_id` и `slug`;
|
||||
- idempotent API публикации и отдельный upload API;
|
||||
- RSS 2.0, sitemap, robots.txt;
|
||||
- canonical, Open Graph и JSON-LD `NewsArticle`;
|
||||
- системный деплой через nginx + systemd.
|
||||
|
||||
## Локальный запуск
|
||||
|
||||
```bash
|
||||
python -m venv .venv
|
||||
.venv/bin/pip install -r requirements.txt
|
||||
export DATABASE_URL=postgresql://forma_n8:forma_n8@127.0.0.1:5432/forma_n8
|
||||
PYTHONPATH=src .venv/bin/python scripts/migrate.py
|
||||
PYTHONPATH=src .venv/bin/uvicorn forma_n8_site.app:app --port 8090
|
||||
```
|
||||
|
||||
## Публикационный API
|
||||
|
||||
Сначала медиа загружается через `POST /api/v1/media` с заголовком `X-Forma-Token`, затем полученный относительный URL передаётся в статью.
|
||||
|
||||
`POST /api/v1/articles`:
|
||||
|
||||
```json
|
||||
{
|
||||
"external_id": "parser:518",
|
||||
"title": "Заголовок новости",
|
||||
"lead": "Краткое описание",
|
||||
"body": "Основной текст без HTML",
|
||||
"category_name": "Снаряжение",
|
||||
"category_tag": "snaryazhenie",
|
||||
"producer_name": "Производитель",
|
||||
"producer_tag": "producer",
|
||||
"source_url": "https://example.com/source",
|
||||
"status": "published",
|
||||
"published_at": "2026-06-21T10:00:00+05:00",
|
||||
"media": [
|
||||
{"type": "image", "url": "/media/ab/hash.jpg", "alt": "Описание изображения"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Повторный запрос с тем же `external_id` обновляет материал, но сохраняет первоначальные slug и дату первой публикации. Это предотвращает дубли URL и потерю поисковой истории.
|
||||
|
||||
Публичное название категории берётся из таблицы `categories` по нормализованному `category_tag`. Описательные названия категорий из AI-контура не используются в URL, H1 и SEO-метаданных.
|
||||
|
||||
После подключения домена меняется только `SITE_BASE_URL` в `/etc/forma-n8.env`; затем настраивается TLS и отправляются sitemap в Google Search Console и Яндекс Вебмастер.
|
||||
|
||||
До подключения домена `SITE_INDEXING_ENABLED=false`: страницы получают `noindex`, а `robots.txt` запрещает обход IP-адреса. После настройки домена, HTTPS и 301-редиректа с IP значение меняется на `true`.
|
||||
@@ -0,0 +1,46 @@
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version TEXT PRIMARY KEY,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS articles (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
external_id TEXT NOT NULL UNIQUE,
|
||||
slug TEXT NOT NULL UNIQUE,
|
||||
title TEXT NOT NULL,
|
||||
lead TEXT NOT NULL DEFAULT '',
|
||||
body TEXT NOT NULL,
|
||||
category_name TEXT NOT NULL,
|
||||
category_tag TEXT NOT NULL,
|
||||
producer_name TEXT NOT NULL,
|
||||
producer_tag TEXT NOT NULL,
|
||||
source_url TEXT,
|
||||
seo_title TEXT NOT NULL DEFAULT '',
|
||||
seo_description TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL DEFAULT 'draft' CHECK (status IN ('draft', 'published')),
|
||||
published_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_articles_feed
|
||||
ON articles(status, published_at DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_articles_category
|
||||
ON articles(category_tag, status, published_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_articles_producer
|
||||
ON articles(producer_tag, status, published_at DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS article_media (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
article_id BIGINT NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
|
||||
media_type TEXT NOT NULL CHECK (media_type IN ('image', 'video')),
|
||||
url TEXT NOT NULL,
|
||||
alt_text TEXT NOT NULL DEFAULT '',
|
||||
width INTEGER,
|
||||
height INTEGER,
|
||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_article_media_order
|
||||
ON article_media(article_id, sort_order, id);
|
||||
@@ -0,0 +1,35 @@
|
||||
CREATE TABLE IF NOT EXISTS categories (
|
||||
tag TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
sort_order INTEGER NOT NULL DEFAULT 100,
|
||||
is_active BOOLEAN NOT NULL DEFAULT TRUE
|
||||
);
|
||||
|
||||
INSERT INTO categories(tag, name, sort_order) VALUES
|
||||
('odezhda', 'Одежда', 1),
|
||||
('snaryazhenie', 'Снаряжение', 2),
|
||||
('bronya', 'Броня', 3),
|
||||
('aksessuary', 'Аксессуары', 4),
|
||||
('svyaz', 'Связь', 5),
|
||||
('patchi', 'Патчи', 6),
|
||||
('navigaciya', 'Навигация', 7),
|
||||
('pricely', 'Прицелы', 8),
|
||||
('nochnye-pribory', 'Ночные приборы', 9),
|
||||
('medicina', 'Медицина', 10),
|
||||
('svet', 'Свет', 11),
|
||||
('elektronika', 'Электроника', 12),
|
||||
('nozhi', 'Ножи и мультитулы', 13),
|
||||
('oruzheynyy-obves', 'Оружейный обвес', 14),
|
||||
('praktika', 'Практика', 15),
|
||||
('teoriya', 'Теория', 16),
|
||||
('pushki', 'Оружие', 17)
|
||||
ON CONFLICT(tag) DO UPDATE SET
|
||||
name=EXCLUDED.name,
|
||||
sort_order=EXCLUDED.sort_order;
|
||||
|
||||
UPDATE articles a
|
||||
SET category_name=c.name,
|
||||
updated_at=NOW()
|
||||
FROM categories c
|
||||
WHERE a.category_tag=c.tag
|
||||
AND a.category_name<>c.name;
|
||||
@@ -0,0 +1,23 @@
|
||||
[Unit]
|
||||
Description=Forma N8 public website
|
||||
After=network.target postgresql.service
|
||||
Wants=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=forma-n8
|
||||
Group=forma-n8
|
||||
WorkingDirectory=/opt/forma-n8
|
||||
EnvironmentFile=/etc/forma-n8.env
|
||||
Environment=PYTHONPATH=/opt/forma-n8/src
|
||||
ExecStart=/opt/forma-n8/.venv/bin/uvicorn forma_n8_site.app:app --host 127.0.0.1 --port 8090 --proxy-headers
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ReadWritePaths=/var/lib/forma-n8
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,31 @@
|
||||
server {
|
||||
listen 80 default_server;
|
||||
listen [::]:80 default_server;
|
||||
server_name _;
|
||||
|
||||
client_max_body_size 52m;
|
||||
|
||||
location /static/ {
|
||||
alias /opt/forma-n8/static/;
|
||||
expires 30d;
|
||||
add_header Cache-Control "public, immutable";
|
||||
access_log off;
|
||||
}
|
||||
|
||||
location /media/ {
|
||||
alias /var/lib/forma-n8/uploads/;
|
||||
expires 30d;
|
||||
add_header Cache-Control "public";
|
||||
add_header X-Content-Type-Options "nosniff";
|
||||
}
|
||||
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:8090;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_read_timeout 60s;
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.8 MiB |
@@ -0,0 +1,431 @@
|
||||
@font-face {
|
||||
font-family: "Roboto Condensed";
|
||||
src: url("../fonts/roboto-condensed-cyrillic-400.woff2") format("woff2");
|
||||
font-weight: 400;
|
||||
font-display: swap;
|
||||
}
|
||||
|
||||
@font-face {
|
||||
font-family: "Roboto Condensed";
|
||||
src: url("../fonts/roboto-condensed-cyrillic-700.woff2") format("woff2");
|
||||
font-weight: 700;
|
||||
font-display: swap;
|
||||
}
|
||||
|
||||
:root {
|
||||
color-scheme: dark;
|
||||
--bg: #090c0d;
|
||||
--surface: #101415;
|
||||
--surface-strong: #151a1b;
|
||||
--text: #eceee8;
|
||||
--muted: #9ba19b;
|
||||
--faint: #686f6b;
|
||||
--border: #29302e;
|
||||
--border-strong: #3a423f;
|
||||
--olive: #a5ad62;
|
||||
--red: #df4a40;
|
||||
--max: 1240px;
|
||||
--gutter: 24px;
|
||||
--font: "Roboto Condensed", "Arial Narrow", Arial, sans-serif;
|
||||
}
|
||||
|
||||
* { box-sizing: border-box; }
|
||||
|
||||
html { background: var(--bg); scroll-behavior: smooth; }
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
min-width: 320px;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
font-family: var(--font);
|
||||
font-size: 18px;
|
||||
line-height: 1.55;
|
||||
letter-spacing: 0;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
}
|
||||
|
||||
body::before {
|
||||
content: "";
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
pointer-events: none;
|
||||
opacity: .22;
|
||||
background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='120' height='120' viewBox='0 0 120 120'%3E%3Cfilter id='n'%3E%3CfeTurbulence type='fractalNoise' baseFrequency='.8' numOctaves='2' stitchTiles='stitch'/%3E%3C/filter%3E%3Crect width='100%25' height='100%25' filter='url(%23n)' opacity='.08'/%3E%3C/svg%3E");
|
||||
}
|
||||
|
||||
a { color: inherit; text-decoration: none; }
|
||||
a:hover { color: var(--olive); }
|
||||
img, video { display: block; max-width: 100%; }
|
||||
button, input { font: inherit; letter-spacing: 0; }
|
||||
|
||||
:focus-visible { outline: 2px solid var(--olive); outline-offset: 4px; }
|
||||
|
||||
.skip-link {
|
||||
position: fixed;
|
||||
z-index: 100;
|
||||
top: 10px;
|
||||
left: 10px;
|
||||
transform: translateY(-160%);
|
||||
padding: 10px 14px;
|
||||
background: var(--text);
|
||||
color: var(--bg);
|
||||
}
|
||||
.skip-link:focus { transform: none; }
|
||||
|
||||
.site-header {
|
||||
position: sticky;
|
||||
z-index: 20;
|
||||
top: 0;
|
||||
border-bottom: 1px solid var(--border);
|
||||
background: rgba(9, 12, 13, .96);
|
||||
backdrop-filter: blur(12px);
|
||||
}
|
||||
|
||||
.header-inner {
|
||||
width: min(var(--max), calc(100% - var(--gutter) * 2));
|
||||
min-height: 76px;
|
||||
margin: 0 auto;
|
||||
display: grid;
|
||||
grid-template-columns: 170px 1fr auto;
|
||||
align-items: center;
|
||||
gap: 32px;
|
||||
}
|
||||
|
||||
.brand {
|
||||
display: inline-flex;
|
||||
width: fit-content;
|
||||
align-items: center;
|
||||
gap: 11px;
|
||||
font-size: 15px;
|
||||
font-weight: 700;
|
||||
line-height: .95;
|
||||
}
|
||||
.brand img { width: 42px; height: 42px; border: 1px solid var(--border-strong); }
|
||||
|
||||
.main-nav { display: flex; justify-content: center; align-self: stretch; gap: 38px; }
|
||||
.main-nav .nav { display: contents; margin: 0; padding: 0; list-style: none; }
|
||||
.main-nav .nav li { display: contents; }
|
||||
.nav-dropdown { position: relative; display: flex; align-items: center; }
|
||||
.nav-dropdown-toggle { display: inline-flex; height: 100%; align-items: center; gap: 8px; padding: 0; border: 0; background: transparent; color: var(--muted); cursor: pointer; }
|
||||
.nav-dropdown-toggle:hover, .nav-dropdown-toggle[aria-expanded="true"] { color: var(--text); }
|
||||
.nav-dropdown-icon { width: 7px; height: 7px; border-right: 1.5px solid currentColor; border-bottom: 1.5px solid currentColor; transform: translateY(-2px) rotate(45deg); transition: transform .18s ease; }
|
||||
.nav-dropdown-toggle[aria-expanded="true"] .nav-dropdown-icon { transform: translateY(2px) rotate(225deg); }
|
||||
.category-menu { position: absolute; z-index: 30; top: calc(100% - 1px); left: 50%; display: none; grid-template-columns: repeat(2, minmax(170px, 1fr)); width: 430px; padding: 12px; border: 1px solid var(--border-strong); background: var(--bg); transform: translateX(-50%); box-shadow: 0 18px 44px rgba(0,0,0,.38); }
|
||||
.category-menu.is-open { display: grid; }
|
||||
.category-menu a { min-height: 40px; padding: 9px 11px; border-bottom: 1px solid var(--border); color: var(--muted); }
|
||||
.category-menu a:hover { background: var(--surface-strong); color: var(--text); }
|
||||
.main-nav a {
|
||||
position: relative;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
color: var(--muted);
|
||||
font-size: 16px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.main-nav a:hover, .main-nav a[aria-current="page"] { color: var(--text); }
|
||||
.main-nav a[aria-current="page"]::after {
|
||||
content: "";
|
||||
position: absolute;
|
||||
right: 0;
|
||||
bottom: -1px;
|
||||
left: 0;
|
||||
height: 3px;
|
||||
background: var(--olive);
|
||||
}
|
||||
|
||||
.header-actions { display: flex; gap: 8px; }
|
||||
.header-actions > .social-links { margin-right: 5px; padding-right: 10px; border-right: 1px solid var(--border); }
|
||||
.icon-button {
|
||||
display: grid;
|
||||
width: 40px;
|
||||
height: 40px;
|
||||
padding: 0;
|
||||
place-items: center;
|
||||
border: 0;
|
||||
background: transparent;
|
||||
color: var(--text);
|
||||
cursor: pointer;
|
||||
}
|
||||
.icon-button:hover { background: var(--surface-strong); color: var(--olive); }
|
||||
.icon-button svg { width: 23px; height: 23px; fill: none; stroke: currentColor; stroke-width: 1.8; stroke-linecap: square; }
|
||||
.menu-button { display: none; }
|
||||
|
||||
.search-form {
|
||||
width: min(760px, calc(100% - var(--gutter) * 2));
|
||||
margin: 0 auto;
|
||||
padding: 16px 0 20px;
|
||||
}
|
||||
.search-form[hidden] { display: none; }
|
||||
.search-form label { display: block; margin-bottom: 8px; color: var(--muted); font-size: 14px; }
|
||||
.search-form > div { display: grid; grid-template-columns: 1fr auto; }
|
||||
.search-form input {
|
||||
min-width: 0;
|
||||
height: 48px;
|
||||
padding: 0 16px;
|
||||
border: 1px solid var(--border-strong);
|
||||
border-right: 0;
|
||||
border-radius: 0;
|
||||
background: var(--surface);
|
||||
color: var(--text);
|
||||
font-size: 17px;
|
||||
}
|
||||
.search-form button, .read-link, .empty-state a {
|
||||
border: 1px solid var(--olive);
|
||||
border-radius: 0;
|
||||
background: transparent;
|
||||
color: var(--text);
|
||||
cursor: pointer;
|
||||
}
|
||||
.search-form button { padding: 0 24px; }
|
||||
.search-form button:hover, .read-link:hover, .empty-state a:hover { background: var(--olive); color: var(--bg); }
|
||||
|
||||
.feed-shell, .article-page, .related-section, .site-footer {
|
||||
position: relative;
|
||||
width: min(var(--max), calc(100% - var(--gutter) * 2));
|
||||
margin: 0 auto;
|
||||
}
|
||||
|
||||
.page-heading { padding: 56px 0 46px; }
|
||||
.page-heading h1 {
|
||||
max-width: 900px;
|
||||
margin: 0;
|
||||
font-size: 68px;
|
||||
font-weight: 700;
|
||||
line-height: .98;
|
||||
}
|
||||
@media (min-width: 981px) {
|
||||
.home-template .page-heading {
|
||||
position: relative;
|
||||
left: 50%;
|
||||
width: min(1600px, calc(100vw - var(--gutter) * 2));
|
||||
transform: translateX(-50%);
|
||||
}
|
||||
.home-template .page-heading h1 {
|
||||
max-width: none;
|
||||
font-size: clamp(58px, 4vw, 68px);
|
||||
text-align: center;
|
||||
}
|
||||
}
|
||||
.page-heading > p { margin: 20px 0 0; color: var(--muted); }
|
||||
|
||||
.lead-story {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1.25fr) minmax(340px, .95fr);
|
||||
min-height: 480px;
|
||||
max-height: 680px;
|
||||
border-bottom: 1px solid var(--border-strong);
|
||||
}
|
||||
.lead-media { min-height: 480px; max-height: 680px; overflow: hidden; background: var(--surface); }
|
||||
.lead-media img, .lead-media video { width: 100%; height: 100%; object-fit: cover; transition: transform .45s ease; }
|
||||
.lead-media:hover img { transform: scale(1.015); }
|
||||
.lead-copy { display: flex; flex-direction: column; justify-content: center; padding: 34px 0 36px 44px; }
|
||||
.lead-copy h2 { margin: 24px 0 22px; font-size: 40px; line-height: 1.14; }
|
||||
.lead-copy h2 a:hover { color: var(--text); text-decoration: underline; text-decoration-color: var(--olive); text-underline-offset: 7px; }
|
||||
.lead-copy > p { max-width: 580px; margin: 0; color: var(--muted); font-size: 20px; line-height: 1.5; }
|
||||
.read-link { width: fit-content; margin-top: 30px; padding: 10px 15px; font-size: 15px; }
|
||||
.read-link span { margin-left: 8px; }
|
||||
|
||||
.article-meta { display: flex; align-items: center; flex-wrap: wrap; gap: 0; color: var(--muted); font-size: 14px; }
|
||||
.article-meta > * { display: inline-flex; align-items: center; }
|
||||
.article-meta > * + *::before { content: ""; width: 1px; height: 18px; margin: 0 16px; background: var(--border-strong); }
|
||||
.article-meta .category-link { color: var(--olive); }
|
||||
.article-meta .producer-link { color: var(--red); }
|
||||
|
||||
.feed-list { width: 100%; }
|
||||
.feed-list-top { border-top: 1px solid var(--border-strong); }
|
||||
.feed-row {
|
||||
display: grid;
|
||||
grid-template-columns: 350px minmax(0, 1fr);
|
||||
gap: 34px;
|
||||
padding: 22px 0;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
.feed-row-media { aspect-ratio: 16 / 9; overflow: hidden; background: var(--surface); }
|
||||
.feed-row-media img, .feed-row-media video { width: 100%; height: 100%; object-fit: cover; transition: transform .35s ease; }
|
||||
.feed-row-media:hover img { transform: scale(1.02); }
|
||||
.feed-row-copy { align-self: center; min-width: 0; }
|
||||
.feed-row h2 { margin: 16px 0 8px; font-size: 25px; line-height: 1.2; }
|
||||
.feed-row h2 a:hover { color: var(--text); text-decoration: underline; text-decoration-color: var(--olive); text-underline-offset: 5px; }
|
||||
.feed-row p { max-width: 760px; margin: 0; color: var(--muted); font-size: 17px; line-height: 1.5; }
|
||||
.preview-social { margin-top: 14px; }
|
||||
.media-placeholder { display: grid; width: 100%; height: 100%; place-items: center; background: var(--surface); }
|
||||
.media-placeholder img { width: 76px; height: 76px; opacity: .28; object-fit: contain; }
|
||||
.feed-media { position: relative; }
|
||||
.feed-media-feature, .feed-media-feature > img, .feed-media-feature > .media-placeholder { display: block; width: 100%; height: 100%; }
|
||||
.feed-media-track { display: flex; width: 100%; height: 100%; overflow-x: auto; overflow-y: hidden; scroll-snap-type: x mandatory; scrollbar-width: none; }
|
||||
.feed-media-track::-webkit-scrollbar { display: none; }
|
||||
.feed-media-slide { flex: 0 0 100%; width: 100%; height: 100%; scroll-snap-align: start; }
|
||||
.feed-media-slide > a, .feed-media-slide img, .feed-media-slide video { display: block; width: 100%; height: 100%; }
|
||||
.feed-media-slide img, .feed-media-slide video { object-fit: cover; }
|
||||
.feed-media-count { position: absolute; top: 10px; right: 10px; z-index: 2; padding: 4px 8px; border-radius: 999px; background: rgba(0,0,0,.68); color: #fff; font-size: 12px; line-height: 1; pointer-events: none; }
|
||||
|
||||
.pagination { display: grid; grid-template-columns: 1fr auto 1fr; align-items: center; min-height: 90px; font-size: 15px; }
|
||||
.pagination > :last-child { justify-self: end; }
|
||||
.pagination span { color: var(--faint); }
|
||||
.pagination .older-posts { grid-column: 3; justify-self: end; }
|
||||
.pagination .newer-posts { grid-column: 1; }
|
||||
.pagination .page-number { grid-column: 2; grid-row: 1; }
|
||||
|
||||
.home-seo {
|
||||
max-width: 960px;
|
||||
margin: 54px auto 0;
|
||||
padding: 40px 0 0;
|
||||
border-top: 1px solid var(--border-strong);
|
||||
}
|
||||
.home-seo h2 { margin: 0 0 24px; font-size: 34px; line-height: 1.15; }
|
||||
.home-seo p { margin: 0 0 18px; color: var(--muted); font-size: 17px; line-height: 1.7; }
|
||||
.home-seo p:last-child { margin-bottom: 0; }
|
||||
|
||||
.taxonomy-heading { padding-bottom: 34px; }
|
||||
.taxonomy-heading > p { margin: 0 0 10px; color: var(--olive); font-size: 15px; text-transform: uppercase; }
|
||||
.taxonomy-heading h1 { font-size: 56px; }
|
||||
|
||||
.article-page { padding-top: 56px; }
|
||||
.article-header { max-width: 960px; margin: 0 auto 42px; }
|
||||
.article-breadcrumbs { display: flex; gap: 12px; margin-bottom: 28px; color: var(--muted); font-size: 14px; }
|
||||
.article-breadcrumbs span { color: var(--faint); }
|
||||
.article-header h1 { margin: 0; font-size: 58px; line-height: 1.06; }
|
||||
.article-lead { max-width: 850px; margin: 26px 0 0; color: var(--muted); font-size: 23px; line-height: 1.45; }
|
||||
.article-byline { display: flex; gap: 24px; margin-top: 28px; color: var(--muted); font-size: 14px; }
|
||||
.article-byline a { color: var(--red); }
|
||||
.article-media { display: grid; gap: 12px; margin-bottom: 46px; }
|
||||
.article-media figure { margin: 0; background: var(--surface); }
|
||||
.article-media img, .article-media video { width: 100%; max-height: 760px; object-fit: contain; }
|
||||
.article-layout { display: grid; grid-template-columns: minmax(0, 760px) 260px; justify-content: center; gap: 78px; }
|
||||
.article-body { font-size: 21px; line-height: 1.7; }
|
||||
.article-body p { margin: 0 0 26px; }
|
||||
.gh-content > * { max-width: 100%; }
|
||||
.gh-content a { color: var(--olive); text-decoration: underline; text-underline-offset: 3px; }
|
||||
.gh-content figure, .gh-content .kg-card { margin: 32px 0; }
|
||||
.gh-content img, .gh-content video { width: 100%; height: auto; }
|
||||
.gh-content blockquote { margin: 30px 0; padding-left: 24px; border-left: 3px solid var(--olive); color: var(--muted); }
|
||||
.kg-width-wide, .kg-width-full { width: 100%; }
|
||||
.kg-gallery-container { display: flex; flex-direction: column; gap: 10px; }
|
||||
.kg-gallery-row { display: flex; gap: 10px; }
|
||||
.kg-gallery-image { flex: 1 1 0; min-width: 0; margin: 0; overflow: hidden; background: var(--surface); }
|
||||
.kg-gallery-image img { width: 100%; height: 100%; margin: 0; object-fit: cover; cursor: zoom-in; }
|
||||
.media-lightbox { width: 100%; height: 100%; max-width: none; max-height: none; padding: 0; border: 0; background: rgba(0,0,0,.94); }
|
||||
.media-lightbox::backdrop { background: rgba(0,0,0,.94); }
|
||||
.media-lightbox img { width: 100%; height: 100%; object-fit: contain; }
|
||||
.media-lightbox button { position: fixed; top: 18px; right: 18px; z-index: 2; width: 44px; height: 44px; border: 1px solid var(--border-strong); background: var(--bg); color: var(--text); font-size: 28px; cursor: pointer; }
|
||||
.article-facts { align-self: start; border-top: 2px solid var(--olive); }
|
||||
.article-facts > div { padding: 18px 0; border-bottom: 1px solid var(--border); }
|
||||
.article-facts span { display: block; margin-bottom: 4px; color: var(--faint); font-size: 13px; text-transform: uppercase; }
|
||||
.article-facts a { font-size: 17px; }
|
||||
.article-facts .source-link { display: block; margin-top: 22px; color: var(--muted); font-size: 14px; }
|
||||
.social-links { display: flex; align-items: center; flex-wrap: wrap; gap: 8px; margin-top: 13px; }
|
||||
.social-link { display: inline-flex; min-height: 34px; align-items: center; gap: 7px; padding: 5px 10px; border: 1px solid var(--border); color: var(--muted); font-size: 14px; line-height: 1; transition: border-color .18s ease, color .18s ease, background .18s ease; }
|
||||
.social-link:hover { border-color: var(--border-strong); background: var(--surface-strong); color: var(--text); }
|
||||
.social-link svg { width: 18px; height: 18px; fill: currentColor; }
|
||||
.social-link--vk:hover { color: #7ca8e8; }
|
||||
.social-link--tg:hover { color: #58b9e8; }
|
||||
.social-links--compact { gap: 4px; margin-top: 0; }
|
||||
.social-links--compact .social-link { width: 34px; min-height: 34px; justify-content: center; padding: 0; }
|
||||
.social-links--compact .social-link svg { width: 17px; height: 17px; }
|
||||
.about-social { margin-top: 42px; padding-top: 24px; border-top: 1px solid var(--border-strong); }
|
||||
.about-social > span { color: var(--faint); font-size: 13px; text-transform: uppercase; }
|
||||
.related-section { margin-top: 90px; border-top: 1px solid var(--border-strong); }
|
||||
.related-section > h2 { margin: 0; padding: 34px 0 18px; font-size: 32px; }
|
||||
|
||||
.empty-state, .not-found { min-height: 56vh; display: flex; flex-direction: column; justify-content: center; align-items: flex-start; }
|
||||
.empty-state h2, .not-found h1 { margin: 0; font-size: 42px; }
|
||||
.empty-state p { color: var(--muted); }
|
||||
.empty-state a { margin-top: 16px; padding: 10px 15px; }
|
||||
.not-found > p { margin: 0; color: var(--red); font-size: 18px; }
|
||||
.not-found a { margin-top: 24px; color: var(--olive); }
|
||||
|
||||
.site-footer {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: flex-end;
|
||||
margin-top: 84px;
|
||||
padding: 34px 0 46px;
|
||||
border-top: 1px solid var(--border-strong);
|
||||
color: var(--muted);
|
||||
font-size: 14px;
|
||||
}
|
||||
.footer-brand { color: var(--text); font-size: 20px; font-weight: 700; }
|
||||
.site-footer p { margin: 7px 0 0; }
|
||||
.site-footer nav { display: flex; gap: 24px; }
|
||||
.site-footer .social-links { margin-top: 18px; }
|
||||
|
||||
@media (max-width: 980px) {
|
||||
.header-inner { grid-template-columns: 150px 1fr auto; gap: 16px; }
|
||||
.main-nav { gap: 22px; }
|
||||
.main-nav a { font-size: 14px; }
|
||||
.page-heading h1 { font-size: 56px; }
|
||||
.lead-story { grid-template-columns: 1.1fr .9fr; min-height: 420px; }
|
||||
.lead-media { min-height: 420px; }
|
||||
.lead-copy { padding-left: 30px; }
|
||||
.lead-copy h2 { font-size: 33px; }
|
||||
.feed-row { grid-template-columns: 290px minmax(0, 1fr); gap: 26px; }
|
||||
.article-layout { grid-template-columns: minmax(0, 680px) 220px; gap: 44px; }
|
||||
}
|
||||
|
||||
@media (max-width: 760px) {
|
||||
:root { --gutter: 16px; }
|
||||
body { font-size: 17px; }
|
||||
.home-seo { margin-top: 38px; padding-top: 30px; }
|
||||
.home-seo h2 { font-size: 28px; }
|
||||
.home-seo p { font-size: 16px; line-height: 1.65; }
|
||||
.header-inner { min-height: 64px; grid-template-columns: 1fr auto; }
|
||||
.brand img { width: 38px; height: 38px; }
|
||||
.main-nav {
|
||||
position: absolute;
|
||||
top: 64px;
|
||||
right: 0;
|
||||
left: 0;
|
||||
display: none;
|
||||
padding: 12px var(--gutter) 18px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
background: var(--bg);
|
||||
}
|
||||
.main-nav.is-open { display: grid; }
|
||||
.main-nav a { min-height: 44px; border-bottom: 1px solid var(--border); font-size: 16px; }
|
||||
.nav-dropdown { display: block; }
|
||||
.nav-dropdown-toggle { width: 100%; min-height: 44px; border-bottom: 1px solid var(--border); text-align: left; font-size: 16px; }
|
||||
.category-menu { position: static; width: 100%; grid-template-columns: 1fr; padding: 0 0 8px 14px; border: 0; box-shadow: none; transform: none; }
|
||||
.category-menu a { font-size: 15px; }
|
||||
.main-nav a[aria-current="page"]::after { display: none; }
|
||||
.menu-button { display: grid; }
|
||||
.page-heading { padding: 38px 0 30px; }
|
||||
.page-heading h1, .taxonomy-heading h1 { font-size: 44px; line-height: 1; }
|
||||
.lead-story { display: block; min-height: 0; max-height: none; }
|
||||
.lead-media { min-height: 0; max-height: min(72vh, 560px); aspect-ratio: 4 / 3; }
|
||||
.lead-copy { padding: 25px 0 34px; }
|
||||
.lead-copy h2 { margin: 20px 0 15px; font-size: 32px; }
|
||||
.lead-copy > p { font-size: 18px; }
|
||||
.feed-row { display: block; padding: 24px 0; }
|
||||
.feed-row-media { width: 100%; max-height: min(72vh, 560px); aspect-ratio: 4 / 3; }
|
||||
.feed-row-copy { padding-top: 16px; }
|
||||
.feed-row h2 { margin: 10px 0 8px; font-size: 24px; }
|
||||
.feed-row p { display: block; font-size: 16px; }
|
||||
.preview-social { margin-top: 10px; }
|
||||
.preview-social .social-link { width: 30px; min-height: 30px; }
|
||||
.article-meta { font-size: 12px; }
|
||||
.article-meta time { display: none; }
|
||||
.article-meta > * + *::before { height: 14px; margin: 0 10px; }
|
||||
.article-meta .category-link::before { display: none; }
|
||||
.article-header { margin-bottom: 30px; }
|
||||
.article-header h1 { font-size: 42px; }
|
||||
.article-lead { font-size: 20px; }
|
||||
.article-layout { display: block; }
|
||||
.article-body { font-size: 19px; }
|
||||
.article-facts { margin-top: 42px; }
|
||||
.related-section { margin-top: 64px; }
|
||||
}
|
||||
|
||||
@media (max-width: 460px) {
|
||||
.feed-row h2 { font-size: 22px; line-height: 1.2; }
|
||||
.article-meta .producer-link::before { display: none; }
|
||||
.article-meta .category-link { display: none; }
|
||||
.article-header h1 { font-size: 36px; }
|
||||
.article-byline { display: grid; gap: 7px; }
|
||||
.site-footer { display: grid; gap: 24px; }
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
html { scroll-behavior: auto; }
|
||||
*, *::before, *::after { transition-duration: .01ms !important; }
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 46 KiB |
@@ -0,0 +1,86 @@
|
||||
(() => {
|
||||
document.querySelectorAll('[data-feed-media]').forEach((container) => {
|
||||
const source = container.querySelector('.feed-media-source');
|
||||
const feature = container.querySelector('.feed-media-feature');
|
||||
if (!source || !feature) return;
|
||||
|
||||
const videos = [...source.querySelectorAll('video')].map((item) => {
|
||||
const video = item.cloneNode(true);
|
||||
video.removeAttribute('poster');
|
||||
video.controls = true;
|
||||
video.playsInline = true;
|
||||
video.preload = 'metadata';
|
||||
return video;
|
||||
});
|
||||
const images = [...source.querySelectorAll('.kg-gallery-card img, .kg-image-card img')].map((item) => {
|
||||
const link = document.createElement('a');
|
||||
link.href = item.currentSrc || item.src;
|
||||
link.setAttribute('aria-label', item.alt ? `Открыть: ${item.alt}` : 'Открыть изображение');
|
||||
link.appendChild(item.cloneNode(true));
|
||||
return link;
|
||||
});
|
||||
|
||||
const items = videos.length ? [...videos, ...images] : [feature, ...images];
|
||||
if (items.length === 1 && !videos.length) return;
|
||||
const track = document.createElement('div');
|
||||
track.className = 'feed-media-track';
|
||||
items.forEach((item) => {
|
||||
const slide = document.createElement('div');
|
||||
slide.className = 'feed-media-slide';
|
||||
slide.appendChild(item);
|
||||
track.appendChild(slide);
|
||||
});
|
||||
container.replaceChildren(track, source);
|
||||
if (items.length > 1) {
|
||||
const count = document.createElement('span');
|
||||
count.className = 'feed-media-count';
|
||||
count.textContent = `1 / ${items.length}`;
|
||||
container.appendChild(count);
|
||||
track.addEventListener('scroll', () => {
|
||||
const index = Math.round(track.scrollLeft / Math.max(track.clientWidth, 1));
|
||||
count.textContent = `${Math.min(index + 1, items.length)} / ${items.length}`;
|
||||
}, { passive: true });
|
||||
}
|
||||
});
|
||||
})();
|
||||
|
||||
(() => {
|
||||
const button = document.querySelector('[data-menu-toggle]');
|
||||
const nav = document.querySelector('#main-nav');
|
||||
if (!button || !nav) return;
|
||||
button.addEventListener('click', () => {
|
||||
const open = nav.classList.toggle('is-open');
|
||||
button.setAttribute('aria-expanded', String(open));
|
||||
});
|
||||
})();
|
||||
|
||||
(() => {
|
||||
const toggle = document.querySelector('[data-category-toggle]');
|
||||
const menu = document.querySelector('[data-category-menu]');
|
||||
if (!toggle || !menu) return;
|
||||
const close = () => { menu.classList.remove('is-open'); toggle.setAttribute('aria-expanded', 'false'); };
|
||||
toggle.addEventListener('click', (event) => {
|
||||
event.stopPropagation();
|
||||
const open = menu.classList.toggle('is-open');
|
||||
toggle.setAttribute('aria-expanded', String(open));
|
||||
});
|
||||
document.addEventListener('click', (event) => { if (!menu.contains(event.target)) close(); });
|
||||
document.addEventListener('keydown', (event) => { if (event.key === 'Escape') close(); });
|
||||
})();
|
||||
|
||||
(() => {
|
||||
const images = document.querySelectorAll('.gh-content .kg-gallery-image img');
|
||||
if (!images.length || !('HTMLDialogElement' in window)) return;
|
||||
const dialog = document.createElement('dialog');
|
||||
dialog.className = 'media-lightbox';
|
||||
dialog.innerHTML = '<button type="button" aria-label="Закрыть">×</button><img alt="">';
|
||||
const preview = dialog.querySelector('img');
|
||||
dialog.querySelector('button').addEventListener('click', () => dialog.close());
|
||||
dialog.addEventListener('click', (event) => { if (event.target === dialog) dialog.close(); });
|
||||
document.body.appendChild(dialog);
|
||||
images.forEach((image) => image.addEventListener('click', () => {
|
||||
preview.src = image.currentSrc || image.src;
|
||||
preview.alt = image.alt || '';
|
||||
dialog.showModal();
|
||||
}));
|
||||
})();
|
||||
@@ -0,0 +1,81 @@
|
||||
<!doctype html>
|
||||
<html lang="ru">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<!-- Google Tag Manager -->
|
||||
<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
|
||||
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
|
||||
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
|
||||
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
|
||||
})(window,document,'script','dataLayer','GTM-MM4FR8LJ');</script>
|
||||
<!-- End Google Tag Manager -->
|
||||
{{#is "home"}}
|
||||
<title>Forma N8 — все новости снаряжения в одном месте</title>
|
||||
{{else}}
|
||||
{{#is "post"}}
|
||||
{{#post}}<title>Forma N8 — {{title}}</title>{{/post}}
|
||||
{{else}}
|
||||
{{#is "tag"}}
|
||||
{{#tag}}<title>Forma N8 — {{name}}</title>{{/tag}}
|
||||
{{else}}
|
||||
<title>Forma N8 — {{meta_title}}</title>
|
||||
{{/is}}
|
||||
{{/is}}
|
||||
{{/is}}
|
||||
<link rel="icon" type="image/png" href="{{asset "images/favi.png"}}">
|
||||
<link rel="stylesheet" href="{{asset "css/screen.css"}}">
|
||||
{{ghost_head}}
|
||||
</head>
|
||||
<body class="{{body_class}}">
|
||||
<!-- Google Tag Manager (noscript) -->
|
||||
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-MM4FR8LJ"
|
||||
height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
|
||||
<!-- End Google Tag Manager (noscript) -->
|
||||
<a class="skip-link" href="#content">К содержанию</a>
|
||||
<header class="site-header">
|
||||
<div class="header-inner">
|
||||
<a class="brand" href="{{@site.url}}" aria-label="Forma N8, главная">
|
||||
<img src="{{asset "images/favi.png"}}" width="42" height="42" alt="">
|
||||
<span>FORMA<br>N8</span>
|
||||
</a>
|
||||
<nav class="main-nav" id="main-nav" aria-label="Основная навигация">
|
||||
<a href="{{@site.url}}">Лента</a>
|
||||
<div class="nav-dropdown">
|
||||
<button type="button" class="nav-dropdown-toggle" data-category-toggle aria-expanded="false">Категории <span class="nav-dropdown-icon" aria-hidden="true"></span></button>
|
||||
<div class="category-menu" data-category-menu>
|
||||
{{#get "tags" filter="visibility:public" limit="100" order="name asc"}}
|
||||
{{#foreach tags}}<a href="{{url}}">{{name}}</a>{{/foreach}}
|
||||
{{/get}}
|
||||
</div>
|
||||
</div>
|
||||
<a href="{{@site.url}}/about/">О проекте</a>
|
||||
</nav>
|
||||
<div class="header-actions">
|
||||
{{> "social-links" compact=true}}
|
||||
<button class="icon-button" type="button" data-ghost-search aria-label="Открыть поиск" title="Поиск">
|
||||
<svg viewBox="0 0 24 24" aria-hidden="true"><circle cx="11" cy="11" r="6.5"></circle><path d="m16 16 4 4"></path></svg>
|
||||
</button>
|
||||
<button class="icon-button menu-button" type="button" data-menu-toggle aria-expanded="false" aria-controls="main-nav" aria-label="Открыть меню" title="Меню">
|
||||
<svg viewBox="0 0 24 24" aria-hidden="true"><path d="M4 7h16M4 12h16M4 17h16"></path></svg>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
<main id="content">{{{body}}}</main>
|
||||
<footer class="site-footer">
|
||||
<div><a class="footer-brand" href="{{@site.url}}">FORMA N8</a><p>Новости снаряжения и экипировки.</p>{{> "social-links"}}</div>
|
||||
<nav aria-label="Служебная навигация"><a href="{{@site.url}}/rss/">RSS</a><a href="{{@site.url}}/sitemap.xml">Карта сайта</a></nav>
|
||||
</footer>
|
||||
<script src="{{asset "js/site.js"}}" defer></script>
|
||||
<script>
|
||||
(function(m,e,t,r,i,k,a){m[i]=m[i]||function(){(m[i].a=m[i].a||[]).push(arguments)};
|
||||
m[i].l=1*new Date();for(var j=0;j<document.scripts.length;j++){if(document.scripts[j].src===r){return;}}
|
||||
k=e.createElement(t),a=e.getElementsByTagName(t)[0],k.async=1,k.src=r,a.parentNode.insertBefore(k,a)})
|
||||
(window,document,"script","https://mc.yandex.ru/metrika/tag.js","ym");
|
||||
ym(110060335,"init",{clickmap:true,trackLinks:true,accurateTrackBounce:true,webvisor:true});
|
||||
</script>
|
||||
<noscript><div><img src="https://mc.yandex.ru/watch/110060335" style="position:absolute;left:-9999px" alt=""></div></noscript>
|
||||
{{ghost_foot}}
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,2 @@
|
||||
{{!< default}}
|
||||
<section class="feed-shell not-found"><p>{{statusCode}}</p><h1>{{message}}</h1><a href="{{@site.url}}">Вернуться в ленту</a></section>
|
||||
@@ -0,0 +1,38 @@
|
||||
{{!< default}}
|
||||
<section class="feed-shell">
|
||||
<header class="page-heading"><h1>Новости тактического снаряжения и экипировки</h1></header>
|
||||
{{#foreach posts}}
|
||||
{{#if @first}}
|
||||
{{#is "home"}}
|
||||
<article class="lead-story">
|
||||
<div class="lead-media feed-media" data-feed-media>
|
||||
<a class="feed-media-feature" href="{{url}}">{{#if feature_image}}<img src="{{img_url feature_image size="xl"}}" alt="{{#if feature_image_alt}}{{feature_image_alt}}{{else}}{{title}}{{/if}}">{{else}}<span class="media-placeholder"><img src="{{asset "images/favi.png"}}" alt=""></span>{{/if}}</a>
|
||||
<div class="feed-media-source" hidden>{{content}}</div>
|
||||
</div>
|
||||
<div class="lead-copy">
|
||||
<div class="article-meta">{{#if primary_tag}}<a class="category-link" href="{{primary_tag.url}}">{{primary_tag.name}}</a>{{/if}}<time datetime="{{date format="YYYY-MM-DD"}}">{{date format="DD.MM.YYYY"}}</time></div>
|
||||
<h2><a href="{{url}}">{{title}}</a></h2>
|
||||
<p>{{excerpt words="42"}}</p>
|
||||
<a class="read-link" href="{{url}}">Читать новость <span aria-hidden="true">→</span></a>
|
||||
<div class="preview-social">{{> "social-links" compact=true}}</div>
|
||||
</div>
|
||||
</article>
|
||||
{{else}}
|
||||
{{> "post-row"}}
|
||||
{{/is}}
|
||||
{{else}}
|
||||
{{> "post-row"}}
|
||||
{{/if}}
|
||||
{{else}}
|
||||
<div class="empty-state"><h2>Публикаций пока нет</h2><p>Лента скоро наполнится новостями.</p></div>
|
||||
{{/foreach}}
|
||||
{{> "pagination-ru"}}
|
||||
{{#is "home"}}
|
||||
<section class="home-seo" aria-labelledby="home-seo-title">
|
||||
<h2 id="home-seo-title">О новостной ленте Forma N8</h2>
|
||||
<p>Forma N8 — лента новостей о тактическом снаряжении, экипировке и EDC. Мы собираем новинки с десятков источников и публикуем в одном месте: плитоносцы и бронезащита, подсумки и разгрузочные системы, боевые брюки и рубашки, прицелы и оптика, тактические фонари, средства связи, медицина и ночные приборы.</p>
|
||||
<p>В ленте — новинки российских и зарубежных производителей: Ars Arma, Wartech, SSO, Stich Profi, Giena Tactical, 5.45 Design, Red Cliff, VOIN, Sturmer, БАРС, SRVV, ANA Tactical, Mordor Tac, Centurion Gear, ENOTACTICAL, Black Buckle, MILGEAR, SKIF Armor, Atlant Armour, БронеЁж, R5 GEAR, VINDEX, JAEGER Equipment, Margul Gear, Gordeev Tactical, IGLA TAC, STRIX TAC, Колчуга, APEX, Пересвет, Гарсинг и многих других. Отслеживаем оптику Диполь, ARTELV и ARKON, гарнитуры VULCAN, Fidem и ОКБ Октава, фонари Nicron, патчи BAD GRINGO и RedWar Patch, ножи DAGGERR, Morpheus Knives и N.C.Custom.</p>
|
||||
<p>Темы ленты охватывают весь комплект тактической экипировки: системы MOLLE и Molle-Minus, плиты стандартов Гранит, SAPI и ESAPI, ткани Cordura и NYCO, расцветки Multicam, Tiger Stripe и EMR. Новинки магазинов Академия Снаряжения, Semper Fi, ФИНИСТ и Империя ножей, обновления линеек, цены и отзывы пользователей — всё в одной новостной ленте о снаряжении.</p>
|
||||
</section>
|
||||
{{/is}}
|
||||
</section>
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"name": "forma-n8",
|
||||
"description": "Strict editorial theme for Forma N8",
|
||||
"version": "1.0.0",
|
||||
"author": {
|
||||
"name": "Forma N8",
|
||||
"email": "admin@f-n8.ru"
|
||||
},
|
||||
"keywords": ["ghost-theme", "forma-n8", "news"],
|
||||
"engines": {
|
||||
"ghost": ">=6.0.0"
|
||||
},
|
||||
"license": "UNLICENSED",
|
||||
"config": {
|
||||
"posts_per_page": 25,
|
||||
"card_assets": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
{{!< default}}
|
||||
{{#post}}<article class="article-page">{{#if @page.show_title_and_feature_image}}<header class="article-header"><h1>{{title}}</h1>{{#if custom_excerpt}}<p class="article-lead">{{custom_excerpt}}</p>{{/if}}</header>{{#if feature_image}}<figure class="article-media"><img src="{{img_url feature_image size="xl"}}" alt="{{title}}"></figure>{{/if}}{{/if}}<div class="article-layout"><div class="article-body gh-content">{{content}}<aside class="about-social"><span>Forma N8 в социальных сетях</span>{{> "social-links"}}</aside></div></div></article>{{/post}}
|
||||
@@ -0,0 +1,9 @@
|
||||
{{#if pagination.pages}}
|
||||
{{#match pagination.pages ">" 1}}
|
||||
<nav class="pagination" role="navigation" aria-label="Пагинация">
|
||||
{{#if pagination.prev}}<a class="newer-posts" href="{{page_url pagination.prev}}">← Новые публикации</a>{{/if}}
|
||||
<span class="page-number">Страница {{pagination.page}} из {{pagination.pages}}</span>
|
||||
{{#if pagination.next}}<a class="older-posts" href="{{page_url pagination.next}}">Старые публикации →</a>{{/if}}
|
||||
</nav>
|
||||
{{/match}}
|
||||
{{/if}}
|
||||
@@ -0,0 +1,12 @@
|
||||
<article class="feed-row">
|
||||
<div class="feed-row-media feed-media" data-feed-media>
|
||||
<a class="feed-media-feature" href="{{url}}">{{#if feature_image}}<img src="{{img_url feature_image size="m"}}" alt="{{#if feature_image_alt}}{{feature_image_alt}}{{else}}{{title}}{{/if}}" loading="lazy">{{else}}<span class="media-placeholder"><img src="{{asset "images/favi.png"}}" alt=""></span>{{/if}}</a>
|
||||
<div class="feed-media-source" hidden>{{content}}</div>
|
||||
</div>
|
||||
<div class="feed-row-copy">
|
||||
<div class="article-meta">{{#if primary_tag}}<a class="category-link" href="{{primary_tag.url}}">{{primary_tag.name}}</a>{{/if}}<time datetime="{{date format="YYYY-MM-DD"}}">{{date format="DD.MM.YYYY"}}</time></div>
|
||||
<h2><a href="{{url}}">{{title}}</a></h2>
|
||||
<p>{{excerpt words="28"}}</p>
|
||||
<div class="preview-social">{{> "social-links" compact=true}}</div>
|
||||
</div>
|
||||
</article>
|
||||
@@ -0,0 +1,10 @@
|
||||
<div class="social-links{{#if compact}} social-links--compact{{/if}}" aria-label="Forma N8 в социальных сетях">
|
||||
<a class="social-link social-link--vk" href="https://vk.com/forma_n8" target="_blank" rel="noopener noreferrer" aria-label="Forma N8 во ВКонтакте" title="ВКонтакте">
|
||||
<svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12.8 17.6C5.4 17.6 1.2 12.5 1 4h3.7c.1 6.2 2.9 8.8 5.1 9.3V4h3.5v5.3c2.2-.2 4.5-2.6 5.3-5.3H22c-.6 3.3-3.1 5.7-4.9 6.7 1.8.8 4.7 2.9 5.8 6.9h-3.8c-.9-2.6-3.2-4.6-5.8-4.9v4.9h-.5Z"/></svg>
|
||||
{{#unless compact}}<span>ВКонтакте</span>{{/unless}}
|
||||
</a>
|
||||
<a class="social-link social-link--tg" href="https://t.me/forma_n8" target="_blank" rel="noopener noreferrer" aria-label="Forma N8 в Telegram" title="Telegram">
|
||||
<svg viewBox="0 0 24 24" aria-hidden="true"><path d="M21.7 3.4 18.5 20.5c-.2 1.2-.9 1.5-1.9.9l-4.9-3.6-2.4 2.3c-.3.3-.5.5-1 .5l.3-5 9.1-8.2c.4-.4-.1-.6-.6-.2L5.9 14.3 1.1 12.8c-1-.3-1-1 .2-1.5L20 2.6c.9-.3 1.8.2 1.7.8Z"/></svg>
|
||||
{{#unless compact}}<span>Telegram</span>{{/unless}}
|
||||
</a>
|
||||
</div>
|
||||
@@ -0,0 +1,20 @@
|
||||
{{!< default}}
|
||||
{{#post}}
|
||||
<article class="article-page">
|
||||
<header class="article-header">
|
||||
<div class="article-breadcrumbs"><a href="{{@site.url}}">Лента</a>{{#if primary_tag}}<span>/</span><a href="{{primary_tag.url}}">{{primary_tag.name}}</a>{{/if}}</div>
|
||||
<h1>{{title}}</h1>
|
||||
{{#if custom_excerpt}}<p class="article-lead">{{custom_excerpt}}</p>{{/if}}
|
||||
<div class="article-byline"><time datetime="{{date format="YYYY-MM-DDTHH:mm:ssZ"}}">{{date format="DD.MM.YYYY · HH:mm"}} ЕКБ</time></div>
|
||||
</header>
|
||||
{{#if feature_image}}<figure class="article-media"><img src="{{img_url feature_image size="xl"}}" alt="{{#if feature_image_alt}}{{feature_image_alt}}{{else}}{{title}}{{/if}}"></figure>{{/if}}
|
||||
<div class="article-layout">
|
||||
<div class="article-body gh-content">{{content}}</div>
|
||||
<aside class="article-facts">
|
||||
{{#if primary_tag}}<div><span>Категория</span><a href="{{primary_tag.url}}">{{primary_tag.name}}</a></div>{{/if}}
|
||||
<div><span>Опубликовано</span><time datetime="{{date format="YYYY-MM-DD"}}">{{date format="DD.MM.YYYY"}}</time></div>
|
||||
<div><span>Наши каналы</span>{{> "social-links"}}</div>
|
||||
</aside>
|
||||
</div>
|
||||
</article>
|
||||
{{/post}}
|
||||
@@ -0,0 +1,6 @@
|
||||
{{!< default}}
|
||||
<section class="feed-shell">
|
||||
{{#tag}}<header class="page-heading taxonomy-heading"><p>Тема</p><h1>{{name}}</h1>{{#if description}}<p>{{description}}</p>{{/if}}</header>{{/tag}}
|
||||
<div class="feed-list">{{#foreach posts}}{{> "post-row"}}{{/foreach}}</div>
|
||||
{{> "pagination-ru"}}
|
||||
</section>
|
||||
@@ -0,0 +1,6 @@
|
||||
fastapi==0.115.12
|
||||
uvicorn[standard]==0.34.2
|
||||
asyncpg==0.30.0
|
||||
jinja2==3.1.6
|
||||
python-multipart==0.0.20
|
||||
pydantic-settings==2.9.1
|
||||
@@ -0,0 +1,28 @@
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import asyncpg
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
conn = await asyncpg.connect(os.environ["DATABASE_URL"])
|
||||
try:
|
||||
await conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW())"
|
||||
)
|
||||
for path in sorted((root / "db" / "migrations").glob("*.sql")):
|
||||
exists = await conn.fetchval("SELECT 1 FROM schema_migrations WHERE version=$1", path.name)
|
||||
if exists:
|
||||
continue
|
||||
async with conn.transaction():
|
||||
await conn.execute(path.read_text(encoding="utf-8"))
|
||||
await conn.execute("INSERT INTO schema_migrations(version) VALUES($1)", path.name)
|
||||
print(f"applied {path.name}")
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1 @@
|
||||
"""Forma N8 public website."""
|
||||
@@ -0,0 +1,448 @@
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from zoneinfo import ZoneInfo
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
from fastapi import Depends, FastAPI, File, Header, HTTPException, Query, Request, UploadFile
|
||||
from fastapi.responses import FileResponse, HTMLResponse, PlainTextResponse, Response
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.templating import Jinja2Templates
|
||||
|
||||
from .config import settings
|
||||
from .db import connect, disconnect, get_pool
|
||||
from .schemas import ArticleUpsert
|
||||
from .text import excerpt, paragraphs, slugify
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
STATIC_DIR = ROOT / "static"
|
||||
TEMPLATE_DIR = ROOT / "templates"
|
||||
ALLOWED_UPLOADS = {
|
||||
"image/jpeg": ".jpg",
|
||||
"image/png": ".png",
|
||||
"image/webp": ".webp",
|
||||
"image/gif": ".gif",
|
||||
"video/mp4": ".mp4",
|
||||
"video/webm": ".webm",
|
||||
}
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_: FastAPI):
|
||||
settings.upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
await connect()
|
||||
yield
|
||||
await disconnect()
|
||||
|
||||
|
||||
app = FastAPI(title=settings.site_name, docs_url=None, redoc_url=None, lifespan=lifespan)
|
||||
app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
|
||||
app.mount("/media", StaticFiles(directory=settings.upload_dir, check_dir=False), name="media")
|
||||
templates = Jinja2Templates(directory=TEMPLATE_DIR)
|
||||
templates.env.filters["paragraphs"] = paragraphs
|
||||
templates.env.filters["excerpt"] = excerpt
|
||||
LOCAL_TZ = ZoneInfo("Asia/Yekaterinburg")
|
||||
templates.env.filters["ekb_date"] = lambda value: value.astimezone(LOCAL_TZ).strftime("%d.%m.%Y") if value else ""
|
||||
templates.env.filters["ekb_time"] = lambda value: value.astimezone(LOCAL_TZ).strftime("%H:%M") if value else ""
|
||||
|
||||
|
||||
def site_url(path: str = "") -> str:
|
||||
return f"{settings.base_url}{path if path.startswith('/') else '/' + path if path else ''}"
|
||||
|
||||
|
||||
def article_path(article: dict) -> str:
|
||||
return f"/news/{article['slug']}"
|
||||
|
||||
|
||||
templates.env.globals.update(
|
||||
site_name=settings.site_name,
|
||||
site_tagline=settings.site_tagline,
|
||||
site_url=site_url,
|
||||
article_path=article_path,
|
||||
)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def security_headers(request: Request, call_next):
|
||||
response = await call_next(request)
|
||||
response.headers.setdefault("X-Content-Type-Options", "nosniff")
|
||||
response.headers.setdefault("X-Frame-Options", "DENY")
|
||||
response.headers.setdefault("Referrer-Policy", "strict-origin-when-cross-origin")
|
||||
response.headers.setdefault("Permissions-Policy", "camera=(), microphone=(), geolocation=()")
|
||||
return response
|
||||
|
||||
|
||||
def base_context(request: Request, **extra):
|
||||
return {
|
||||
"request": request,
|
||||
"current_path": request.url.path,
|
||||
"canonical_url": site_url(request.url.path),
|
||||
"meta_title": f"{settings.site_tagline} — {settings.site_name}",
|
||||
"meta_description": "Новости производителей тактического снаряжения, одежды, брони и аксессуаров.",
|
||||
"noindex": not settings.site_indexing_enabled or bool(request.query_params.get("q")),
|
||||
**extra,
|
||||
}
|
||||
|
||||
|
||||
ARTICLE_COLUMNS = """
|
||||
a.*,
|
||||
COALESCE(
|
||||
jsonb_agg(
|
||||
jsonb_build_object(
|
||||
'type', m.media_type,
|
||||
'url', m.url,
|
||||
'alt', m.alt_text,
|
||||
'width', m.width,
|
||||
'height', m.height
|
||||
) ORDER BY m.sort_order, m.id
|
||||
) FILTER (WHERE m.id IS NOT NULL),
|
||||
'[]'::jsonb
|
||||
) AS media
|
||||
"""
|
||||
|
||||
|
||||
def normalize_article(row) -> dict:
|
||||
article = dict(row)
|
||||
media = article.get("media") or []
|
||||
if isinstance(media, str):
|
||||
try:
|
||||
media = json.loads(media)
|
||||
except json.JSONDecodeError:
|
||||
media = []
|
||||
article["media"] = media if isinstance(media, list) else []
|
||||
return article
|
||||
|
||||
|
||||
async def fetch_feed(*, page: int, q: str = "", category: str = "", producer: str = "", page_size: int | None = None):
|
||||
pool = await get_pool()
|
||||
where = ["a.status='published'", "a.published_at <= NOW()"]
|
||||
args: list[object] = []
|
||||
if q:
|
||||
args.append(f"%{q}%")
|
||||
where.append(f"(a.title ILIKE ${len(args)} OR a.lead ILIKE ${len(args)} OR a.body ILIKE ${len(args)} OR a.producer_name ILIKE ${len(args)})")
|
||||
if category:
|
||||
args.append(category)
|
||||
where.append(f"a.category_tag=${len(args)}")
|
||||
if producer:
|
||||
args.append(producer)
|
||||
where.append(f"a.producer_tag=${len(args)}")
|
||||
where_sql = " AND ".join(where)
|
||||
total = int(await pool.fetchval(f"SELECT COUNT(*) FROM articles a WHERE {where_sql}", *args) or 0)
|
||||
limit = max(1, min(50, page_size or settings.page_size))
|
||||
offset = (page - 1) * limit
|
||||
rows = await pool.fetch(
|
||||
f"""
|
||||
SELECT {ARTICLE_COLUMNS}
|
||||
FROM articles a
|
||||
LEFT JOIN article_media m ON m.article_id=a.id
|
||||
WHERE {where_sql}
|
||||
GROUP BY a.id
|
||||
ORDER BY a.published_at DESC, a.id DESC
|
||||
LIMIT ${len(args) + 1} OFFSET ${len(args) + 2}
|
||||
""",
|
||||
*args,
|
||||
limit,
|
||||
offset,
|
||||
)
|
||||
return [normalize_article(row) for row in rows], total, limit
|
||||
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
async def feed(request: Request, page: int = Query(1, ge=1), q: str = Query("", max_length=100)):
|
||||
articles, total, limit = await fetch_feed(page=page, q=q.strip())
|
||||
pages = max(1, (total + limit - 1) // limit)
|
||||
canonical = site_url("/") if page == 1 else site_url(f"/?page={page}")
|
||||
return templates.TemplateResponse(
|
||||
"feed.html",
|
||||
base_context(
|
||||
request,
|
||||
articles=articles,
|
||||
page=page,
|
||||
pages=pages,
|
||||
total=total,
|
||||
q=q.strip(),
|
||||
canonical_url=canonical,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@app.get("/category/{tag}", response_class=HTMLResponse)
|
||||
async def category_feed(request: Request, tag: str, page: int = Query(1, ge=1)):
|
||||
pool = await get_pool()
|
||||
category_name = await pool.fetchval("SELECT name FROM categories WHERE tag=$1 AND is_active=TRUE", tag)
|
||||
if not category_name:
|
||||
raise HTTPException(status_code=404)
|
||||
articles, total, limit = await fetch_feed(page=page, category=tag)
|
||||
name = str(category_name)
|
||||
pages = max(1, (total + limit - 1) // limit)
|
||||
return templates.TemplateResponse(
|
||||
"taxonomy.html",
|
||||
base_context(
|
||||
request,
|
||||
articles=articles,
|
||||
page=page,
|
||||
pages=pages,
|
||||
heading=name,
|
||||
taxonomy_label="Категория",
|
||||
canonical_url=site_url(f"/category/{tag}" + (f"?page={page}" if page > 1 else "")),
|
||||
meta_title=f"{name}: новости — {settings.site_name}",
|
||||
meta_description=f"Последние новости в категории «{name}»: производители, новинки и обзоры снаряжения.",
|
||||
noindex=total == 0,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@app.get("/brand/{tag}", response_class=HTMLResponse)
|
||||
async def producer_feed(request: Request, tag: str, page: int = Query(1, ge=1)):
|
||||
articles, total, limit = await fetch_feed(page=page, producer=tag)
|
||||
if not articles and page == 1:
|
||||
raise HTTPException(status_code=404)
|
||||
name = articles[0]["producer_name"] if articles else tag
|
||||
pages = max(1, (total + limit - 1) // limit)
|
||||
return templates.TemplateResponse(
|
||||
"taxonomy.html",
|
||||
base_context(
|
||||
request,
|
||||
articles=articles,
|
||||
page=page,
|
||||
pages=pages,
|
||||
heading=name,
|
||||
taxonomy_label="Производитель",
|
||||
canonical_url=site_url(f"/brand/{tag}" + (f"?page={page}" if page > 1 else "")),
|
||||
meta_title=f"{name}: новости и новинки — {settings.site_name}",
|
||||
meta_description=f"Новости и новинки производителя {name}: снаряжение, одежда и экипировка.",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@app.get("/news/{slug}", response_class=HTMLResponse)
|
||||
async def article(request: Request, slug: str):
|
||||
pool = await get_pool()
|
||||
row = await pool.fetchrow(
|
||||
f"""
|
||||
SELECT {ARTICLE_COLUMNS}
|
||||
FROM articles a
|
||||
LEFT JOIN article_media m ON m.article_id=a.id
|
||||
WHERE a.slug=$1 AND a.status='published' AND a.published_at <= NOW()
|
||||
GROUP BY a.id
|
||||
""",
|
||||
slug,
|
||||
)
|
||||
if not row:
|
||||
raise HTTPException(status_code=404)
|
||||
item = normalize_article(row)
|
||||
related = await pool.fetch(
|
||||
f"""
|
||||
SELECT {ARTICLE_COLUMNS}
|
||||
FROM articles a
|
||||
LEFT JOIN article_media m ON m.article_id=a.id
|
||||
WHERE a.status='published' AND a.id<>$1
|
||||
AND (a.category_tag=$2 OR a.producer_tag=$3)
|
||||
GROUP BY a.id
|
||||
ORDER BY (a.producer_tag=$3) DESC, a.published_at DESC
|
||||
LIMIT 3
|
||||
""",
|
||||
item["id"],
|
||||
item["category_tag"],
|
||||
item["producer_tag"],
|
||||
)
|
||||
canonical = site_url(article_path(item))
|
||||
image = item["media"][0]["url"] if item["media"] else site_url("/static/favi.png")
|
||||
if image.startswith("/"):
|
||||
image = site_url(image)
|
||||
json_ld = {
|
||||
"@context": "https://schema.org",
|
||||
"@type": "NewsArticle",
|
||||
"headline": item["title"],
|
||||
"description": item["seo_description"] or item["lead"] or excerpt(item["body"]),
|
||||
"datePublished": item["published_at"].isoformat(),
|
||||
"dateModified": item["updated_at"].isoformat(),
|
||||
"mainEntityOfPage": canonical,
|
||||
"image": [image],
|
||||
"author": {"@type": "Organization", "name": settings.site_name},
|
||||
"publisher": {
|
||||
"@type": "Organization",
|
||||
"name": settings.site_name,
|
||||
"logo": {"@type": "ImageObject", "url": site_url("/static/favi.png")},
|
||||
},
|
||||
}
|
||||
return templates.TemplateResponse(
|
||||
"article.html",
|
||||
base_context(
|
||||
request,
|
||||
article=item,
|
||||
related=[normalize_article(x) for x in related],
|
||||
canonical_url=canonical,
|
||||
meta_title=item["seo_title"] or f"{item['title']} — {settings.site_name}",
|
||||
meta_description=item["seo_description"] or item["lead"] or excerpt(item["body"]),
|
||||
og_image=image,
|
||||
json_ld=json.dumps(json_ld, ensure_ascii=False),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def require_api_token(x_forma_token: str = Header(default="")) -> None:
|
||||
if not settings.publish_api_token or not hmac.compare_digest(x_forma_token, settings.publish_api_token):
|
||||
raise HTTPException(status_code=401, detail="invalid token")
|
||||
|
||||
|
||||
@app.post("/api/v1/media", dependencies=[Depends(require_api_token)])
|
||||
async def upload_media(file: UploadFile = File(...)):
|
||||
content_type = (file.content_type or "").split(";", 1)[0].lower()
|
||||
if content_type not in ALLOWED_UPLOADS:
|
||||
raise HTTPException(status_code=415, detail="unsupported media type")
|
||||
data = await file.read(50 * 1024 * 1024 + 1)
|
||||
if len(data) > 50 * 1024 * 1024:
|
||||
raise HTTPException(status_code=413, detail="file too large")
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
suffix = ALLOWED_UPLOADS[content_type]
|
||||
target = settings.upload_dir / digest[:2] / f"{digest}{suffix}"
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
if not target.exists():
|
||||
target.write_bytes(data)
|
||||
return {"url": f"/media/{digest[:2]}/{target.name}", "sha256": digest, "content_type": content_type}
|
||||
|
||||
|
||||
@app.post("/api/v1/articles", dependencies=[Depends(require_api_token)])
|
||||
async def upsert_article(payload: ArticleUpsert):
|
||||
pool = await get_pool()
|
||||
existing = await pool.fetchrow("SELECT id, slug, published_at FROM articles WHERE external_id=$1", payload.external_id)
|
||||
requested_slug = slugify(payload.slug or payload.title)
|
||||
stable_slug = existing["slug"] if existing else f"{requested_slug}-{slugify(payload.external_id)[-24:]}"
|
||||
published_at = payload.published_at or datetime.now(timezone.utc)
|
||||
category_tag = slugify(payload.category_tag)
|
||||
category_name = await pool.fetchval("SELECT name FROM categories WHERE tag=$1 AND is_active=TRUE", category_tag)
|
||||
category_name = str(category_name or payload.category_name).strip()
|
||||
async with pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
INSERT INTO articles(
|
||||
external_id, slug, title, lead, body, category_name, category_tag,
|
||||
producer_name, producer_tag, source_url, seo_title, seo_description,
|
||||
status, published_at
|
||||
)
|
||||
VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14)
|
||||
ON CONFLICT(external_id) DO UPDATE SET
|
||||
title=EXCLUDED.title,
|
||||
lead=EXCLUDED.lead,
|
||||
body=EXCLUDED.body,
|
||||
category_name=EXCLUDED.category_name,
|
||||
category_tag=EXCLUDED.category_tag,
|
||||
producer_name=EXCLUDED.producer_name,
|
||||
producer_tag=EXCLUDED.producer_tag,
|
||||
source_url=EXCLUDED.source_url,
|
||||
seo_title=EXCLUDED.seo_title,
|
||||
seo_description=EXCLUDED.seo_description,
|
||||
status=EXCLUDED.status,
|
||||
published_at=COALESCE(articles.published_at, EXCLUDED.published_at),
|
||||
updated_at=NOW()
|
||||
RETURNING id, slug, status, published_at, updated_at
|
||||
""",
|
||||
payload.external_id,
|
||||
stable_slug,
|
||||
payload.title.strip(),
|
||||
payload.lead.strip(),
|
||||
payload.body.strip(),
|
||||
category_name,
|
||||
category_tag,
|
||||
payload.producer_name.strip(),
|
||||
slugify(payload.producer_tag),
|
||||
str(payload.source_url) if payload.source_url else None,
|
||||
(payload.seo_title or payload.title).strip(),
|
||||
(payload.seo_description or payload.lead or excerpt(payload.body)).strip(),
|
||||
payload.status,
|
||||
published_at,
|
||||
)
|
||||
await conn.execute("DELETE FROM article_media WHERE article_id=$1", row["id"])
|
||||
for index, media in enumerate(payload.media):
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO article_media(article_id, media_type, url, alt_text, width, height, sort_order)
|
||||
VALUES($1,$2,$3,$4,$5,$6,$7)
|
||||
""",
|
||||
row["id"], media.type, media.url, media.alt or payload.title,
|
||||
media.width, media.height, media.sort_order or index,
|
||||
)
|
||||
return {
|
||||
"id": row["id"],
|
||||
"external_id": payload.external_id,
|
||||
"status": row["status"],
|
||||
"url": site_url(f"/news/{row['slug']}") if row["status"] == "published" else None,
|
||||
"published_at": row["published_at"],
|
||||
"updated_at": row["updated_at"],
|
||||
}
|
||||
|
||||
|
||||
@app.get("/rss.xml")
|
||||
async def rss():
|
||||
articles, _, _ = await fetch_feed(page=1, page_size=50)
|
||||
root = ET.Element("rss", {"version": "2.0", "xmlns:media": "http://search.yahoo.com/mrss/"})
|
||||
channel = ET.SubElement(root, "channel")
|
||||
ET.SubElement(channel, "title").text = f"{settings.site_name} — {settings.site_tagline}"
|
||||
ET.SubElement(channel, "link").text = settings.base_url
|
||||
ET.SubElement(channel, "description").text = "Новости производителей тактического снаряжения и экипировки."
|
||||
ET.SubElement(channel, "language").text = "ru"
|
||||
for item in articles[:50]:
|
||||
node = ET.SubElement(channel, "item")
|
||||
url = site_url(article_path(item))
|
||||
ET.SubElement(node, "title").text = item["title"]
|
||||
ET.SubElement(node, "link").text = url
|
||||
ET.SubElement(node, "guid", {"isPermaLink": "true"}).text = url
|
||||
ET.SubElement(node, "description").text = item["lead"] or excerpt(item["body"])
|
||||
ET.SubElement(node, "pubDate").text = item["published_at"].strftime("%a, %d %b %Y %H:%M:%S +0000")
|
||||
ET.SubElement(node, "category").text = item["category_name"]
|
||||
if item["media"]:
|
||||
ET.SubElement(node, "media:content", {"url": site_url(item["media"][0]["url"]) if item["media"][0]["url"].startswith("/") else item["media"][0]["url"], "medium": "image"})
|
||||
return Response(ET.tostring(root, encoding="utf-8", xml_declaration=True), media_type="application/rss+xml")
|
||||
|
||||
|
||||
@app.get("/sitemap.xml")
|
||||
async def sitemap():
|
||||
pool = await get_pool()
|
||||
rows = await pool.fetch("SELECT slug, updated_at FROM articles WHERE status='published' ORDER BY id")
|
||||
categories = await pool.fetch("SELECT category_tag AS tag, MAX(updated_at) AS updated_at FROM articles WHERE status='published' GROUP BY category_tag")
|
||||
producers = await pool.fetch("SELECT producer_tag AS tag, MAX(updated_at) AS updated_at FROM articles WHERE status='published' GROUP BY producer_tag")
|
||||
root = ET.Element("urlset", {"xmlns": "http://www.sitemaps.org/schemas/sitemap/0.9"})
|
||||
entries = [(site_url("/"), None)]
|
||||
entries += [(site_url(f"/category/{row['tag']}"), row["updated_at"]) for row in categories]
|
||||
entries += [(site_url(f"/brand/{row['tag']}"), row["updated_at"]) for row in producers]
|
||||
entries += [(site_url(f"/news/{row['slug']}"), row["updated_at"]) for row in rows]
|
||||
for loc, updated in entries:
|
||||
node = ET.SubElement(root, "url")
|
||||
ET.SubElement(node, "loc").text = loc
|
||||
if updated:
|
||||
ET.SubElement(node, "lastmod").text = updated.date().isoformat()
|
||||
return Response(ET.tostring(root, encoding="utf-8", xml_declaration=True), media_type="application/xml")
|
||||
|
||||
|
||||
@app.get("/robots.txt", response_class=PlainTextResponse)
|
||||
async def robots():
|
||||
if not settings.site_indexing_enabled:
|
||||
return "User-agent: *\nDisallow: /\n"
|
||||
return f"User-agent: *\nAllow: /\nDisallow: /api/\nSitemap: {site_url('/sitemap.xml')}\n"
|
||||
|
||||
|
||||
@app.get("/favicon.ico", include_in_schema=False)
|
||||
async def favicon():
|
||||
return FileResponse(STATIC_DIR / "favi.png", media_type="image/png")
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
async def health():
|
||||
pool = await get_pool()
|
||||
await pool.fetchval("SELECT 1")
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.exception_handler(404)
|
||||
async def not_found(request: Request, _):
|
||||
return templates.TemplateResponse(
|
||||
"404.html",
|
||||
base_context(request, meta_title=f"Страница не найдена — {settings.site_name}", noindex=True),
|
||||
status_code=404,
|
||||
)
|
||||
@@ -0,0 +1,23 @@
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
database_url: str = "postgresql://forma_n8:forma_n8@127.0.0.1:5432/forma_n8"
|
||||
site_base_url: str = "http://127.0.0.1:8090"
|
||||
site_name: str = "Forma N8"
|
||||
site_tagline: str = "Новости снаряжения"
|
||||
site_indexing_enabled: bool = False
|
||||
publish_api_token: str = ""
|
||||
upload_dir: Path = Path("/var/lib/forma-n8/uploads")
|
||||
page_size: int = 12
|
||||
|
||||
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
|
||||
|
||||
@property
|
||||
def base_url(self) -> str:
|
||||
return self.site_base_url.rstrip("/")
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -0,0 +1,24 @@
|
||||
import asyncpg
|
||||
|
||||
from .config import settings
|
||||
|
||||
|
||||
pool: asyncpg.Pool | None = None
|
||||
|
||||
|
||||
async def connect() -> asyncpg.Pool:
|
||||
global pool
|
||||
if pool is None:
|
||||
pool = await asyncpg.create_pool(settings.database_url, min_size=1, max_size=10, command_timeout=30)
|
||||
return pool
|
||||
|
||||
|
||||
async def disconnect() -> None:
|
||||
global pool
|
||||
if pool is not None:
|
||||
await pool.close()
|
||||
pool = None
|
||||
|
||||
|
||||
async def get_pool() -> asyncpg.Pool:
|
||||
return pool or await connect()
|
||||
@@ -0,0 +1,31 @@
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field, HttpUrl
|
||||
|
||||
|
||||
class MediaInput(BaseModel):
|
||||
type: Literal["image", "video"] = "image"
|
||||
url: str = Field(min_length=1, max_length=2000)
|
||||
alt: str = Field(default="", max_length=500)
|
||||
width: int | None = Field(default=None, ge=1)
|
||||
height: int | None = Field(default=None, ge=1)
|
||||
sort_order: int = Field(default=0, ge=0)
|
||||
|
||||
|
||||
class ArticleUpsert(BaseModel):
|
||||
external_id: str = Field(min_length=1, max_length=160)
|
||||
slug: str | None = Field(default=None, max_length=220)
|
||||
title: str = Field(min_length=5, max_length=300)
|
||||
lead: str = Field(default="", max_length=1000)
|
||||
body: str = Field(min_length=20, max_length=50000)
|
||||
category_name: str = Field(min_length=1, max_length=120)
|
||||
category_tag: str = Field(min_length=1, max_length=120)
|
||||
producer_name: str = Field(min_length=1, max_length=200)
|
||||
producer_tag: str = Field(min_length=1, max_length=160)
|
||||
source_url: HttpUrl | None = None
|
||||
seo_title: str | None = Field(default=None, max_length=300)
|
||||
seo_description: str | None = Field(default=None, max_length=500)
|
||||
status: Literal["draft", "published"] = "published"
|
||||
published_at: datetime | None = None
|
||||
media: list[MediaInput] = Field(default_factory=list, max_length=20)
|
||||
@@ -0,0 +1,37 @@
|
||||
import re
|
||||
from html import escape
|
||||
|
||||
from markupsafe import Markup
|
||||
|
||||
|
||||
TRANSLIT = str.maketrans(
|
||||
{
|
||||
"а": "a", "б": "b", "в": "v", "г": "g", "д": "d", "е": "e", "ё": "e",
|
||||
"ж": "zh", "з": "z", "и": "i", "й": "y", "к": "k", "л": "l", "м": "m",
|
||||
"н": "n", "о": "o", "п": "p", "р": "r", "с": "s", "т": "t", "у": "u",
|
||||
"ф": "f", "х": "h", "ц": "c", "ч": "ch", "ш": "sh", "щ": "sch",
|
||||
"ъ": "", "ы": "y", "ь": "", "э": "e", "ю": "yu", "я": "ya",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def slugify(value: str) -> str:
|
||||
value = value.strip().lower().translate(TRANSLIT)
|
||||
value = re.sub(r"[^a-z0-9]+", "-", value).strip("-")
|
||||
return value[:180] or "article"
|
||||
|
||||
|
||||
def paragraphs(value: str) -> Markup:
|
||||
blocks = []
|
||||
for part in re.split(r"\n\s*\n", value.strip()):
|
||||
lines = "<br>".join(escape(line.strip()) for line in part.splitlines() if line.strip())
|
||||
if lines:
|
||||
blocks.append(f"<p>{lines}</p>")
|
||||
return Markup("".join(blocks))
|
||||
|
||||
|
||||
def excerpt(value: str, limit: int = 180) -> str:
|
||||
compact = re.sub(r"\s+", " ", value or "").strip()
|
||||
if len(compact) <= limit:
|
||||
return compact
|
||||
return compact[: limit - 1].rsplit(" ", 1)[0] + "…"
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 46 KiB |
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,348 @@
|
||||
@font-face {
|
||||
font-family: "Roboto Condensed";
|
||||
src: url("/static/fonts/roboto-condensed-cyrillic-400.woff2") format("woff2");
|
||||
font-weight: 400;
|
||||
font-display: swap;
|
||||
}
|
||||
|
||||
@font-face {
|
||||
font-family: "Roboto Condensed";
|
||||
src: url("/static/fonts/roboto-condensed-cyrillic-700.woff2") format("woff2");
|
||||
font-weight: 700;
|
||||
font-display: swap;
|
||||
}
|
||||
|
||||
:root {
|
||||
color-scheme: dark;
|
||||
--bg: #090c0d;
|
||||
--surface: #101415;
|
||||
--surface-strong: #151a1b;
|
||||
--text: #eceee8;
|
||||
--muted: #9ba19b;
|
||||
--faint: #686f6b;
|
||||
--border: #29302e;
|
||||
--border-strong: #3a423f;
|
||||
--olive: #a5ad62;
|
||||
--red: #df4a40;
|
||||
--max: 1240px;
|
||||
--gutter: 24px;
|
||||
--font: "Roboto Condensed", "Arial Narrow", Arial, sans-serif;
|
||||
}
|
||||
|
||||
* { box-sizing: border-box; }
|
||||
|
||||
html { background: var(--bg); scroll-behavior: smooth; }
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
min-width: 320px;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
font-family: var(--font);
|
||||
font-size: 18px;
|
||||
line-height: 1.55;
|
||||
letter-spacing: 0;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
}
|
||||
|
||||
body::before {
|
||||
content: "";
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
pointer-events: none;
|
||||
opacity: .22;
|
||||
background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='120' height='120' viewBox='0 0 120 120'%3E%3Cfilter id='n'%3E%3CfeTurbulence type='fractalNoise' baseFrequency='.8' numOctaves='2' stitchTiles='stitch'/%3E%3C/filter%3E%3Crect width='100%25' height='100%25' filter='url(%23n)' opacity='.08'/%3E%3C/svg%3E");
|
||||
}
|
||||
|
||||
a { color: inherit; text-decoration: none; }
|
||||
a:hover { color: var(--olive); }
|
||||
img, video { display: block; max-width: 100%; }
|
||||
button, input { font: inherit; letter-spacing: 0; }
|
||||
|
||||
:focus-visible { outline: 2px solid var(--olive); outline-offset: 4px; }
|
||||
|
||||
.skip-link {
|
||||
position: fixed;
|
||||
z-index: 100;
|
||||
top: 10px;
|
||||
left: 10px;
|
||||
transform: translateY(-160%);
|
||||
padding: 10px 14px;
|
||||
background: var(--text);
|
||||
color: var(--bg);
|
||||
}
|
||||
.skip-link:focus { transform: none; }
|
||||
|
||||
.site-header {
|
||||
position: sticky;
|
||||
z-index: 20;
|
||||
top: 0;
|
||||
border-bottom: 1px solid var(--border);
|
||||
background: rgba(9, 12, 13, .96);
|
||||
backdrop-filter: blur(12px);
|
||||
}
|
||||
|
||||
.header-inner {
|
||||
width: min(var(--max), calc(100% - var(--gutter) * 2));
|
||||
min-height: 76px;
|
||||
margin: 0 auto;
|
||||
display: grid;
|
||||
grid-template-columns: 170px 1fr auto;
|
||||
align-items: center;
|
||||
gap: 32px;
|
||||
}
|
||||
|
||||
.brand {
|
||||
display: inline-flex;
|
||||
width: fit-content;
|
||||
align-items: center;
|
||||
gap: 11px;
|
||||
font-size: 15px;
|
||||
font-weight: 700;
|
||||
line-height: .95;
|
||||
}
|
||||
.brand img { width: 42px; height: 42px; border: 1px solid var(--border-strong); }
|
||||
|
||||
.main-nav { display: flex; justify-content: center; align-self: stretch; gap: 38px; }
|
||||
.main-nav a {
|
||||
position: relative;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
color: var(--muted);
|
||||
font-size: 16px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.main-nav a:hover, .main-nav a[aria-current="page"] { color: var(--text); }
|
||||
.main-nav a[aria-current="page"]::after {
|
||||
content: "";
|
||||
position: absolute;
|
||||
right: 0;
|
||||
bottom: -1px;
|
||||
left: 0;
|
||||
height: 3px;
|
||||
background: var(--olive);
|
||||
}
|
||||
|
||||
.header-actions { display: flex; gap: 8px; }
|
||||
.icon-button {
|
||||
display: grid;
|
||||
width: 40px;
|
||||
height: 40px;
|
||||
padding: 0;
|
||||
place-items: center;
|
||||
border: 0;
|
||||
background: transparent;
|
||||
color: var(--text);
|
||||
cursor: pointer;
|
||||
}
|
||||
.icon-button:hover { background: var(--surface-strong); color: var(--olive); }
|
||||
.icon-button svg { width: 23px; height: 23px; fill: none; stroke: currentColor; stroke-width: 1.8; stroke-linecap: square; }
|
||||
.menu-button { display: none; }
|
||||
|
||||
.search-form {
|
||||
width: min(760px, calc(100% - var(--gutter) * 2));
|
||||
margin: 0 auto;
|
||||
padding: 16px 0 20px;
|
||||
}
|
||||
.search-form[hidden] { display: none; }
|
||||
.search-form label { display: block; margin-bottom: 8px; color: var(--muted); font-size: 14px; }
|
||||
.search-form > div { display: grid; grid-template-columns: 1fr auto; }
|
||||
.search-form input {
|
||||
min-width: 0;
|
||||
height: 48px;
|
||||
padding: 0 16px;
|
||||
border: 1px solid var(--border-strong);
|
||||
border-right: 0;
|
||||
border-radius: 0;
|
||||
background: var(--surface);
|
||||
color: var(--text);
|
||||
font-size: 17px;
|
||||
}
|
||||
.search-form button, .read-link, .empty-state a {
|
||||
border: 1px solid var(--olive);
|
||||
border-radius: 0;
|
||||
background: transparent;
|
||||
color: var(--text);
|
||||
cursor: pointer;
|
||||
}
|
||||
.search-form button { padding: 0 24px; }
|
||||
.search-form button:hover, .read-link:hover, .empty-state a:hover { background: var(--olive); color: var(--bg); }
|
||||
|
||||
.feed-shell, .article-page, .related-section, .site-footer {
|
||||
position: relative;
|
||||
width: min(var(--max), calc(100% - var(--gutter) * 2));
|
||||
margin: 0 auto;
|
||||
}
|
||||
|
||||
.page-heading { padding: 56px 0 46px; }
|
||||
.page-heading h1 {
|
||||
max-width: 900px;
|
||||
margin: 0;
|
||||
font-size: 68px;
|
||||
font-weight: 700;
|
||||
line-height: .98;
|
||||
}
|
||||
.page-heading > p { margin: 20px 0 0; color: var(--muted); }
|
||||
|
||||
.lead-story {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1.25fr) minmax(340px, .95fr);
|
||||
min-height: 480px;
|
||||
border-bottom: 1px solid var(--border-strong);
|
||||
}
|
||||
.lead-media { min-height: 480px; overflow: hidden; background: var(--surface); }
|
||||
.lead-media img, .lead-media video { width: 100%; height: 100%; object-fit: cover; transition: transform .45s ease; }
|
||||
.lead-media:hover img { transform: scale(1.015); }
|
||||
.lead-copy { display: flex; flex-direction: column; justify-content: center; padding: 34px 0 36px 44px; }
|
||||
.lead-copy h2 { margin: 24px 0 22px; font-size: 40px; line-height: 1.14; }
|
||||
.lead-copy h2 a:hover { color: var(--text); text-decoration: underline; text-decoration-color: var(--olive); text-underline-offset: 7px; }
|
||||
.lead-copy > p { max-width: 580px; margin: 0; color: var(--muted); font-size: 20px; line-height: 1.5; }
|
||||
.read-link { width: fit-content; margin-top: 30px; padding: 10px 15px; font-size: 15px; }
|
||||
.read-link span { margin-left: 8px; }
|
||||
|
||||
.article-meta { display: flex; align-items: center; flex-wrap: wrap; gap: 0; color: var(--muted); font-size: 14px; }
|
||||
.article-meta > * { display: inline-flex; align-items: center; }
|
||||
.article-meta > * + *::before { content: ""; width: 1px; height: 18px; margin: 0 16px; background: var(--border-strong); }
|
||||
.article-meta .category-link { color: var(--olive); }
|
||||
.article-meta .producer-link { color: var(--red); }
|
||||
|
||||
.feed-list { width: 100%; }
|
||||
.feed-list-top { border-top: 1px solid var(--border-strong); }
|
||||
.feed-row {
|
||||
display: grid;
|
||||
grid-template-columns: 350px minmax(0, 1fr);
|
||||
gap: 34px;
|
||||
padding: 22px 0;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
.feed-row-media { aspect-ratio: 16 / 9; overflow: hidden; background: var(--surface); }
|
||||
.feed-row-media img, .feed-row-media video { width: 100%; height: 100%; object-fit: cover; transition: transform .35s ease; }
|
||||
.feed-row-media:hover img { transform: scale(1.02); }
|
||||
.feed-row-copy { align-self: center; min-width: 0; }
|
||||
.feed-row h2 { margin: 16px 0 8px; font-size: 25px; line-height: 1.2; }
|
||||
.feed-row h2 a:hover { color: var(--text); text-decoration: underline; text-decoration-color: var(--olive); text-underline-offset: 5px; }
|
||||
.feed-row p { max-width: 760px; margin: 0; color: var(--muted); font-size: 17px; line-height: 1.5; }
|
||||
.media-placeholder { display: grid; width: 100%; height: 100%; place-items: center; background: var(--surface); }
|
||||
.media-placeholder img { width: 76px; height: 76px; opacity: .28; object-fit: contain; }
|
||||
|
||||
.pagination { display: grid; grid-template-columns: 1fr auto 1fr; align-items: center; min-height: 90px; font-size: 15px; }
|
||||
.pagination > :last-child { justify-self: end; }
|
||||
.pagination span { color: var(--faint); }
|
||||
|
||||
.taxonomy-heading { padding-bottom: 34px; }
|
||||
.taxonomy-heading > p { margin: 0 0 10px; color: var(--olive); font-size: 15px; text-transform: uppercase; }
|
||||
.taxonomy-heading h1 { font-size: 56px; }
|
||||
|
||||
.article-page { padding-top: 56px; }
|
||||
.article-header { max-width: 960px; margin: 0 auto 42px; }
|
||||
.article-breadcrumbs { display: flex; gap: 12px; margin-bottom: 28px; color: var(--muted); font-size: 14px; }
|
||||
.article-breadcrumbs span { color: var(--faint); }
|
||||
.article-header h1 { margin: 0; font-size: 58px; line-height: 1.06; }
|
||||
.article-lead { max-width: 850px; margin: 26px 0 0; color: var(--muted); font-size: 23px; line-height: 1.45; }
|
||||
.article-byline { display: flex; gap: 24px; margin-top: 28px; color: var(--muted); font-size: 14px; }
|
||||
.article-byline a { color: var(--red); }
|
||||
.article-media { display: grid; gap: 12px; margin-bottom: 46px; }
|
||||
.article-media figure { margin: 0; background: var(--surface); }
|
||||
.article-media img, .article-media video { width: 100%; max-height: 760px; object-fit: contain; }
|
||||
.article-layout { display: grid; grid-template-columns: minmax(0, 760px) 260px; justify-content: center; gap: 78px; }
|
||||
.article-body { font-size: 21px; line-height: 1.7; }
|
||||
.article-body p { margin: 0 0 26px; }
|
||||
.article-facts { align-self: start; border-top: 2px solid var(--olive); }
|
||||
.article-facts > div { padding: 18px 0; border-bottom: 1px solid var(--border); }
|
||||
.article-facts span { display: block; margin-bottom: 4px; color: var(--faint); font-size: 13px; text-transform: uppercase; }
|
||||
.article-facts a { font-size: 17px; }
|
||||
.article-facts .source-link { display: block; margin-top: 22px; color: var(--muted); font-size: 14px; }
|
||||
.related-section { margin-top: 90px; border-top: 1px solid var(--border-strong); }
|
||||
.related-section > h2 { margin: 0; padding: 34px 0 18px; font-size: 32px; }
|
||||
|
||||
.empty-state, .not-found { min-height: 56vh; display: flex; flex-direction: column; justify-content: center; align-items: flex-start; }
|
||||
.empty-state h2, .not-found h1 { margin: 0; font-size: 42px; }
|
||||
.empty-state p { color: var(--muted); }
|
||||
.empty-state a { margin-top: 16px; padding: 10px 15px; }
|
||||
.not-found > p { margin: 0; color: var(--red); font-size: 18px; }
|
||||
.not-found a { margin-top: 24px; color: var(--olive); }
|
||||
|
||||
.site-footer {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: flex-end;
|
||||
margin-top: 84px;
|
||||
padding: 34px 0 46px;
|
||||
border-top: 1px solid var(--border-strong);
|
||||
color: var(--muted);
|
||||
font-size: 14px;
|
||||
}
|
||||
.footer-brand { color: var(--text); font-size: 20px; font-weight: 700; }
|
||||
.site-footer p { margin: 7px 0 0; }
|
||||
.site-footer nav { display: flex; gap: 24px; }
|
||||
|
||||
@media (max-width: 980px) {
|
||||
.header-inner { grid-template-columns: 150px 1fr auto; gap: 16px; }
|
||||
.main-nav { gap: 22px; }
|
||||
.main-nav a { font-size: 14px; }
|
||||
.page-heading h1 { font-size: 56px; }
|
||||
.lead-story { grid-template-columns: 1.1fr .9fr; min-height: 420px; }
|
||||
.lead-media { min-height: 420px; }
|
||||
.lead-copy { padding-left: 30px; }
|
||||
.lead-copy h2 { font-size: 33px; }
|
||||
.feed-row { grid-template-columns: 290px minmax(0, 1fr); gap: 26px; }
|
||||
.article-layout { grid-template-columns: minmax(0, 680px) 220px; gap: 44px; }
|
||||
}
|
||||
|
||||
@media (max-width: 760px) {
|
||||
:root { --gutter: 16px; }
|
||||
body { font-size: 17px; }
|
||||
.header-inner { min-height: 64px; grid-template-columns: 1fr auto; }
|
||||
.brand img { width: 38px; height: 38px; }
|
||||
.main-nav {
|
||||
position: absolute;
|
||||
top: 64px;
|
||||
right: 0;
|
||||
left: 0;
|
||||
display: none;
|
||||
padding: 12px var(--gutter) 18px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
background: var(--bg);
|
||||
}
|
||||
.main-nav.is-open { display: grid; }
|
||||
.main-nav a { min-height: 44px; border-bottom: 1px solid var(--border); font-size: 16px; }
|
||||
.main-nav a[aria-current="page"]::after { display: none; }
|
||||
.menu-button { display: grid; }
|
||||
.page-heading { padding: 38px 0 30px; }
|
||||
.page-heading h1, .taxonomy-heading h1 { font-size: 44px; line-height: 1; }
|
||||
.lead-story { display: block; min-height: 0; }
|
||||
.lead-media { min-height: 0; aspect-ratio: 4 / 3; }
|
||||
.lead-copy { padding: 25px 0 34px; }
|
||||
.lead-copy h2 { margin: 20px 0 15px; font-size: 32px; }
|
||||
.lead-copy > p { font-size: 18px; }
|
||||
.feed-row { grid-template-columns: 132px minmax(0, 1fr); gap: 16px; padding: 18px 0; }
|
||||
.feed-row-media { aspect-ratio: 4 / 3; }
|
||||
.feed-row h2 { margin: 10px 0 0; font-size: 20px; }
|
||||
.feed-row p { display: none; }
|
||||
.article-meta { font-size: 12px; }
|
||||
.article-meta time { display: none; }
|
||||
.article-meta > * + *::before { height: 14px; margin: 0 10px; }
|
||||
.article-meta .category-link::before { display: none; }
|
||||
.article-header { margin-bottom: 30px; }
|
||||
.article-header h1 { font-size: 42px; }
|
||||
.article-lead { font-size: 20px; }
|
||||
.article-layout { display: block; }
|
||||
.article-body { font-size: 19px; }
|
||||
.article-facts { margin-top: 42px; }
|
||||
.related-section { margin-top: 64px; }
|
||||
}
|
||||
|
||||
@media (max-width: 460px) {
|
||||
.feed-row { grid-template-columns: 112px minmax(0, 1fr); gap: 13px; }
|
||||
.feed-row h2 { font-size: 18px; line-height: 1.18; }
|
||||
.article-meta .producer-link::before { display: none; }
|
||||
.article-meta .category-link { display: none; }
|
||||
.article-header h1 { font-size: 36px; }
|
||||
.article-byline { display: grid; gap: 7px; }
|
||||
.site-footer { display: grid; gap: 24px; }
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
html { scroll-behavior: auto; }
|
||||
*, *::before, *::after { transition-duration: .01ms !important; }
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
const menuButton = document.querySelector('[data-menu-toggle]');
|
||||
const nav = document.querySelector('#main-nav');
|
||||
const searchButton = document.querySelector('[data-search-toggle]');
|
||||
const searchForm = document.querySelector('[data-search-form]');
|
||||
|
||||
menuButton?.addEventListener('click', () => {
|
||||
const open = menuButton.getAttribute('aria-expanded') === 'true';
|
||||
menuButton.setAttribute('aria-expanded', String(!open));
|
||||
nav?.classList.toggle('is-open', !open);
|
||||
});
|
||||
|
||||
searchButton?.addEventListener('click', () => {
|
||||
const opening = searchForm?.hasAttribute('hidden');
|
||||
searchForm?.toggleAttribute('hidden', !opening);
|
||||
if (opening) searchForm?.querySelector('input')?.focus();
|
||||
});
|
||||
@@ -0,0 +1,4 @@
|
||||
{% extends "base.html" %}
|
||||
{% block content %}
|
||||
<section class="not-found"><p>404</p><h1>Страница не найдена</h1><a href="/">Вернуться в ленту</a></section>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,29 @@
|
||||
{% macro media(article, eager=false) %}
|
||||
{% set item = article.media[0] if article.media else none %}
|
||||
{% if item and item.type == 'image' %}
|
||||
<img src="{{ item.url }}" alt="{{ item.alt or article.title }}" {% if item.width %}width="{{ item.width }}"{% endif %} {% if item.height %}height="{{ item.height }}"{% endif %} loading="{{ 'eager' if eager else 'lazy' }}" decoding="async">
|
||||
{% elif item and item.type == 'video' %}
|
||||
<video src="{{ item.url }}" muted playsinline preload="metadata" aria-label="{{ item.alt or article.title }}"></video>
|
||||
{% else %}
|
||||
<span class="media-placeholder"><img src="/static/favi.png" alt="" width="96" height="96"></span>
|
||||
{% endif %}
|
||||
{% endmacro %}
|
||||
|
||||
{% macro meta(article) %}
|
||||
<div class="article-meta">
|
||||
<time datetime="{{ article.published_at.isoformat() }}">{{ article.published_at|ekb_date }} · {{ article.published_at|ekb_time }}</time>
|
||||
<a class="category-link" href="/category/{{ article.category_tag }}">{{ article.category_name }}</a>
|
||||
<a class="producer-link" href="/brand/{{ article.producer_tag }}">{{ article.producer_name }}</a>
|
||||
</div>
|
||||
{% endmacro %}
|
||||
|
||||
{% macro row(article) %}
|
||||
<article class="feed-row">
|
||||
<a class="feed-row-media" href="{{ article_path(article) }}" tabindex="-1">{{ media(article) }}</a>
|
||||
<div class="feed-row-copy">
|
||||
{{ meta(article) }}
|
||||
<h2><a href="{{ article_path(article) }}">{{ article.title }}</a></h2>
|
||||
<p>{{ article.lead or article.body|excerpt(210) }}</p>
|
||||
</div>
|
||||
</article>
|
||||
{% endmacro %}
|
||||
@@ -0,0 +1,7 @@
|
||||
{% if pages > 1 %}
|
||||
<nav class="pagination" aria-label="Страницы ленты">
|
||||
{% if page > 1 %}<a rel="prev" href="?page={{ page - 1 }}{% if q %}&q={{ q|urlencode }}{% endif %}">← Новее</a>{% else %}<span></span>{% endif %}
|
||||
<span>{{ page }} / {{ pages }}</span>
|
||||
{% if page < pages %}<a rel="next" href="?page={{ page + 1 }}{% if q %}&q={{ q|urlencode }}{% endif %}">Старее →</a>{% endif %}
|
||||
</nav>
|
||||
{% endif %}
|
||||
@@ -0,0 +1,38 @@
|
||||
{% extends "base.html" %}
|
||||
{% from "_article_rows.html" import media, row %}
|
||||
{% block og_type %}article{% endblock %}
|
||||
{% block content %}
|
||||
<article class="article-page">
|
||||
<header class="article-header">
|
||||
<div class="article-breadcrumbs"><a href="/">Лента</a><span>/</span><a href="/category/{{ article.category_tag }}">{{ article.category_name }}</a></div>
|
||||
<h1>{{ article.title }}</h1>
|
||||
{% if article.lead %}<p class="article-lead">{{ article.lead }}</p>{% endif %}
|
||||
<div class="article-byline">
|
||||
<time datetime="{{ article.published_at.isoformat() }}">{{ article.published_at|ekb_date }} · {{ article.published_at|ekb_time }} ЕКБ</time>
|
||||
<a href="/brand/{{ article.producer_tag }}">{{ article.producer_name }}</a>
|
||||
</div>
|
||||
</header>
|
||||
{% if article.media %}
|
||||
<div class="article-media">
|
||||
{% for item in article.media %}
|
||||
{% if item.type == 'image' %}<figure><img src="{{ item.url }}" alt="{{ item.alt or article.title }}" loading="{{ 'eager' if loop.first else 'lazy' }}" decoding="async"></figure>{% endif %}
|
||||
{% if item.type == 'video' %}<figure><video src="{{ item.url }}" controls playsinline preload="metadata"></video></figure>{% endif %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
<div class="article-layout">
|
||||
<div class="article-body">{{ article.body|paragraphs }}</div>
|
||||
<aside class="article-facts">
|
||||
<div><span>Категория</span><a href="/category/{{ article.category_tag }}">{{ article.category_name }}</a></div>
|
||||
<div><span>Производитель</span><a href="/brand/{{ article.producer_tag }}">{{ article.producer_name }}</a></div>
|
||||
{% if article.source_url %}<a class="source-link" href="{{ article.source_url }}" rel="nofollow noopener" target="_blank">Первоисточник ↗</a>{% endif %}
|
||||
</aside>
|
||||
</div>
|
||||
</article>
|
||||
{% if related %}
|
||||
<section class="related-section">
|
||||
<h2>По теме</h2>
|
||||
<div class="feed-list">{% for item in related %}{{ row(item) }}{% endfor %}</div>
|
||||
</section>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,67 @@
|
||||
<!doctype html>
|
||||
<html lang="ru">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>{{ meta_title }}</title>
|
||||
<meta name="description" content="{{ meta_description }}">
|
||||
{% if noindex %}<meta name="robots" content="noindex,follow">{% else %}<meta name="robots" content="index,follow,max-image-preview:large">{% endif %}
|
||||
<link rel="canonical" href="{{ canonical_url }}">
|
||||
<link rel="alternate" type="application/rss+xml" title="Forma N8 RSS" href="{{ site_url('/rss.xml') }}">
|
||||
<link rel="icon" type="image/png" href="/static/favi.png">
|
||||
<meta property="og:site_name" content="Forma N8">
|
||||
<meta property="og:locale" content="ru_RU">
|
||||
<meta property="og:type" content="{% block og_type %}website{% endblock %}">
|
||||
<meta property="og:title" content="{{ meta_title }}">
|
||||
<meta property="og:description" content="{{ meta_description }}">
|
||||
<meta property="og:url" content="{{ canonical_url }}">
|
||||
{% if og_image %}<meta property="og:image" content="{{ og_image }}">{% endif %}
|
||||
<link rel="stylesheet" href="/static/site.css?v=1">
|
||||
{% if json_ld %}<script type="application/ld+json">{{ json_ld|safe }}</script>{% endif %}
|
||||
</head>
|
||||
<body>
|
||||
<a class="skip-link" href="#content">К содержанию</a>
|
||||
<header class="site-header">
|
||||
<div class="header-inner">
|
||||
<a class="brand" href="/" aria-label="Forma N8, главная">
|
||||
<img src="/static/favi.png" width="44" height="44" alt="">
|
||||
<span>FORMA<br>N8</span>
|
||||
</a>
|
||||
<nav class="main-nav" id="main-nav" aria-label="Основная навигация">
|
||||
<a href="/" {% if current_path == '/' %}aria-current="page"{% endif %}>Лента</a>
|
||||
<a href="/category/snaryazhenie">Снаряжение</a>
|
||||
<a href="/category/odezhda">Одежда</a>
|
||||
<a href="/category/bronya">Броня</a>
|
||||
<a href="/category/aksessuary">Аксессуары</a>
|
||||
</nav>
|
||||
<div class="header-actions">
|
||||
<button class="icon-button" type="button" data-search-toggle aria-label="Открыть поиск" title="Поиск">
|
||||
<svg viewBox="0 0 24 24" aria-hidden="true"><circle cx="11" cy="11" r="6.5"></circle><path d="m16 16 4 4"></path></svg>
|
||||
</button>
|
||||
<button class="icon-button menu-button" type="button" data-menu-toggle aria-expanded="false" aria-controls="main-nav" aria-label="Открыть меню" title="Меню">
|
||||
<svg viewBox="0 0 24 24" aria-hidden="true"><path d="M4 7h16M4 12h16M4 17h16"></path></svg>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<form class="search-form" action="/" method="get" data-search-form {% if not q %}hidden{% endif %}>
|
||||
<label for="site-search">Поиск по новостям</label>
|
||||
<div>
|
||||
<input id="site-search" name="q" value="{{ q or '' }}" type="search" maxlength="100" placeholder="Название, производитель, снаряжение">
|
||||
<button type="submit">Найти</button>
|
||||
</div>
|
||||
</form>
|
||||
</header>
|
||||
<main id="content">{% block content %}{% endblock %}</main>
|
||||
<footer class="site-footer">
|
||||
<div>
|
||||
<a class="footer-brand" href="/">FORMA N8</a>
|
||||
<p>Новости снаряжения и экипировки.</p>
|
||||
</div>
|
||||
<nav aria-label="Служебная навигация">
|
||||
<a href="/rss.xml">RSS</a>
|
||||
<a href="/sitemap.xml">Sitemap</a>
|
||||
</nav>
|
||||
</footer>
|
||||
<script src="/static/site.js?v=1" defer></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,34 @@
|
||||
{% extends "base.html" %}
|
||||
{% from "_article_rows.html" import media, meta, row %}
|
||||
{% block content %}
|
||||
<section class="feed-shell">
|
||||
<header class="page-heading">
|
||||
<h1>Новости снаряжения</h1>
|
||||
{% if q %}<p>Результаты поиска: «{{ q }}»</p>{% endif %}
|
||||
</header>
|
||||
{% if articles %}
|
||||
{% if page == 1 and not q %}
|
||||
{% set lead = articles[0] %}
|
||||
<article class="lead-story">
|
||||
<a class="lead-media" href="{{ article_path(lead) }}">{{ media(lead, true) }}</a>
|
||||
<div class="lead-copy">
|
||||
{{ meta(lead) }}
|
||||
<h2><a href="{{ article_path(lead) }}">{{ lead.title }}</a></h2>
|
||||
<p>{{ lead.lead or lead.body|excerpt(340) }}</p>
|
||||
<a class="read-link" href="{{ article_path(lead) }}">Читать новость <span aria-hidden="true">→</span></a>
|
||||
</div>
|
||||
</article>
|
||||
<div class="feed-list">
|
||||
{% for article in articles[1:] %}{{ row(article) }}{% endfor %}
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="feed-list feed-list-top">
|
||||
{% for article in articles %}{{ row(article) }}{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
{% include "_pagination.html" %}
|
||||
{% else %}
|
||||
<div class="empty-state"><h2>Ничего не найдено</h2><p>Попробуйте изменить запрос или вернуться к ленте.</p><a href="/">Вернуться в ленту</a></div>
|
||||
{% endif %}
|
||||
</section>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,18 @@
|
||||
{% extends "base.html" %}
|
||||
{% from "_article_rows.html" import row %}
|
||||
{% block content %}
|
||||
<section class="feed-shell taxonomy-shell">
|
||||
<header class="page-heading taxonomy-heading">
|
||||
<p>{{ taxonomy_label }}</p>
|
||||
<h1>{{ heading }}</h1>
|
||||
</header>
|
||||
{% if articles %}
|
||||
<div class="feed-list feed-list-top">
|
||||
{% for article in articles %}{{ row(article) }}{% endfor %}
|
||||
</div>
|
||||
{% include "_pagination.html" %}
|
||||
{% else %}
|
||||
<div class="empty-state"><h2>Публикаций пока нет</h2><p>Новые материалы появятся здесь после публикации.</p><a href="/">Вернуться в ленту</a></div>
|
||||
{% endif %}
|
||||
</section>
|
||||
{% endblock %}
|
||||
@@ -6,6 +6,7 @@ jinja2==3.1.6
|
||||
loguru==0.7.3
|
||||
litellm==1.76.0
|
||||
pydantic-settings==2.10.1
|
||||
Pillow==11.3.0
|
||||
python-multipart==0.0.20
|
||||
uvicorn[standard]==0.35.0
|
||||
yt-dlp==2026.6.9
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from aiogram.types import FSInputFile
|
||||
|
||||
from vk_parser_app.constants import MEDIA_STATUS_FAILED, MEDIA_STATUS_LINK_ONLY
|
||||
from vk_parser_app.db import get_pool
|
||||
from vk_parser_app.workers.vk_storage_uploader import TMP_DIR, TMP_PREFIX, TelegramStorageUploader
|
||||
|
||||
|
||||
def jlog(event: str, **payload: Any) -> None:
|
||||
print(json.dumps({"event": event, **payload}, ensure_ascii=False, default=str), flush=True)
|
||||
|
||||
|
||||
async def load_pending_videos(worker: TelegramStorageUploader, limit: int) -> list[dict[str, Any]]:
|
||||
rows = await worker.pool.fetch(
|
||||
"""
|
||||
SELECT m.*, rp.original_url AS post_url
|
||||
FROM raw_post_media m
|
||||
JOIN raw_posts rp ON rp.id=m.raw_post_id
|
||||
WHERE m.media_type='video'
|
||||
AND m.status='pending'
|
||||
AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = ''
|
||||
AND COALESCE(m.error, '') ILIKE '%timeout%'
|
||||
AND COALESCE(m.duration_sec, 0) <= $1
|
||||
ORDER BY COALESCE(rp.publication_status, 'pending')='published', m.id DESC
|
||||
LIMIT $2
|
||||
""",
|
||||
worker.video_max_duration_sec,
|
||||
limit,
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
async def send_video_to_storage(worker: TelegramStorageUploader, media: dict[str, Any], path: str) -> None:
|
||||
caption = (
|
||||
f"Backfill video media #{int(media['id'])} for raw_post #{int(media['raw_post_id'])}\n"
|
||||
f'<a href="{html.escape(str(media.get("post_url") or ""), quote=True)}">source post</a>'
|
||||
)
|
||||
message = await worker.tg_retry(
|
||||
lambda: worker.bot.send_video(
|
||||
video=FSInputFile(path, filename=f"video_{int(media['id'])}.mp4"),
|
||||
caption=caption[: worker.caption_limit],
|
||||
parse_mode="HTML",
|
||||
**worker.chat_kwargs(),
|
||||
)
|
||||
)
|
||||
if not message.video:
|
||||
raise RuntimeError("Telegram accepted message without video payload")
|
||||
await worker.mark_media_uploaded(int(media["id"]), message.video.file_id, message.video.file_unique_id)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
limit = int(os.getenv("BACKFILL_LIMIT", "200"))
|
||||
worker = TelegramStorageUploader()
|
||||
await worker.init()
|
||||
videos = await load_pending_videos(worker, limit)
|
||||
jlog("start", count=len(videos), limit=limit)
|
||||
ok = 0
|
||||
failed = 0
|
||||
link_only = 0
|
||||
try:
|
||||
for media in videos:
|
||||
media_id = int(media["id"])
|
||||
raw_post_id = int(media["raw_post_id"])
|
||||
path = str(TMP_DIR / f"{TMP_PREFIX}backfill_{raw_post_id}_{media_id}.mp4")
|
||||
jlog(
|
||||
"video_start",
|
||||
media_id=media_id,
|
||||
raw_post_id=raw_post_id,
|
||||
url=media.get("original_url"),
|
||||
duration_sec=media.get("duration_sec"),
|
||||
)
|
||||
try:
|
||||
info = await worker.download_video(str(media.get("original_url") or ""), path)
|
||||
if not info:
|
||||
await worker.mark_media_failed_attempt(media_id, "video download failed")
|
||||
failed += 1
|
||||
jlog("video_failed", media_id=media_id, raw_post_id=raw_post_id, error="video download failed")
|
||||
continue
|
||||
if info.get("error"):
|
||||
if info.get("permanent"):
|
||||
await worker.mark_media_link_only(media_id, str(info["error"]))
|
||||
link_only += 1
|
||||
else:
|
||||
await worker.mark_media_failed_attempt(media_id, str(info["error"]))
|
||||
failed += 1
|
||||
jlog("video_failed", media_id=media_id, raw_post_id=raw_post_id, error=info.get("error"))
|
||||
continue
|
||||
await send_video_to_storage(worker, media, path)
|
||||
ok += 1
|
||||
jlog("video_uploaded", media_id=media_id, raw_post_id=raw_post_id, size_bytes=info.get("size_bytes"))
|
||||
await asyncio.sleep(worker.media_upload_delay_sec)
|
||||
except Exception as exc:
|
||||
await worker.mark_media_failed_attempt(media_id, str(exc))
|
||||
failed += 1
|
||||
jlog("video_failed", media_id=media_id, raw_post_id=raw_post_id, error=str(exc))
|
||||
finally:
|
||||
Path(path).unlink(missing_ok=True)
|
||||
finally:
|
||||
await worker.close()
|
||||
if worker.pool:
|
||||
await worker.pool.close()
|
||||
jlog("done", ok=ok, failed=failed, link_only=link_only, total=len(videos))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,524 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import html
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import fcntl
|
||||
except ImportError: # pragma: no cover - production runs on Linux
|
||||
fcntl = None
|
||||
|
||||
import aiohttp
|
||||
from aiogram import Bot
|
||||
from aiogram.client.session.aiohttp import AiohttpSession
|
||||
from aiogram.client.telegram import TelegramAPIServer
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
from vk_parser_app.config import settings
|
||||
from vk_parser_app.db import close_pool, fetch_setting, get_pool
|
||||
|
||||
|
||||
SEPARATOR_RE = re.compile(r"^━+$")
|
||||
HASHTAG_LINE_RE = re.compile(r"^(?:#[\wа-яё]+\s*)+$", re.IGNORECASE)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Publish selected accepted posts to Ghost")
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument("--ids", help="Comma-separated raw_post ids")
|
||||
group.add_argument("--all-ready", action="store_true", help="Publish every photo-ready post not yet published to site")
|
||||
group.add_argument("--all-ready-videos", action="store_true", help="Publish video-only ready posts")
|
||||
parser.add_argument("--ghost-url", default="https://f-n8.ru")
|
||||
parser.add_argument("--key-file", default="/opt/vk-parser/.site-poster-key")
|
||||
parser.add_argument("--limit", type=int, default=0, help="Maximum posts to publish in this run; 0 means unlimited")
|
||||
parser.add_argument("--retry-failed", action="store_true", help="Include previous site publish_failed rows")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def ghost_token(key: str) -> str:
|
||||
key_id, secret = key.strip().split(":", 1)
|
||||
now = int(time.time())
|
||||
|
||||
def encode(value: dict[str, Any] | bytes) -> str:
|
||||
raw = value if isinstance(value, bytes) else json.dumps(value, separators=(",", ":")).encode()
|
||||
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode()
|
||||
|
||||
header = encode({"alg": "HS256", "typ": "JWT", "kid": key_id})
|
||||
payload = encode({"iat": now, "exp": now + 300, "aud": "/admin/"})
|
||||
signature = encode(hmac.new(bytes.fromhex(secret), f"{header}.{payload}".encode(), hashlib.sha256).digest())
|
||||
return f"{header}.{payload}.{signature}"
|
||||
|
||||
|
||||
def split_post_text(value: str) -> tuple[str, str]:
|
||||
lines = [line.strip() for line in str(value or "").replace("\r", "").split("\n")]
|
||||
while lines and not lines[0]:
|
||||
lines.pop(0)
|
||||
title = lines.pop(0) if lines else "Новость Forma N8"
|
||||
while lines and (not lines[0] or SEPARATOR_RE.fullmatch(lines[0])):
|
||||
lines.pop(0)
|
||||
while lines and (not lines[-1] or HASHTAG_LINE_RE.fullmatch(lines[-1])):
|
||||
lines.pop()
|
||||
body = "\n".join(lines).strip()
|
||||
return title[:255], body
|
||||
|
||||
|
||||
def paragraphs(value: str) -> str:
|
||||
chunks = [chunk.strip() for chunk in re.split(r"\n\s*\n", value) if chunk.strip()]
|
||||
return "".join(f"<p>{html.escape(chunk).replace(chr(10), '<br>')}</p>" for chunk in chunks)
|
||||
|
||||
|
||||
def excerpt(value: str, limit: int = 260) -> str:
|
||||
clean = re.sub(r"\s+", " ", value).strip()
|
||||
if len(clean) <= limit:
|
||||
return clean
|
||||
return clean[: limit - 1].rsplit(" ", 1)[0] + "…"
|
||||
|
||||
|
||||
def compress_photo(data: bytes) -> tuple[bytes, int, int]:
|
||||
with Image.open(io.BytesIO(data)) as source:
|
||||
image = ImageOps.exif_transpose(source)
|
||||
if image.mode not in {"RGB", "RGBA"}:
|
||||
image = image.convert("RGB")
|
||||
image.thumbnail((2000, 2000), Image.Resampling.LANCZOS)
|
||||
output = io.BytesIO()
|
||||
image.save(output, "WEBP", quality=82, method=6)
|
||||
return output.getvalue(), image.width, image.height
|
||||
|
||||
|
||||
def video_thumbnail(data: bytes) -> tuple[bytes, int, int]:
|
||||
video_path = ""
|
||||
image_path = ""
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as video_file:
|
||||
video_file.write(data)
|
||||
video_path = video_file.name
|
||||
with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as image_file:
|
||||
image_path = image_file.name
|
||||
command = [
|
||||
"ffmpeg", "-y", "-ss", "1", "-i", video_path,
|
||||
"-frames:v", "1", "-vf", "scale='min(2000,iw)':-2", "-q:v", "3", image_path,
|
||||
]
|
||||
result = subprocess.run(command, capture_output=True, timeout=90)
|
||||
if result.returncode != 0 or not Path(image_path).exists() or Path(image_path).stat().st_size == 0:
|
||||
command[3] = "0"
|
||||
result = subprocess.run(command, capture_output=True, timeout=90)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError("ffmpeg thumbnail extraction failed")
|
||||
return compress_photo(Path(image_path).read_bytes())
|
||||
finally:
|
||||
for path in (video_path, image_path):
|
||||
if path:
|
||||
try:
|
||||
os.unlink(path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
def normalize_video(data: bytes, duration: int, height: int | None) -> bytes:
|
||||
max_bytes = 40 * 1024 * 1024
|
||||
if len(data) <= max_bytes and (not height or height <= 720):
|
||||
return data
|
||||
input_path = ""
|
||||
output_path = ""
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as input_file:
|
||||
input_file.write(data)
|
||||
input_path = input_file.name
|
||||
with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as output_file:
|
||||
output_path = output_file.name
|
||||
safe_duration = max(1, int(duration or 1))
|
||||
target_total_kbps = int((38 * 1024 * 8) / safe_duration)
|
||||
video_kbps = max(280, min(2800, target_total_kbps - 96))
|
||||
command = [
|
||||
"ffmpeg", "-y", "-i", input_path,
|
||||
"-map_metadata", "-1", "-c:v", "libx264", "-preset", "fast",
|
||||
"-b:v", f"{video_kbps}k", "-maxrate", f"{video_kbps}k", "-bufsize", f"{video_kbps * 2}k",
|
||||
]
|
||||
if height and int(height) > 720:
|
||||
command.extend(["-vf", "scale=-2:720"])
|
||||
command.extend(["-c:a", "aac", "-b:a", "96k", "-movflags", "+faststart", output_path])
|
||||
result = subprocess.run(command, capture_output=True, timeout=max(180, safe_duration * 3))
|
||||
if result.returncode != 0 or not Path(output_path).exists():
|
||||
raise RuntimeError("ffmpeg video normalization failed")
|
||||
normalized = Path(output_path).read_bytes()
|
||||
if len(normalized) > 48 * 1024 * 1024:
|
||||
raise RuntimeError(f"normalized video remains too large: {len(normalized)} bytes")
|
||||
return normalized
|
||||
finally:
|
||||
for path in (input_path, output_path):
|
||||
if path:
|
||||
try:
|
||||
os.unlink(path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
def gallery_html(images: list[dict[str, Any]], title: str) -> str:
|
||||
if not images:
|
||||
return ""
|
||||
if len(images) == 1:
|
||||
image = images[0]
|
||||
return (
|
||||
'<figure class="kg-card kg-image-card kg-width-wide">'
|
||||
f'<img src="{html.escape(image["url"], quote=True)}" '
|
||||
f'width="{image["width"]}" height="{image["height"]}" '
|
||||
f'alt="{html.escape(title, quote=True)}" loading="lazy">'
|
||||
"</figure>"
|
||||
)
|
||||
rows = []
|
||||
for offset in range(0, len(images), 3):
|
||||
cells = []
|
||||
for image in images[offset : offset + 3]:
|
||||
cells.append(
|
||||
'<div class="kg-gallery-image">'
|
||||
f'<img src="{html.escape(image["url"], quote=True)}" '
|
||||
f'width="{image["width"]}" height="{image["height"]}" '
|
||||
f'alt="{html.escape(title, quote=True)}" loading="lazy">'
|
||||
"</div>"
|
||||
)
|
||||
rows.append('<div class="kg-gallery-row">' + "".join(cells) + "</div>")
|
||||
return '<figure class="kg-card kg-gallery-card kg-width-wide"><div class="kg-gallery-container">' + "".join(rows) + "</div></figure>"
|
||||
|
||||
|
||||
def videos_html(videos: list[dict[str, Any]], title: str, poster_url: str) -> str:
|
||||
return "".join(
|
||||
'<figure class="kg-card kg-video-card kg-width-wide">'
|
||||
f'<video src="{html.escape(video["url"], quote=True)}" '
|
||||
f'poster="{html.escape(poster_url, quote=True)}" controls playsinline preload="metadata" '
|
||||
f'aria-label="{html.escape(title, quote=True)}"></video>'
|
||||
"</figure>"
|
||||
for video in videos
|
||||
)
|
||||
|
||||
|
||||
def lexical_paragraph(text: str) -> dict[str, Any]:
|
||||
return {
|
||||
"children": [{"detail": 0, "format": 0, "mode": "normal", "style": "", "text": text, "type": "extended-text", "version": 1}],
|
||||
"direction": None,
|
||||
"format": "",
|
||||
"indent": 0,
|
||||
"type": "paragraph",
|
||||
"version": 1,
|
||||
}
|
||||
|
||||
|
||||
def video_lexical(videos: list[dict[str, Any]], body: str, thumbnail: dict[str, Any]) -> str:
|
||||
children: list[dict[str, Any]] = []
|
||||
for video in videos:
|
||||
children.append(
|
||||
{
|
||||
"type": "video",
|
||||
"version": 1,
|
||||
"src": video["url"],
|
||||
"caption": "",
|
||||
"fileName": video["file_name"],
|
||||
"mimeType": "video/mp4",
|
||||
"width": video.get("width"),
|
||||
"height": video.get("height"),
|
||||
"duration": video.get("duration") or 0,
|
||||
"thumbnailSrc": thumbnail["url"],
|
||||
"customThumbnailSrc": "",
|
||||
"thumbnailWidth": thumbnail["width"],
|
||||
"thumbnailHeight": thumbnail["height"],
|
||||
"cardWidth": "wide",
|
||||
"loop": False,
|
||||
}
|
||||
)
|
||||
children.extend(lexical_paragraph(chunk.strip()) for chunk in re.split(r"\n\s*\n", body) if chunk.strip())
|
||||
return json.dumps(
|
||||
{"root": {"children": children, "direction": None, "format": "", "indent": 0, "type": "root", "version": 1}},
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
|
||||
|
||||
class GhostClient:
|
||||
def __init__(self, session: aiohttp.ClientSession, base_url: str, key: str) -> None:
|
||||
self.session = session
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.key = key
|
||||
|
||||
def headers(self) -> dict[str, str]:
|
||||
return {"Authorization": f"Ghost {ghost_token(self.key)}", "Accept-Version": "v6.0"}
|
||||
|
||||
async def upload_image(self, data: bytes, name: str, ref: str) -> str:
|
||||
form = aiohttp.FormData()
|
||||
form.add_field("file", data, filename=name, content_type="image/webp")
|
||||
form.add_field("purpose", "image")
|
||||
form.add_field("ref", ref)
|
||||
async with self.session.post(
|
||||
f"{self.base_url}/ghost/api/admin/images/upload/", data=form, headers=self.headers()
|
||||
) as response:
|
||||
payload = await response.json(content_type=None)
|
||||
if response.status >= 300:
|
||||
raise RuntimeError(f"Ghost image upload {response.status}: {payload}")
|
||||
return str(payload["images"][0]["url"])
|
||||
|
||||
async def upload_media(self, data: bytes, name: str, content_type: str) -> str:
|
||||
form = aiohttp.FormData()
|
||||
form.add_field("file", data, filename=name, content_type=content_type)
|
||||
async with self.session.post(
|
||||
f"{self.base_url}/ghost/api/admin/media/upload/", data=form, headers=self.headers()
|
||||
) as response:
|
||||
payload = await response.json(content_type=None)
|
||||
if response.status >= 300:
|
||||
raise RuntimeError(f"Ghost media upload {response.status}: {payload}")
|
||||
return str(payload["media"][0]["url"])
|
||||
|
||||
async def create_post(self, post: dict[str, Any]) -> dict[str, Any]:
|
||||
suffix = "/" if "lexical" in post else "/?source=html"
|
||||
async with self.session.post(
|
||||
f"{self.base_url}/ghost/api/admin/posts{suffix}",
|
||||
json={"posts": [post]},
|
||||
headers=self.headers(),
|
||||
) as response:
|
||||
payload = await response.json(content_type=None)
|
||||
if response.status >= 300:
|
||||
raise RuntimeError(f"Ghost post create {response.status}: {payload}")
|
||||
return dict(payload["posts"][0])
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
args = parse_args()
|
||||
key = Path(args.key_file).read_text(encoding="utf-8").strip()
|
||||
pool = await get_pool()
|
||||
if args.all_ready or args.all_ready_videos:
|
||||
media_condition = (
|
||||
"""EXISTS (
|
||||
SELECT 1 FROM raw_post_media rpm
|
||||
WHERE rpm.raw_post_id=rp.id AND rpm.media_type='video'
|
||||
AND COALESCE(rpm.editor_hidden,FALSE)=FALSE
|
||||
AND COALESCE(rpm.tg_file_id,rpm.storage_attachment_id,'')<>''
|
||||
) AND NOT EXISTS (
|
||||
SELECT 1 FROM raw_post_media rpm
|
||||
WHERE rpm.raw_post_id=rp.id AND rpm.media_type='photo'
|
||||
AND COALESCE(rpm.editor_hidden,FALSE)=FALSE
|
||||
AND COALESCE(rpm.tg_file_id,rpm.storage_attachment_id,'')<>''
|
||||
)"""
|
||||
if args.all_ready_videos
|
||||
else """EXISTS (
|
||||
SELECT 1 FROM raw_post_media rpm
|
||||
WHERE rpm.raw_post_id=rp.id AND rpm.media_type='photo'
|
||||
AND COALESCE(rpm.editor_hidden,FALSE)=FALSE
|
||||
AND COALESCE(rpm.tg_file_id,rpm.storage_attachment_id,'')<>''
|
||||
)"""
|
||||
)
|
||||
candidates = [
|
||||
dict(row)
|
||||
for row in await pool.fetch(
|
||||
f"""
|
||||
SELECT rp.id,
|
||||
COALESCE(rp.final_category_tag,rp.rewrite_category_tag,'') AS category,
|
||||
COALESCE(rp.final_source_tag,rp.rewrite_source_tag,s.tag,'') AS source
|
||||
FROM raw_posts rp
|
||||
JOIN sources s ON s.id=rp.source_id
|
||||
JOIN content_categories cc ON cc.sort_order=COALESCE(rp.final_category_id,rp.rewrite_category_id)
|
||||
LEFT JOIN post_publications pp ON pp.raw_post_id=rp.id AND pp.platform='site'
|
||||
WHERE rp.status='storage_ready'
|
||||
AND rp.rewrite_status='ready'
|
||||
AND COALESCE(rp.editorial_status,'review') IN ('accepted','published','publish_failed')
|
||||
AND cc.site_enabled=TRUE
|
||||
AND COALESCE(pp.status,'pending') {"<> 'published'" if args.retry_failed else "= 'pending'"}
|
||||
AND {media_condition}
|
||||
ORDER BY COALESCE(rp.reviewed_at,rp.edited_at,rp.rewritten_at,rp.created_at),rp.id
|
||||
"""
|
||||
)
|
||||
]
|
||||
recent = [
|
||||
dict(row)
|
||||
for row in await pool.fetch(
|
||||
"""
|
||||
SELECT COALESCE(rp.final_category_tag,rp.rewrite_category_tag,'') AS category,
|
||||
COALESCE(rp.final_source_tag,rp.rewrite_source_tag,s.tag,'') AS source
|
||||
FROM post_publications pp
|
||||
JOIN raw_posts rp ON rp.id=pp.raw_post_id
|
||||
JOIN sources s ON s.id=rp.source_id
|
||||
WHERE pp.platform='site' AND pp.status='published'
|
||||
ORDER BY pp.published_at DESC NULLS LAST,pp.id DESC LIMIT 20
|
||||
"""
|
||||
)
|
||||
]
|
||||
selected: list[dict[str, Any]] = []
|
||||
while candidates:
|
||||
previous = selected[-1] if selected else (recent[0] if recent else {})
|
||||
recent_window = recent + selected[-20:]
|
||||
best = min(
|
||||
range(len(candidates)),
|
||||
key=lambda index: (
|
||||
int(bool(candidates[index]["category"]) and candidates[index]["category"] == previous.get("category"))
|
||||
+ int(bool(candidates[index]["source"]) and candidates[index]["source"] == previous.get("source")),
|
||||
sum(3 for row in recent_window if row.get("category") == candidates[index]["category"])
|
||||
+ sum(4 for row in recent_window if row.get("source") == candidates[index]["source"]),
|
||||
index,
|
||||
),
|
||||
)
|
||||
selected.append(candidates.pop(best))
|
||||
ids = [int(row["id"]) for row in selected]
|
||||
if args.limit > 0:
|
||||
ids = ids[: args.limit]
|
||||
else:
|
||||
ids = [int(item) for item in str(args.ids or "").split(",") if item.strip()]
|
||||
if not ids:
|
||||
print("NO_READY_POSTS")
|
||||
await close_pool()
|
||||
return
|
||||
print(f"QUEUE {len(ids)}")
|
||||
rows = await pool.fetch(
|
||||
"""
|
||||
SELECT rp.*, s.name AS source_name, s.tag AS source_tag,
|
||||
cc.site_name, cc.site_slug, cc.site_enabled,
|
||||
pp.status AS site_publication_status
|
||||
FROM raw_posts rp
|
||||
JOIN sources s ON s.id=rp.source_id
|
||||
JOIN content_categories cc ON cc.sort_order=COALESCE(rp.final_category_id, rp.rewrite_category_id)
|
||||
LEFT JOIN post_publications pp ON pp.raw_post_id=rp.id AND pp.platform='site'
|
||||
WHERE rp.id=ANY($1::bigint[])
|
||||
""",
|
||||
ids,
|
||||
)
|
||||
by_id = {int(row["id"]): dict(row) for row in rows}
|
||||
local_api = str(await fetch_setting("local_bot_api_url", settings.local_bot_api_url) or "").strip()
|
||||
bot_session = AiohttpSession(api=TelegramAPIServer.from_base(local_api.rstrip("/"), is_local=True)) if local_api else AiohttpSession()
|
||||
bot = Bot(settings.tg_bot_token, session=bot_session)
|
||||
timeout = aiohttp.ClientTimeout(total=120)
|
||||
published_base = datetime.now(timezone.utc) - timedelta(minutes=len(ids))
|
||||
try:
|
||||
async with aiohttp.ClientSession(timeout=timeout) as http_session:
|
||||
ghost = GhostClient(http_session, args.ghost_url, key)
|
||||
for position, raw_post_id in enumerate(ids):
|
||||
post = by_id.get(raw_post_id)
|
||||
if not post:
|
||||
print(f"SKIP {raw_post_id}: missing or unmapped category")
|
||||
continue
|
||||
if not post.get("site_enabled"):
|
||||
print(f"SKIP {raw_post_id}: category disabled for site")
|
||||
continue
|
||||
if post.get("site_publication_status") == "published":
|
||||
print(f"SKIP {raw_post_id}: already published")
|
||||
continue
|
||||
try:
|
||||
media_rows = await pool.fetch(
|
||||
"""
|
||||
SELECT * FROM raw_post_media
|
||||
WHERE raw_post_id=$1 AND media_type IN ('photo','video')
|
||||
AND COALESCE(editor_hidden,FALSE)=FALSE
|
||||
AND COALESCE(tg_file_id,storage_attachment_id,'')<>''
|
||||
ORDER BY sort_order,id
|
||||
""",
|
||||
raw_post_id,
|
||||
)
|
||||
uploaded_photos = []
|
||||
uploaded_videos = []
|
||||
thumbnail = None
|
||||
for index, media in enumerate(media_rows):
|
||||
file_id = str(media["tg_file_id"] or media["storage_attachment_id"])
|
||||
telegram_file = await bot.get_file(file_id)
|
||||
buffer = io.BytesIO()
|
||||
await bot.download_file(telegram_file.file_path, destination=buffer)
|
||||
data = buffer.getvalue()
|
||||
if media["media_type"] == "photo":
|
||||
compressed, width, height = compress_photo(data)
|
||||
url = await ghost.upload_image(compressed, f"post-{raw_post_id}-{index + 1}.webp", f"raw-post-{raw_post_id}-{index + 1}")
|
||||
uploaded_photos.append({"type": "photo", "url": url, "width": width, "height": height, "bytes": len(compressed)})
|
||||
else:
|
||||
data = normalize_video(data, int(media.get("duration_sec") or 0), media.get("height"))
|
||||
url = await ghost.upload_media(data, f"post-{raw_post_id}-{index + 1}.mp4", "video/mp4")
|
||||
uploaded_videos.append({
|
||||
"type": "video", "url": url, "bytes": len(data),
|
||||
"file_name": f"post-{raw_post_id}-{index + 1}.mp4",
|
||||
"width": media.get("width"), "height": media.get("height"),
|
||||
"duration": media.get("duration_sec") or 0,
|
||||
})
|
||||
if thumbnail is None:
|
||||
compressed, width, height = video_thumbnail(data)
|
||||
thumb_url = await ghost.upload_image(compressed, f"post-{raw_post_id}-video-cover.webp", f"raw-post-{raw_post_id}-video-cover")
|
||||
thumbnail = {"type": "thumbnail", "url": thumb_url, "width": width, "height": height, "bytes": len(compressed)}
|
||||
feature = uploaded_photos[0] if uploaded_photos else thumbnail
|
||||
if not feature:
|
||||
raise RuntimeError("no downloadable site media")
|
||||
title, body = split_post_text(post.get("final_text") or post.get("rewritten_text") or "")
|
||||
producer = str(post.get("source_name") or "").strip()
|
||||
content = gallery_html(uploaded_photos[1:], title) + videos_html(uploaded_videos, title, feature["url"]) + paragraphs(body)
|
||||
attachments = uploaded_photos + uploaded_videos + ([thumbnail] if thumbnail else [])
|
||||
payload = {
|
||||
"title": title,
|
||||
"status": "published",
|
||||
"published_at": (published_base + timedelta(minutes=position)).isoformat(),
|
||||
"custom_excerpt": excerpt(body),
|
||||
"meta_description": excerpt(body),
|
||||
"feature_image": feature["url"],
|
||||
"feature_image_alt": title,
|
||||
"tags": [
|
||||
{"name": str(post["site_name"]), "slug": str(post["site_slug"])},
|
||||
{"name": f"#producer:{producer}"},
|
||||
{"name": f"#raw-post:{raw_post_id}"},
|
||||
],
|
||||
}
|
||||
if uploaded_videos and not uploaded_photos:
|
||||
payload["lexical"] = video_lexical(uploaded_videos, body, thumbnail)
|
||||
else:
|
||||
payload["html"] = content
|
||||
created = await ghost.create_post(payload)
|
||||
await pool.execute(
|
||||
"""
|
||||
INSERT INTO post_publications(raw_post_id,platform,status,target_id,external_id,url,error,attachments_json,published_at)
|
||||
VALUES($1,'site','published',$2,$3,$4,NULL,$5::jsonb,NOW())
|
||||
ON CONFLICT(raw_post_id,platform) DO UPDATE SET
|
||||
status='published',target_id=EXCLUDED.target_id,external_id=EXCLUDED.external_id,
|
||||
url=EXCLUDED.url,error=NULL,attachments_json=EXCLUDED.attachments_json,
|
||||
published_at=NOW(),updated_at=NOW()
|
||||
""",
|
||||
raw_post_id,
|
||||
args.ghost_url,
|
||||
str(created["id"]),
|
||||
str(created["url"]),
|
||||
json.dumps(attachments, ensure_ascii=False),
|
||||
)
|
||||
print(f"PUBLISHED {raw_post_id} {created['url']} media={len(attachments)} bytes={sum(x['bytes'] for x in attachments)}")
|
||||
except Exception as exc:
|
||||
await pool.execute(
|
||||
"""
|
||||
INSERT INTO post_publications(raw_post_id,platform,status,target_id,error,updated_at)
|
||||
VALUES($1,'site','publish_failed',$2,$3,NOW())
|
||||
ON CONFLICT(raw_post_id,platform) DO UPDATE SET
|
||||
status='publish_failed',target_id=EXCLUDED.target_id,error=EXCLUDED.error,updated_at=NOW()
|
||||
""",
|
||||
raw_post_id,
|
||||
args.ghost_url,
|
||||
str(exc)[:2000],
|
||||
)
|
||||
print(f"FAILED {raw_post_id}: {exc}")
|
||||
finally:
|
||||
await bot.session.close()
|
||||
await close_pool()
|
||||
|
||||
|
||||
def run_locked() -> None:
|
||||
if fcntl is None:
|
||||
asyncio.run(main())
|
||||
return
|
||||
with Path("/tmp/forma-n8-site-poster.lock").open("w") as lock_file:
|
||||
try:
|
||||
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
print("ALREADY_RUNNING")
|
||||
return
|
||||
asyncio.run(main())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_locked()
|
||||
@@ -0,0 +1,35 @@
|
||||
import {createHmac} from "node:crypto";
|
||||
import {readFileSync} from "node:fs";
|
||||
|
||||
const baseUrl = "https://f-n8.ru";
|
||||
const [keyId, secret] = readFileSync("/root/ghost-admin-api-key", "utf8").trim().split(":");
|
||||
const encode = (value) => Buffer.from(JSON.stringify(value)).toString("base64url");
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const unsigned = `${encode({alg: "HS256", typ: "JWT", kid: keyId})}.${encode({iat: now, exp: now + 300, aud: "/admin/"})}`;
|
||||
const token = `${unsigned}.${createHmac("sha256", Buffer.from(secret, "hex")).update(unsigned).digest("base64url")}`;
|
||||
const headers = {Authorization: `Ghost ${token}`, "Content-Type": "application/json"};
|
||||
|
||||
const oldText = "Мы собираем материалы производителей и профильных магазинов, проверяем их релевантность и приводим к единому удобному формату.";
|
||||
const newText = "Ежедневно мы собираем новости снаряжения с сотен источников, отбираем лучшее и публикуем в одном месте!";
|
||||
|
||||
const response = await fetch(`${baseUrl}/ghost/api/admin/pages/slug/about/?formats=html`, {headers});
|
||||
if (!response.ok) throw new Error(`Ghost page read failed: ${response.status} ${await response.text()}`);
|
||||
const page = (await response.json()).pages[0];
|
||||
const html = String(page.html || "");
|
||||
if (!html.includes(oldText) && !String(page.custom_excerpt || "").includes(oldText)) {
|
||||
throw new Error("Expected about text was not found");
|
||||
}
|
||||
const update = {
|
||||
id: page.id,
|
||||
updated_at: page.updated_at,
|
||||
html: html.replaceAll(oldText, newText),
|
||||
custom_excerpt: String(page.custom_excerpt || "").replaceAll(oldText, newText),
|
||||
};
|
||||
const saved = await fetch(`${baseUrl}/ghost/api/admin/pages/${page.id}/?source=html`, {
|
||||
method: "PUT",
|
||||
headers,
|
||||
body: JSON.stringify({pages: [update]}),
|
||||
});
|
||||
if (!saved.ok) throw new Error(`Ghost page update failed: ${saved.status} ${await saved.text()}`);
|
||||
const result = (await saved.json()).pages[0];
|
||||
console.log(JSON.stringify({id: result.id, slug: result.slug, updated_at: result.updated_at}));
|
||||
+1027
-44
File diff suppressed because it is too large
Load Diff
@@ -11,6 +11,10 @@ POST_STATUS_STORAGE_READY = "storage_ready"
|
||||
POST_STATUS_SKIPPED = "skipped"
|
||||
POST_STATUS_FAILED = "failed"
|
||||
|
||||
PUBLICATION_STATUS_PENDING = "pending"
|
||||
PUBLICATION_STATUS_PUBLISHED = "published"
|
||||
PUBLICATION_STATUS_FAILED = "publish_failed"
|
||||
|
||||
MEDIA_STATUS_PENDING = "pending"
|
||||
MEDIA_STATUS_UPLOADED = "uploaded"
|
||||
MEDIA_STATUS_LINK_ONLY = "link_only"
|
||||
@@ -28,3 +32,9 @@ WORKER_PARSER = "vk-parser"
|
||||
WORKER_STORAGE_UPLOADER = "vk-storage-uploader"
|
||||
WORKER_AI_QUALIFIER = "ai-qualifier"
|
||||
WORKER_AI_WRITER = "ai-writer"
|
||||
WORKER_TG_POSTER = "tg-poster"
|
||||
WORKER_TG_REACTOR = "tg-reactor"
|
||||
WORKER_VK_POSTER = "vk-poster"
|
||||
WORKER_MAX_POSTER = "max-poster"
|
||||
WORKER_SITE_POSTER = "site-poster"
|
||||
WORKER_DAILY_REPORT = "daily-report"
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<link rel="icon" type="image/png" href="/favi.png">
|
||||
<title>{{ title or "VK Parser Admin" }}</title>
|
||||
<style>
|
||||
:root { color-scheme: light; --bg:#f4f6f8; --panel:#fff; --panel-soft:#f8fafc; --line:#d8dee8; --line-strong:#c8d0dc; --text:#20242b; --muted:#647086; --accent:#245fd6; --accent-soft:#eef4ff; --danger:#b42318; --ok:#027a48; --shadow:0 1px 2px rgba(16,24,40,.04), 0 10px 30px rgba(16,24,40,.06); }
|
||||
@@ -65,7 +66,7 @@
|
||||
.source-cell a { font-weight:700; }
|
||||
.stage-cell { min-width:112px; }
|
||||
.raw-table { table-layout:fixed; }
|
||||
.raw-table th:nth-child(1), .raw-table td:nth-child(1) { width:58px; }
|
||||
.raw-table th:nth-child(1), .raw-table td:nth-child(1) { width:82px; }
|
||||
.raw-table th:nth-child(2), .raw-table td:nth-child(2) { width:170px; }
|
||||
.raw-table th:nth-child(3), .raw-table td:nth-child(3) { width:120px; }
|
||||
.raw-table th:nth-child(5), .raw-table td:nth-child(5) { width:96px; }
|
||||
@@ -191,6 +192,8 @@
|
||||
.sort-head:hover { color:var(--accent); }
|
||||
.sort-head.active:after { content:"↓"; font-size:11px; color:var(--accent); }
|
||||
.sort-head.active.asc:after { content:"↑"; }
|
||||
.raw-select-head, .raw-select-cell { display:flex; align-items:center; gap:8px; margin:0; }
|
||||
.raw-select-head input, .raw-select-cell input { width:16px; height:16px; margin:0; flex:0 0 auto; }
|
||||
.editor-list { display:grid; gap:12px; }
|
||||
.editor-card { display:grid; grid-template-columns:112px minmax(0,1fr) 132px; gap:16px; align-items:start; background:var(--panel); border:1px solid var(--line); border-radius:8px; padding:14px; box-shadow:var(--shadow); }
|
||||
.editor-media .media-strip { display:grid; grid-template-columns:repeat(2, 48px); overflow:visible; }
|
||||
@@ -198,7 +201,9 @@
|
||||
.editor-meta { display:flex; gap:8px; align-items:center; flex-wrap:wrap; color:var(--muted); font-size:13px; margin-bottom:8px; }
|
||||
.editor-meta a { color:var(--text); font-weight:800; text-decoration:none; }
|
||||
.editor-title-row { display:flex; gap:8px; align-items:center; flex-wrap:wrap; margin-bottom:8px; }
|
||||
.editor-text { white-space:pre-wrap; max-height:190px; overflow:hidden; line-height:1.55; }
|
||||
.publication-badges { display:flex; gap:6px; flex-wrap:wrap; margin:0 0 8px; }
|
||||
.publication-badges .pill { text-decoration:none; }
|
||||
.editor-text { white-space:pre-wrap; line-height:1.55; overflow-wrap:anywhere; }
|
||||
.source-compare { margin-top:10px; border-top:1px solid var(--line); padding-top:8px; }
|
||||
.source-compare summary { cursor:pointer; color:var(--muted); font-weight:700; }
|
||||
.compare-grid { display:grid; grid-template-columns:1.3fr .7fr; gap:14px; margin-top:8px; }
|
||||
@@ -230,6 +235,7 @@
|
||||
<a href="/editor">Редакторская</a>
|
||||
<a href="/prompt-test">Тест промптов</a>
|
||||
<a href="/workers">Воркеры</a>
|
||||
<a href="/logs">Логи</a>
|
||||
<a href="/users">Пользователи</a>
|
||||
</nav>
|
||||
<form method="post" action="/logout"><button class="btn" type="submit">{{ user.login }} · выйти</button></form>
|
||||
|
||||
@@ -69,13 +69,22 @@
|
||||
<span>{{ p.posted_at_fmt or p.created_at_fmt }}</span>
|
||||
</div>
|
||||
<div class="editor-title-row">
|
||||
<span class="pill {% if p.editorial_status == 'accepted' %}ok{% elif p.editorial_status == 'rejected' %}bad{% elif p.editorial_status == 'regenerating' %}warn{% endif %}">
|
||||
<span class="pill {% if p.editorial_status in ['accepted', 'published'] %}ok{% elif p.editorial_status in ['rejected', 'publish_failed'] %}bad{% elif p.editorial_status == 'regenerating' %}warn{% endif %}" title="{{ p.publication_error or p.tg_publication_url or '' }}">
|
||||
{{ p.editorial_status_label }}
|
||||
</span>
|
||||
<span class="pill">{{ p.review_category or "без категории" }}</span>
|
||||
<span class="muted">AI {{ p.qualification_score or "?" }}/10</span>
|
||||
</div>
|
||||
<div class="editor-text">{{ p.review_text }}</div>
|
||||
<div class="publication-badges">
|
||||
{% for pub in p.publication_platforms %}
|
||||
{% if pub.url %}
|
||||
<a class="pill {{ pub.class }}" href="{{ pub.url }}" target="_blank" title="{{ pub.error or pub.status_label }}">{{ pub.label }} · {{ pub.status_label }}</a>
|
||||
{% else %}
|
||||
<span class="pill {{ pub.class }}" title="{{ pub.error or pub.status_label }}">{{ pub.label }} · {{ pub.status_label }}</span>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
<div class="editor-text">{{ p.publication_preview_text }}</div>
|
||||
<details class="source-compare">
|
||||
<summary>Исходник и AI-заметки</summary>
|
||||
<div class="compare-grid">
|
||||
@@ -154,10 +163,11 @@
|
||||
<label><span>Категория</span>
|
||||
<select name="final_category">
|
||||
{% for category in categories %}
|
||||
<option value="{{ category }}" {% if p.review_category == category %}selected{% endif %}>{{ category }}</option>
|
||||
<option value="{{ category.name }}" data-tag="{{ category.tag }}" {% if p.review_category == category.name %}selected{% endif %}>{{ category.name }}</option>
|
||||
{% endfor %}
|
||||
{% if p.review_category and p.review_category not in categories %}
|
||||
<option value="{{ p.review_category }}" selected>{{ p.review_category }} · старая категория</option>
|
||||
{% set category_names = categories | map(attribute="name") | list %}
|
||||
{% if p.review_category and p.review_category not in category_names %}
|
||||
<option value="{{ p.review_category }}" data-tag="{{ p.review_category_tag }}" selected>{{ p.review_category }} · старая категория</option>
|
||||
{% endif %}
|
||||
</select>
|
||||
</label>
|
||||
@@ -165,6 +175,9 @@
|
||||
<input name="final_source_tag" value="{{ p.review_source_tag or p.source_tag or '' }}">
|
||||
</label>
|
||||
</div>
|
||||
<label><span>Предпросмотр публикации</span>
|
||||
<div class="publication-preview" data-publication-preview>{{ p.publication_preview_text }}</div>
|
||||
</label>
|
||||
<div class="hashtag-preview" data-hashtag-preview>#{{ p.review_category_tag or p.review_category or "category" }} #{{ p.review_source_tag or p.source_tag or "source" }}</div>
|
||||
<label><span>Заметка редактора</span>
|
||||
<textarea name="editor_notes" rows="3">{{ p.editor_notes or "" }}</textarea>
|
||||
@@ -253,9 +266,11 @@
|
||||
button.addEventListener("click", () => document.getElementById(button.dataset.closeDialog)?.close());
|
||||
});
|
||||
document.querySelectorAll(".edit-form").forEach((form) => {
|
||||
const textArea = form.querySelector('[name="final_text"]');
|
||||
const category = form.querySelector('[name="final_category"]');
|
||||
const source = form.querySelector('[name="final_source_tag"]');
|
||||
const preview = form.querySelector("[data-hashtag-preview]");
|
||||
const hashtagPreview = form.querySelector("[data-hashtag-preview]");
|
||||
const publicationPreview = form.querySelector("[data-publication-preview]");
|
||||
const normalize = (value, fallback) => {
|
||||
const tag = String(value || "").trim().replace(/^#+/, "").toLowerCase()
|
||||
.replace(/\s+/g, "_")
|
||||
@@ -265,9 +280,15 @@
|
||||
return tag || fallback;
|
||||
};
|
||||
const update = () => {
|
||||
if (!preview) return;
|
||||
preview.textContent = `#${normalize(category?.value, "category")} #${normalize(source?.value, "source")}`;
|
||||
const option = category?.selectedOptions?.[0];
|
||||
const tags = `#${normalize(option?.dataset?.tag || category?.value, "category")} #${normalize(source?.value, "source")}`;
|
||||
if (hashtagPreview) hashtagPreview.textContent = tags;
|
||||
if (publicationPreview) {
|
||||
const cleanText = String(textArea?.value || "").trim();
|
||||
publicationPreview.textContent = cleanText ? `${cleanText}\n\n${tags}` : tags;
|
||||
}
|
||||
};
|
||||
textArea?.addEventListener("input", update);
|
||||
category?.addEventListener("change", update);
|
||||
source?.addEventListener("input", update);
|
||||
update();
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
<div class="page-head">
|
||||
<div>
|
||||
<h1>Логи</h1>
|
||||
<div class="muted">Журнал действий и запросов админки. События старше 30 дней удаляются автоматически.</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<section class="panel">
|
||||
<form class="filter-grid" method="get" action="/logs">
|
||||
<label><span>Поиск</span><input name="q" value="{{ q }}" placeholder="action, entity, JSON"></label>
|
||||
<label><span>Действие</span>
|
||||
<select name="action">
|
||||
<option value="">Все</option>
|
||||
{% for item in actions %}
|
||||
<option value="{{ item.action }}" {% if action == item.action %}selected{% endif %}>{{ item.action }} · {{ item.count }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</label>
|
||||
<label><span>Пользователь</span>
|
||||
<select name="actor_id">
|
||||
<option value="0">Все</option>
|
||||
{% for item in actors %}
|
||||
<option value="{{ item.id }}" {% if actor_id == item.id %}selected{% endif %}>{{ item.login }} · {{ item.count }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</label>
|
||||
<label><span>Сущность</span>
|
||||
<select name="entity_type">
|
||||
<option value="">Все</option>
|
||||
{% for item in entity_types %}
|
||||
<option value="{{ item.entity_type }}" {% if entity_type == item.entity_type %}selected{% endif %}>{{ item.entity_type }} · {{ item.count }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</label>
|
||||
<label><span>С даты</span><input type="date" name="date_from" value="{{ date_from }}"></label>
|
||||
<label><span>По дату</span><input type="date" name="date_to" value="{{ date_to }}"></label>
|
||||
<button class="btn primary filter-submit" type="submit">Показать</button>
|
||||
<a class="btn filter-submit" href="/logs">Сбросить</a>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<div class="list-footer">
|
||||
<div class="muted">Всего: {{ pagination.total }}</div>
|
||||
<form class="row per-page-form" method="get" action="/logs">
|
||||
{% for item in preserved_query %}
|
||||
{% if item.key != "per_page" %}
|
||||
<input type="hidden" name="{{ item.key }}" value="{{ item.value }}">
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
<label><span>На странице</span>
|
||||
<select name="per_page" data-autosubmit>
|
||||
{% for n in [25,50,100,200] %}
|
||||
<option value="{{ n }}" {% if pagination.per_page == n %}selected{% endif %}>{{ n }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</label>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<table class="compact-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Время</th>
|
||||
<th>Пользователь</th>
|
||||
<th>Действие</th>
|
||||
<th>Сущность</th>
|
||||
<th>Детали</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for item in logs %}
|
||||
<tr>
|
||||
<td class="mono">{{ item.created_at_fmt }}</td>
|
||||
<td>{{ item.actor_label }}</td>
|
||||
<td><span class="pill">{{ item.action }}</span></td>
|
||||
<td>{{ item.entity_type }}{% if item.entity_id %} #{{ item.entity_id }}{% endif %}</td>
|
||||
<td>
|
||||
{% if item.details_pretty %}
|
||||
<details class="nested-details">
|
||||
<summary>JSON</summary>
|
||||
<pre class="json-box">{{ item.details_pretty }}</pre>
|
||||
</details>
|
||||
{% else %}
|
||||
<span class="muted">—</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% else %}
|
||||
<tr><td colspan="5" class="empty-state">Записей нет.</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
{% include "pagination.html" %}
|
||||
</section>
|
||||
|
||||
<script>
|
||||
document.querySelectorAll("[data-autosubmit]").forEach((input) => {
|
||||
input.addEventListener("change", () => input.form?.submit());
|
||||
});
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -8,12 +8,24 @@
|
||||
</div>
|
||||
<div class="catalog-layout">
|
||||
<section class="panel catalog-main">
|
||||
<div class="list-toolbar">
|
||||
<div class="muted">Всего: {{ pagination.total }}</div>
|
||||
</div>
|
||||
<table class="raw-table">
|
||||
<form id="raw-bulk-form" method="post" action="/raw/send-to-rewrite">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<input type="hidden" name="next_url" value="{{ request.url.path }}{% if request.url.query %}?{{ request.url.query }}{% endif %}">
|
||||
<div class="list-toolbar">
|
||||
<div>
|
||||
<div class="muted">Всего: {{ pagination.total }}</div>
|
||||
<div class="muted">Выбранные storage-ready посты можно вручную отправить в рерайт.</div>
|
||||
</div>
|
||||
<button class="btn primary" type="submit" data-bulk-submit disabled>Отправить в рерайт</button>
|
||||
</div>
|
||||
<table class="raw-table">
|
||||
<tr>
|
||||
<th><a class="sort-head {% if sort_headers.id.active %}active {{ sort_headers.id.direction }}{% endif %}" href="{{ sort_headers.id.url }}">#</a></th>
|
||||
<th>
|
||||
<label class="raw-select-head">
|
||||
<input type="checkbox" data-raw-select-all aria-label="Выбрать все посты на странице">
|
||||
<a class="sort-head {% if sort_headers.id.active %}active {{ sort_headers.id.direction }}{% endif %}" href="{{ sort_headers.id.url }}">#</a>
|
||||
</label>
|
||||
</th>
|
||||
<th><a class="sort-head {% if sort_headers.source.active %}active {{ sort_headers.source.direction }}{% endif %}" href="{{ sort_headers.source.url }}">Источник</a></th>
|
||||
<th><a class="sort-head {% if sort_headers.stage.active %}active {{ sort_headers.stage.direction }}{% endif %}" href="{{ sort_headers.stage.url }}">Этап</a></th>
|
||||
<th><a class="sort-head {% if sort_headers.post.active %}active {{ sort_headers.post.direction }}{% endif %}" href="{{ sort_headers.post.url }}">Пост</a></th>
|
||||
@@ -21,7 +33,12 @@
|
||||
</tr>
|
||||
{% for p in posts %}
|
||||
<tr>
|
||||
<td><a class="raw-id" href="/raw/{{ p.id }}">{{ p.id }}</a></td>
|
||||
<td>
|
||||
<label class="raw-select-cell">
|
||||
<input type="checkbox" name="raw_post_id" value="{{ p.id }}" data-raw-select aria-label="Выбрать пост {{ p.id }}">
|
||||
<a class="raw-id" href="/raw/{{ p.id }}">{{ p.id }}</a>
|
||||
</label>
|
||||
</td>
|
||||
<td class="source-cell">
|
||||
<a href="{{ p.original_url }}" target="_blank">{{ p.source_name }}</a>
|
||||
<div class="muted">{{ p.source_platform }} · #{{ p.source_tag or "—" }}</div>
|
||||
@@ -66,7 +83,8 @@
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
</table>
|
||||
</form>
|
||||
<div class="list-footer">
|
||||
<form class="row per-page-form" method="get" action="/raw">
|
||||
{% for item in preserved_query %}
|
||||
@@ -206,6 +224,26 @@
|
||||
});
|
||||
});
|
||||
});
|
||||
const rawChecks = Array.from(document.querySelectorAll("[data-raw-select]"));
|
||||
const rawSelectAll = document.querySelector("[data-raw-select-all]");
|
||||
const bulkSubmit = document.querySelector("[data-bulk-submit]");
|
||||
function syncBulkState() {
|
||||
const selected = rawChecks.filter((item) => item.checked).length;
|
||||
if (bulkSubmit) {
|
||||
bulkSubmit.disabled = selected === 0;
|
||||
bulkSubmit.textContent = selected ? `Отправить в рерайт (${selected})` : "Отправить в рерайт";
|
||||
}
|
||||
if (rawSelectAll) {
|
||||
rawSelectAll.checked = selected > 0 && selected === rawChecks.length;
|
||||
rawSelectAll.indeterminate = selected > 0 && selected < rawChecks.length;
|
||||
}
|
||||
}
|
||||
rawChecks.forEach((item) => item.addEventListener("change", syncBulkState));
|
||||
rawSelectAll?.addEventListener("change", () => {
|
||||
rawChecks.forEach((item) => { item.checked = rawSelectAll.checked; });
|
||||
syncBulkState();
|
||||
});
|
||||
syncBulkState();
|
||||
})();
|
||||
</script>
|
||||
{% endblock %}
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
<!doctype html>
|
||||
<html lang="ru" data-theme="light">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<link rel="icon" type="image/png" href="/favi.png">
|
||||
<title>{{ title or "VK Parser Admin V2" }}</title>
|
||||
|
||||
<!-- Tailwind CSS & DaisyUI -->
|
||||
<link href="https://cdn.jsdelivr.net/npm/daisyui@4.10.1/dist/full.min.css" rel="stylesheet" type="text/css" />
|
||||
<script src="https://cdn.tailwindcss.com"></script>
|
||||
|
||||
<!-- HTMX -->
|
||||
<script src="https://unpkg.com/htmx.org@1.9.11"></script>
|
||||
<!-- AlpineJS (optional but useful for local state) -->
|
||||
<script defer src="https://cdn.jsdelivr.net/npm/alpinejs@3.13.8/dist/cdn.min.js"></script>
|
||||
|
||||
<style>
|
||||
/* Custom tweaks to match Tailwind typography nicely */
|
||||
body { font-family: 'Inter', system-ui, sans-serif; }
|
||||
.htmx-indicator { display:none; }
|
||||
.htmx-request .htmx-indicator { display:inline-block; }
|
||||
.htmx-request.htmx-indicator { display:inline-block; }
|
||||
</style>
|
||||
</head>
|
||||
<body class="bg-base-200 min-h-screen">
|
||||
|
||||
<div class="navbar bg-base-100 shadow-sm sticky top-0 z-50">
|
||||
<div class="flex-1 px-2 mx-2">
|
||||
<span class="text-lg font-bold">Parser V2</span>
|
||||
<div class="hidden md:flex ml-6 gap-2">
|
||||
<a href="/v2/raw" class="btn btn-ghost btn-sm {{ 'btn-active' if 'raw' in request.url.path }}">Raw-посты</a>
|
||||
<!-- Temporary links to V1 for others -->
|
||||
<a href="/editor" class="btn btn-ghost btn-sm">Редактор (V1)</a>
|
||||
<a href="/workers" class="btn btn-ghost btn-sm">Воркеры (V1)</a>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex-none hidden lg:block">
|
||||
<ul class="menu menu-horizontal gap-2">
|
||||
<li><a href="/" class="btn btn-outline btn-sm">Вернуться в V1</a></li>
|
||||
<li>
|
||||
<form method="post" action="/logout" class="m-0">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<button class="btn btn-ghost btn-sm text-error">Выйти</button>
|
||||
</form>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<main class="container mx-auto p-4 md:p-6 max-w-7xl">
|
||||
{% block body %}{% endblock %}
|
||||
</main>
|
||||
|
||||
<script>
|
||||
// Handle HTMX errors
|
||||
document.body.addEventListener('htmx:responseError', function(evt) {
|
||||
alert("Ошибка при загрузке: " + evt.detail.xhr.status);
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,14 @@
|
||||
{% extends "v2/base.html" %}
|
||||
|
||||
{% block body %}
|
||||
<div class="mb-6 flex justify-between items-end">
|
||||
<div>
|
||||
<h1 class="text-3xl font-bold mb-1">Raw-посты</h1>
|
||||
<div class="text-base-content/60">Исходный пост, этап обработки и AI-оценка.</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="content-area">
|
||||
{% include "v2/raw_posts_content.html" %}
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,152 @@
|
||||
<div class="flex flex-col lg:flex-row gap-6">
|
||||
<!-- Sidebar Filters -->
|
||||
<aside class="w-full lg:w-72 flex-shrink-0">
|
||||
<div class="card bg-base-100 shadow-sm border border-base-200">
|
||||
<div class="card-body p-4">
|
||||
<form hx-get="/v2/raw" hx-target="#content-area" hx-push-url="true" hx-trigger="submit, change delay:300ms">
|
||||
<input type="hidden" name="sort" value="{{ sort }}">
|
||||
|
||||
<div class="flex justify-between items-center mb-4">
|
||||
<h2 class="card-title text-lg">Фильтры</h2>
|
||||
<a href="/v2/raw" class="text-sm link link-hover text-base-content/60" hx-boost="true" hx-target="#content-area">Сбросить</a>
|
||||
</div>
|
||||
|
||||
<div class="form-control w-full mb-4">
|
||||
<label class="label"><span class="label-text font-bold">Оценка AI</span></label>
|
||||
<div class="flex gap-2">
|
||||
<input type="number" name="score_min" placeholder="От" value="{{ score_min }}" class="input input-bordered input-sm w-full" />
|
||||
<input type="number" name="score_max" placeholder="До" value="{{ score_max }}" class="input input-bordered input-sm w-full" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-control w-full mb-4">
|
||||
<label class="label"><span class="label-text font-bold">Дата поста</span></label>
|
||||
<div class="flex flex-col gap-2">
|
||||
<input type="date" name="date_from" value="{{ date_from }}" class="input input-bordered input-sm w-full" />
|
||||
<input type="date" name="date_to" value="{{ date_to }}" class="input input-bordered input-sm w-full" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-control w-full mb-4">
|
||||
<label class="label"><span class="label-text font-bold">Квалификация</span></label>
|
||||
<div class="bg-base-200 rounded-lg p-2 max-h-48 overflow-y-auto">
|
||||
{% for item in facets.qualification_statuses %}
|
||||
<label class="label cursor-pointer justify-start gap-2 p-1">
|
||||
<input type="checkbox" name="qualification_status" value="{{ item.value }}" class="checkbox checkbox-sm checkbox-primary" {% if item.value in qualification_statuses %}checked{% endif %} />
|
||||
<span class="label-text flex-1">{{ item.value }}</span>
|
||||
<span class="badge badge-sm badge-ghost">{{ item.count }}</span>
|
||||
</label>
|
||||
{% endendfor %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-control w-full mb-4">
|
||||
<label class="label"><span class="label-text font-bold">Рерайт</span></label>
|
||||
<div class="bg-base-200 rounded-lg p-2 max-h-48 overflow-y-auto">
|
||||
{% for item in facets.rewrite_statuses %}
|
||||
<label class="label cursor-pointer justify-start gap-2 p-1">
|
||||
<input type="checkbox" name="rewrite_status" value="{{ item.value }}" class="checkbox checkbox-sm checkbox-primary" {% if item.value in rewrite_statuses %}checked{% endif %} />
|
||||
<span class="label-text flex-1">{{ item.value }}</span>
|
||||
<span class="badge badge-sm badge-ghost">{{ item.count }}</span>
|
||||
</label>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<button class="btn btn-primary w-full mt-4" type="submit">Показать</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
<!-- Main Content (Table) -->
|
||||
<div class="flex-1 min-w-0">
|
||||
<div class="card bg-base-100 shadow-sm border border-base-200">
|
||||
<div class="card-body p-0">
|
||||
|
||||
<!-- Toolbar -->
|
||||
<div class="p-4 border-b border-base-200 flex justify-between items-center bg-base-100/50">
|
||||
<div class="text-sm text-base-content/70">
|
||||
Всего: <span class="font-bold">{{ pagination.total }}</span>
|
||||
</div>
|
||||
<!-- Pagination top (optional, or per page select) -->
|
||||
<div class="flex items-center gap-2">
|
||||
<span class="text-sm">На странице:</span>
|
||||
<select name="per_page" class="select select-bordered select-sm" form="pagination-form" hx-get="/v2/raw" hx-target="#content-area" hx-include="[name='date_from'], [name='date_to']">
|
||||
{% for n in [25,50,100,200] %}
|
||||
<option value="{{ n }}" {% if pagination.per_page == n %}selected{% endif %}>{{ n }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="overflow-x-auto">
|
||||
<table class="table table-zebra w-full text-sm">
|
||||
<thead>
|
||||
<tr class="bg-base-200 text-base-content">
|
||||
<th class="w-16">#</th>
|
||||
<th class="w-48">Источник</th>
|
||||
<th class="w-32">Этап</th>
|
||||
<th>Пост</th>
|
||||
<th class="w-32 text-center">AI</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for p in posts %}
|
||||
<tr class="hover">
|
||||
<td class="font-bold">
|
||||
<a href="/raw/{{ p.id }}" class="link link-primary">{{ p.id }}</a>
|
||||
</td>
|
||||
<td>
|
||||
<a href="{{ p.original_url }}" target="_blank" class="font-medium hover:underline">{{ p.source_name }}</a>
|
||||
<div class="text-xs text-base-content/60 mt-1">{{ p.source_platform }} · #{{ p.source_tag or "—" }}</div>
|
||||
{% if p.posted_at_fmt %}<div class="text-xs text-base-content/60 mt-1">{{ p.posted_at_fmt }}</div>{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
<div class="badge badge-sm {% if p.stage.class == 'ok' %}badge-success{% elif p.stage.class == 'bad' %}badge-error{% elif p.stage.class == 'warn' %}badge-warning{% else %}badge-ghost{% endif %} gap-1">
|
||||
{{ p.stage.label }}
|
||||
</div>
|
||||
{% if p.error_reason %}<div class="text-xs text-error mt-1">{{ p.error_reason }}</div>{% endif %}
|
||||
</td>
|
||||
<td class="max-w-md">
|
||||
<div class="prose prose-sm leading-snug">
|
||||
{{ p.raw_text[:200] }}{% if p.raw_text|length > 200 %}...{% endif %}
|
||||
</div>
|
||||
</td>
|
||||
<td class="text-center">
|
||||
<div class="flex flex-col items-center gap-1">
|
||||
<span class="text-lg font-bold {% if p.ai_badge.class == 'ok' %}text-success{% elif p.ai_badge.class == 'bad' %}text-error{% elif p.ai_badge.class == 'warn' %}text-warning{% endif %}">
|
||||
{{ p.ai_badge.label }}
|
||||
</span>
|
||||
<span class="text-xs text-base-content/60">{{ p.ai_badge.sublabel }}</span>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<!-- Pagination Bottom -->
|
||||
<div class="p-4 border-t border-base-200 flex justify-end">
|
||||
<div class="join">
|
||||
{% if pagination.has_prev %}
|
||||
<button class="join-item btn btn-sm" hx-get="{{ pagination.prev_url.replace('/raw', '/v2/raw') }}" hx-target="#content-area" hx-push-url="true">« Пред</button>
|
||||
{% else %}
|
||||
<button class="join-item btn btn-sm btn-disabled">« Пред</button>
|
||||
{% endif %}
|
||||
|
||||
<button class="join-item btn btn-sm btn-active">Стр. {{ pagination.page }} из {{ pagination.total_pages }}</button>
|
||||
|
||||
{% if pagination.has_next %}
|
||||
<button class="join-item btn btn-sm" hx-get="{{ pagination.next_url.replace('/raw', '/v2/raw') }}" hx-target="#content-area" hx-push-url="true">След »</button>
|
||||
{% else %}
|
||||
<button class="join-item btn btn-sm btn-disabled">След »</button>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -1,17 +1,37 @@
|
||||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
<div class="panel" style="max-width:760px;margin:8vh auto 0;">
|
||||
<h1>VK OAuth callback</h1>
|
||||
{% if code %}
|
||||
<p class="muted">Скопируйте code и code_verifier для обмена на access token.</p>
|
||||
<h1>VK OAuth</h1>
|
||||
{% if result %}
|
||||
{% if result.ok %}
|
||||
<p class="pill ok">Токен VK-постера сохранён</p>
|
||||
<dl class="kv compact-kv">
|
||||
<dt>Сообщество</dt><dd>{{ result.owner_id }}</dd>
|
||||
<dt>User ID</dt><dd>{{ result.user_id or "—" }}</dd>
|
||||
<dt>Живёт, сек</dt><dd>{{ result.expires_in or "—" }}</dd>
|
||||
<dt>Истекает</dt><dd>{{ result.expires_at or "—" }}</dd>
|
||||
<dt>Refresh token</dt><dd>{% if result.has_refresh_token %}OK{% else %}Не получен{% endif %}</dd>
|
||||
<dt>wall.get</dt><dd>{% if result.wall_get_ok %}OK{% else %}Ошибка: {{ result.wall_get_error }}{% endif %}</dd>
|
||||
<dt>Фото upload</dt>
|
||||
<dd>
|
||||
{% if result.photo_upload_server_ok %}
|
||||
OK
|
||||
{% else %}
|
||||
Недоступно: {{ result.photo_upload_server_error }}
|
||||
<div class="muted">VK выдаёт право photos для upload API отдельно; постер всё равно опубликует текст и существующие VK-вложения.</div>
|
||||
{% endif %}
|
||||
</dd>
|
||||
</dl>
|
||||
<p><a class="btn primary" href="/workers">Вернуться к воркерам</a></p>
|
||||
{% else %}
|
||||
<p class="pill bad">VK OAuth не завершился</p>
|
||||
<p>{{ result.error_description or result.error or "Неизвестная ошибка" }}</p>
|
||||
<p><a class="btn" href="/workers">Вернуться к воркерам</a> <a class="btn primary" href="/vk-oauth/start">Повторить</a></p>
|
||||
{% endif %}
|
||||
{% elif code %}
|
||||
<p class="muted">VK вернул code. Новый обработчик должен обменивать его автоматически через <span class="mono">/vk-oauth/callback</span>.</p>
|
||||
<label>code</label>
|
||||
<textarea readonly style="min-height:110px;" onclick="this.select()">{{ code }}</textarea>
|
||||
<label>code_verifier</label>
|
||||
<textarea readonly style="min-height:110px;" onclick="this.select()">{{ code_verifier }}</textarea>
|
||||
{% if device_id %}
|
||||
<label>device_id</label>
|
||||
<textarea readonly style="min-height:80px;" onclick="this.select()">{{ device_id }}</textarea>
|
||||
{% endif %}
|
||||
{% if state and expected_state and state != expected_state %}
|
||||
<p class="pill bad">state не совпал, такой code лучше не использовать.</p>
|
||||
{% endif %}
|
||||
@@ -19,34 +39,7 @@
|
||||
<p class="pill bad">{{ error }}</p>
|
||||
<p>{{ error_description }}</p>
|
||||
{% else %}
|
||||
<p class="muted">VK вернул callback без code.</p>
|
||||
<p class="muted">VK вернул callback без данных.</p>
|
||||
{% endif %}
|
||||
<div id="fragment-token" style="display:none;margin-top:24px;">
|
||||
<h2>Access token сообщества</h2>
|
||||
<p class="muted">VK передал токен в fragment URL. Скопируйте значение ниже.</p>
|
||||
<textarea id="fragment-token-value" readonly style="min-height:140px;" onclick="this.select()"></textarea>
|
||||
</div>
|
||||
<div id="fragment-data" style="display:none;margin-top:16px;">
|
||||
<h2>Параметры fragment</h2>
|
||||
<textarea id="fragment-data-value" readonly style="min-height:120px;" onclick="this.select()"></textarea>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
const hash = window.location.hash.replace(/^#/, "");
|
||||
if (hash) {
|
||||
const params = new URLSearchParams(hash);
|
||||
const pairs = Array.from(params.entries());
|
||||
const tokenPair = pairs.find(([key]) => key.startsWith("access_token"));
|
||||
if (tokenPair) {
|
||||
document.getElementById("fragment-token").style.display = "block";
|
||||
document.getElementById("fragment-token-value").value = tokenPair[1];
|
||||
}
|
||||
if (pairs.length) {
|
||||
document.getElementById("fragment-data").style.display = "block";
|
||||
document.getElementById("fragment-data-value").value = pairs
|
||||
.map(([key, value]) => `${key}=${value}`)
|
||||
.join("\n");
|
||||
}
|
||||
}
|
||||
</script>
|
||||
{% endblock %}
|
||||
|
||||
@@ -19,15 +19,18 @@
|
||||
|
||||
<details class="panel" open>
|
||||
<summary><strong>Категории публикаций</strong></summary>
|
||||
<p class="muted">Активные категории попадают в редакторскую и передаются AI-райтеру как ID, название и тэг. ID стабилен: модель возвращает его в JSON, а тэг подставляется из БД.</p>
|
||||
<p class="muted">Название описывает категорию для AI, тэг используется в соцсетях. Название и URL сайта хранятся отдельно и не меняют рабочие хэштеги.</p>
|
||||
<form class="category-create" method="post" action="/categories/create">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<label><span>Название</span><input name="name" placeholder="например, разгрузка" required></label>
|
||||
<label><span>Тэг</span><input name="tag" placeholder="если пусто, соберётся из названия"></label>
|
||||
<label><span>На сайте</span><input name="site_name" placeholder="Снаряжение" required></label>
|
||||
<label><span>URL сайта</span><input name="site_slug" placeholder="snaryazhenie" pattern="[a-z0-9-]+" required></label>
|
||||
<label class="checkline"><input type="checkbox" name="site_enabled" checked> Публиковать</label>
|
||||
<button class="btn primary" type="submit">Добавить</button>
|
||||
</form>
|
||||
<table class="compact-table category-table">
|
||||
<tr><th>Название</th><th>Тэг</th><th>ID</th><th>Статус</th><th></th></tr>
|
||||
<tr><th>Название AI</th><th>Тэг</th><th>Название сайта</th><th>URL сайта</th><th>ID</th><th>Статус</th><th></th></tr>
|
||||
{% for c in categories %}
|
||||
<tr>
|
||||
<td>
|
||||
@@ -37,8 +40,10 @@
|
||||
</form>
|
||||
</td>
|
||||
<td><input form="category-update-{{ c.id }}" name="tag" value="{{ c.tag }}"></td>
|
||||
<td><input form="category-update-{{ c.id }}" name="site_name" value="{{ c.site_name }}" required></td>
|
||||
<td><input form="category-update-{{ c.id }}" name="site_slug" value="{{ c.site_slug }}" pattern="[a-z0-9-]+" required></td>
|
||||
<td><input value="{{ c.sort_order }}" type="number" readonly title="Стабильный ID категории для AI-райтера"></td>
|
||||
<td>{% if c.is_active %}<span class="pill ok">активна</span>{% else %}<span class="pill">выключена</span>{% endif %}</td>
|
||||
<td>{% if c.is_active %}<span class="pill ok">активна</span>{% else %}<span class="pill">выключена</span>{% endif %}<label class="checkline"><input form="category-update-{{ c.id }}" type="checkbox" name="site_enabled" {% if c.site_enabled %}checked{% endif %}> сайт</label></td>
|
||||
<td class="icon-actions">
|
||||
<button class="icon-btn" form="category-update-{{ c.id }}" type="submit" title="Сохранить">✓</button>
|
||||
<form method="post" action="/categories/{{ c.id }}/toggle">
|
||||
@@ -55,6 +60,75 @@
|
||||
</table>
|
||||
</details>
|
||||
|
||||
<details class="panel" open>
|
||||
<summary><strong>Расписание TG-постера</strong></summary>
|
||||
<p class="muted">Время публикации задаётся по Екатеринбургу. Для запуска нужны оба флага: worker <span class="mono">tg-poster</span> и настройка <span class="mono">tg_poster_enabled</span>.</p>
|
||||
<form class="category-create" method="post" action="/tg-poster-schedule/create">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<label><span>Время Екб</span><input type="time" name="time" value="10:00" required></label>
|
||||
<label><span>Постов</span><input type="number" name="count" value="1" min="1" max="20" required></label>
|
||||
<label class="checkline"><input type="checkbox" name="enabled" checked> Активно</label>
|
||||
<button class="btn primary" type="submit">Добавить</button>
|
||||
</form>
|
||||
<table class="compact-table category-table">
|
||||
<tr><th>Время Екб</th><th>Постов</th><th>Статус</th><th></th></tr>
|
||||
{% for row in tg_schedule %}
|
||||
<tr>
|
||||
<td>
|
||||
<form id="tg-schedule-update-{{ row.id }}" method="post" action="/tg-poster-schedule/{{ row.id }}/update">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<input type="time" name="time" value="{{ row.time }}" required>
|
||||
</form>
|
||||
</td>
|
||||
<td><input form="tg-schedule-update-{{ row.id }}" type="number" name="count" value="{{ row.count }}" min="1" max="20" required></td>
|
||||
<td><label class="checkline"><input form="tg-schedule-update-{{ row.id }}" type="checkbox" name="enabled" {% if row.enabled %}checked{% endif %}> Активно</label></td>
|
||||
<td class="icon-actions">
|
||||
<button class="icon-btn" form="tg-schedule-update-{{ row.id }}" type="submit" title="Сохранить">✓</button>
|
||||
<form method="post" action="/tg-poster-schedule/{{ row.id }}/delete">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<button class="icon-btn danger" type="submit" title="Удалить">×</button>
|
||||
</form>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
</details>
|
||||
|
||||
<details class="panel" open>
|
||||
<summary><strong>Расписание VK-постера</strong></summary>
|
||||
<p class="muted">Время публикации задаётся по Екатеринбургу. Для запуска нужны оба флага: worker <span class="mono">vk-poster</span> и настройка <span class="mono">vk_poster_enabled</span>.</p>
|
||||
<p><a class="btn" href="/vk-oauth/start">Получить VK user token</a></p>
|
||||
<form class="category-create" method="post" action="/vk-poster-schedule/create">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<label><span>Время Екб</span><input type="time" name="time" value="10:05" required></label>
|
||||
<label><span>Постов</span><input type="number" name="count" value="1" min="1" max="20" required></label>
|
||||
<label class="checkline"><input type="checkbox" name="enabled" checked> Активно</label>
|
||||
<button class="btn primary" type="submit">Добавить</button>
|
||||
</form>
|
||||
<table class="compact-table category-table">
|
||||
<tr><th>Время Екб</th><th>Постов</th><th>Статус</th><th></th></tr>
|
||||
{% for row in vk_schedule %}
|
||||
<tr>
|
||||
<td>
|
||||
<form id="vk-schedule-update-{{ row.id }}" method="post" action="/vk-poster-schedule/{{ row.id }}/update">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<input type="time" name="time" value="{{ row.time }}" required>
|
||||
</form>
|
||||
</td>
|
||||
<td><input form="vk-schedule-update-{{ row.id }}" type="number" name="count" value="{{ row.count }}" min="1" max="20" required></td>
|
||||
<td><label class="checkline"><input form="vk-schedule-update-{{ row.id }}" type="checkbox" name="enabled" {% if row.enabled %}checked{% endif %}> Активно</label></td>
|
||||
<td class="icon-actions">
|
||||
<button class="icon-btn" form="vk-schedule-update-{{ row.id }}" type="submit" title="Сохранить">✓</button>
|
||||
<form method="post" action="/vk-poster-schedule/{{ row.id }}/delete">
|
||||
<input type="hidden" name="csrf_token" value="{{ user.csrf_token }}">
|
||||
<button class="icon-btn danger" type="submit" title="Удалить">×</button>
|
||||
</form>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
</details>
|
||||
|
||||
<h2>Настройки</h2>
|
||||
{% for category, rows in settings|groupby("category") %}
|
||||
<details class="panel" {% if category == "AI Qualifier" %}open{% endif %}>
|
||||
@@ -126,6 +200,8 @@
|
||||
{% endif %}
|
||||
<textarea name="value" rows="12" style="width:720px; max-width:100%;">{{ s.value_json }}</textarea>
|
||||
</div>
|
||||
{% elif s.value_type == "json" %}
|
||||
<textarea name="value" rows="5" style="width:520px; max-width:100%;" spellcheck="false">{{ s.value_json | tojson }}</textarea>
|
||||
{% else %}
|
||||
<input name="value" value="{{ s.value_json }}" style="width:180px;">
|
||||
{% endif %}
|
||||
|
||||
@@ -11,6 +11,34 @@ def normalize_hash_tag(value: str, fallback: str = "source") -> str:
|
||||
return tag or fallback
|
||||
|
||||
|
||||
def strip_trailing_hashtag_line(text: str) -> str:
|
||||
lines = str(text or "").rstrip().splitlines()
|
||||
while lines and not lines[-1].strip():
|
||||
lines.pop()
|
||||
if not lines:
|
||||
return ""
|
||||
parts = lines[-1].split()
|
||||
if parts and all(part.startswith("#") and normalize_hash_tag(part, "") for part in parts):
|
||||
lines.pop()
|
||||
return "\n".join(lines).rstrip()
|
||||
|
||||
|
||||
def publication_hashtags(category_tag: str, source_tag: str) -> str:
|
||||
tags = [
|
||||
normalize_hash_tag(category_tag, "category"),
|
||||
normalize_hash_tag(source_tag, "source"),
|
||||
]
|
||||
return " ".join(f"#{tag}" for tag in tags if tag)
|
||||
|
||||
|
||||
def build_publication_text(text: str, category_tag: str, source_tag: str) -> str:
|
||||
base = strip_trailing_hashtag_line(text)
|
||||
hashtags = publication_hashtags(category_tag, source_tag)
|
||||
if not hashtags:
|
||||
return base
|
||||
return f"{base}\n\n{hashtags}" if base else hashtags
|
||||
|
||||
|
||||
def parse_categories(value: object) -> list[str]:
|
||||
if isinstance(value, list):
|
||||
raw_items = [str(item) for item in value]
|
||||
|
||||
@@ -155,6 +155,20 @@ class VKAPIClient:
|
||||
raise VKAPIError(None, f"photos.saveWallPhoto returned invalid response: {json.dumps(saved)[:300]}")
|
||||
return saved[0]
|
||||
|
||||
async def upload_wall_photo_url(self, group_id: int, url: str) -> dict:
|
||||
if not self.session:
|
||||
raise RuntimeError("VKAPIClient is not initialized")
|
||||
async with self.session.get(url) as resp:
|
||||
resp.raise_for_status()
|
||||
content_type = resp.headers.get("content-type") or "image/jpeg"
|
||||
data = await resp.read()
|
||||
suffix = ".jpg"
|
||||
if "png" in content_type:
|
||||
suffix = ".png"
|
||||
elif "webp" in content_type:
|
||||
suffix = ".webp"
|
||||
return await self.upload_wall_photo_bytes(group_id, data, filename=f"photo{suffix}")
|
||||
|
||||
async def create_wall_post(
|
||||
self,
|
||||
owner_id: int,
|
||||
|
||||
@@ -151,6 +151,22 @@ class AIQualifierWorker:
|
||||
FROM raw_posts
|
||||
WHERE status='storage_ready'
|
||||
AND COALESCE(qualification_status, 'pending') IN ('pending', 'failed')
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media m
|
||||
WHERE m.raw_post_id=raw_posts.id
|
||||
AND COALESCE(m.editor_hidden, FALSE)=FALSE
|
||||
AND m.media_type IN ('photo', 'video')
|
||||
AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') <> ''
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media m
|
||||
WHERE m.raw_post_id=raw_posts.id
|
||||
AND COALESCE(m.editor_hidden, FALSE)=FALSE
|
||||
AND m.media_type IN ('photo', 'video')
|
||||
AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = ''
|
||||
)
|
||||
"""
|
||||
)
|
||||
if int(count or 0) < batch_size:
|
||||
@@ -163,6 +179,22 @@ class AIQualifierWorker:
|
||||
FROM raw_posts
|
||||
WHERE status='storage_ready'
|
||||
AND COALESCE(qualification_status, 'pending') IN ('pending', 'failed')
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media m
|
||||
WHERE m.raw_post_id=raw_posts.id
|
||||
AND COALESCE(m.editor_hidden, FALSE)=FALSE
|
||||
AND m.media_type IN ('photo', 'video')
|
||||
AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') <> ''
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media m
|
||||
WHERE m.raw_post_id=raw_posts.id
|
||||
AND COALESCE(m.editor_hidden, FALSE)=FALSE
|
||||
AND m.media_type IN ('photo', 'video')
|
||||
AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = ''
|
||||
)
|
||||
ORDER BY created_at ASC, id ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT $1
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
@@ -15,7 +16,14 @@ from ..constants import WORKER_AI_WRITER
|
||||
from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool
|
||||
from ..heartbeat import HeartbeatReporter
|
||||
from ..jobs import is_worker_enabled
|
||||
from ..text_utils import normalize_hash_tag, parse_categories
|
||||
from ..text_utils import build_publication_text, normalize_hash_tag, parse_categories
|
||||
|
||||
|
||||
NON_TARGET_CATEGORY_ID = 18
|
||||
NON_TARGET_CATEGORY_TAGS = {"нцк", "nck"}
|
||||
EMOJI_LEADING_MARKER_RE = re.compile(
|
||||
r"^[ \t]*(?:🧵|🎯|🛡|🧤|📦|💵|⚙\ufe0f?|🎨|🎒|🔧|👕|💪)\s+"
|
||||
)
|
||||
|
||||
|
||||
def now_utc() -> datetime:
|
||||
@@ -30,6 +38,14 @@ def normalize_prompt(prompt: str) -> str:
|
||||
return (prompt or "").replace("\\r\\n", "\n").replace("\\n", "\n").strip()
|
||||
|
||||
|
||||
def strip_leading_emoji_markers(text: str) -> str:
|
||||
lines = str(text or "").splitlines()
|
||||
cleaned: list[str] = []
|
||||
for line in lines:
|
||||
cleaned.append(EMOJI_LEADING_MARKER_RE.sub("", line))
|
||||
return "\n".join(cleaned).strip()
|
||||
|
||||
|
||||
DEFAULT_CONTRACT_PROMPT = """
|
||||
OUTPUT SCHEMA (return array matching input order):
|
||||
{"rewrites":[{"id":123,"category_id":2,"text":"готовый текст без хэштегов","notes":"короткая заметка для редактора или null"}]}
|
||||
@@ -38,7 +54,9 @@ Rules:
|
||||
- Input is a JSON object with key "posts" containing accepted posts.
|
||||
- Input has key "categories" with objects: id, name, tag.
|
||||
- Each post includes producer_name and producer_tag. producer_name can be a manufacturer, shop, or publishing source.
|
||||
- The "text" field for every rewrite must explicitly mention producer_name at least once. Do not satisfy this only through producer_tag, hashtags, notes, or metadata.
|
||||
- Pick exactly one category from the categories list in input and return its numeric id as category_id.
|
||||
- If the post is non-target content, choose category_id 18 ("Не целевой контент", tag "НЦК"); the app will reject it automatically.
|
||||
- Do not add links or hashtags to rewritten text.
|
||||
- Return JSON only. No markdown. No text outside JSON.
|
||||
- Never return an empty object. Never omit the "rewrites" key.
|
||||
@@ -127,7 +145,7 @@ def validate_rewrites(
|
||||
post_id = int(item.get("id"))
|
||||
if post_id not in expected_ids:
|
||||
raise ValueError(f"AI returned unexpected id={post_id}")
|
||||
text = str(item.get("text") or "").strip()
|
||||
text = strip_leading_emoji_markers(str(item.get("text") or "").strip())
|
||||
if len(text) < 40:
|
||||
raise ValueError(f"AI returned too short rewrite for id={post_id}")
|
||||
by_id = {int(c["id"]): c for c in categories if c.get("id") is not None}
|
||||
@@ -144,15 +162,18 @@ def validate_rewrites(
|
||||
raise ValueError(f"AI returned unknown category={category_name!r} for id={post_id}")
|
||||
notes = str(item.get("notes") or "").strip()[:1000]
|
||||
seen.add(post_id)
|
||||
category_tag = normalize_hash_tag(category_row.get("tag") or category_row.get("name") or "", "category")
|
||||
reject_by_category = int(category_row["id"]) == NON_TARGET_CATEGORY_ID or category_tag.lower() in NON_TARGET_CATEGORY_TAGS
|
||||
out.append(
|
||||
{
|
||||
"id": post_id,
|
||||
"category_id": int(category_row["id"]),
|
||||
"category": str(category_row["name"]),
|
||||
"category_tag": normalize_hash_tag(category_row.get("tag") or category_row.get("name") or "", "category"),
|
||||
"category_tag": category_tag,
|
||||
"source_tag": normalize_hash_tag(source_tags_by_id.get(post_id) or "", "source"),
|
||||
"text": text,
|
||||
"notes": notes,
|
||||
"reject_by_category": reject_by_category,
|
||||
}
|
||||
)
|
||||
missing = expected_ids - seen
|
||||
@@ -209,6 +230,22 @@ class AIWriterWorker:
|
||||
WHERE status='storage_ready'
|
||||
AND qualification_status='accepted'
|
||||
AND COALESCE(rewrite_status, 'pending') IN ('pending', 'failed')
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media m
|
||||
WHERE m.raw_post_id=raw_posts.id
|
||||
AND COALESCE(m.editor_hidden, FALSE)=FALSE
|
||||
AND m.media_type IN ('photo', 'video')
|
||||
AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') <> ''
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media m
|
||||
WHERE m.raw_post_id=raw_posts.id
|
||||
AND COALESCE(m.editor_hidden, FALSE)=FALSE
|
||||
AND m.media_type IN ('photo', 'video')
|
||||
AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = ''
|
||||
)
|
||||
"""
|
||||
)
|
||||
if int(count or 0) < batch_size:
|
||||
@@ -222,6 +259,22 @@ class AIWriterWorker:
|
||||
WHERE status='storage_ready'
|
||||
AND qualification_status='accepted'
|
||||
AND COALESCE(rewrite_status, 'pending') IN ('pending', 'failed')
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media m
|
||||
WHERE m.raw_post_id=raw_posts.id
|
||||
AND COALESCE(m.editor_hidden, FALSE)=FALSE
|
||||
AND m.media_type IN ('photo', 'video')
|
||||
AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') <> ''
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media m
|
||||
WHERE m.raw_post_id=raw_posts.id
|
||||
AND COALESCE(m.editor_hidden, FALSE)=FALSE
|
||||
AND m.media_type IN ('photo', 'video')
|
||||
AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = ''
|
||||
)
|
||||
ORDER BY qualified_at ASC NULLS LAST, id ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT $1
|
||||
@@ -316,6 +369,9 @@ class AIWriterWorker:
|
||||
|
||||
async def apply_rewrites(self, batch_id: int, rewrites: list[dict], model: str, prompt: str) -> None:
|
||||
for item in rewrites:
|
||||
final_text = build_publication_text(item["text"], item["category_tag"], item["source_tag"])
|
||||
editorial_status = "rejected" if item.get("reject_by_category") else "review"
|
||||
editor_notes = "Отклонено AI: Не целевой контент" if item.get("reject_by_category") else None
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE raw_posts
|
||||
@@ -329,12 +385,19 @@ class AIWriterWorker:
|
||||
rewrite_category_tag=$8,
|
||||
rewrite_source_tag=$9,
|
||||
rewrite_category_id=$10,
|
||||
editorial_status='review',
|
||||
final_text=$2,
|
||||
editorial_status=$11,
|
||||
final_text=$12,
|
||||
final_category=$7,
|
||||
final_category_tag=$8,
|
||||
final_source_tag=$9,
|
||||
final_category_id=$10,
|
||||
editor_notes=$13,
|
||||
qualification_status=CASE WHEN $11='rejected' THEN 'rejected' ELSE qualification_status END,
|
||||
qualification_decision=CASE WHEN $11='rejected' THEN 'rejected' ELSE qualification_decision END,
|
||||
qualification_model_decision=CASE WHEN $11='rejected' THEN 'rejected' ELSE qualification_model_decision END,
|
||||
qualification_reason=CASE WHEN $11='rejected' THEN 'AI writer selected non-target category' ELSE qualification_reason END,
|
||||
qualification_reject_tag=CASE WHEN $11='rejected' THEN 'non_target_content' ELSE qualification_reject_tag END,
|
||||
reviewed_at=CASE WHEN $11='rejected' THEN NOW() ELSE reviewed_at END,
|
||||
rewritten_at=NOW(),
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
@@ -349,6 +412,9 @@ class AIWriterWorker:
|
||||
item["category_tag"],
|
||||
item["source_tag"],
|
||||
item["category_id"],
|
||||
editorial_status,
|
||||
final_text,
|
||||
editor_notes,
|
||||
)
|
||||
|
||||
async def call_ai(
|
||||
|
||||
@@ -0,0 +1,457 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import date, datetime, time, timedelta, timezone
|
||||
from typing import Any
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from aiogram import Bot
|
||||
from aiogram.exceptions import TelegramRetryAfter
|
||||
from loguru import logger
|
||||
|
||||
from ..config import settings
|
||||
from ..constants import WORKER_DAILY_REPORT
|
||||
from ..db import fetch_bool_setting, fetch_int_setting, fetch_setting, get_pool
|
||||
from ..heartbeat import HeartbeatReporter
|
||||
from ..jobs import is_worker_enabled
|
||||
|
||||
LOCAL_TZ = ZoneInfo("Asia/Yekaterinburg")
|
||||
MESSAGE_LIMIT = 4096
|
||||
|
||||
|
||||
def valid_time(value: str) -> bool:
|
||||
return bool(re.fullmatch(r"(?:[01]\d|2[0-3]):[0-5]\d", str(value or "").strip()))
|
||||
|
||||
|
||||
def parse_recipients(value: Any) -> list[int]:
|
||||
raw_items: list[Any]
|
||||
if isinstance(value, list):
|
||||
raw_items = value
|
||||
elif isinstance(value, str):
|
||||
raw = value.strip()
|
||||
if not raw:
|
||||
return []
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
raw_items = parsed if isinstance(parsed, list) else [parsed]
|
||||
except json.JSONDecodeError:
|
||||
raw_items = re.split(r"[\s,;]+", raw)
|
||||
else:
|
||||
raw_items = [value]
|
||||
out: list[int] = []
|
||||
for item in raw_items:
|
||||
try:
|
||||
recipient_id = int(str(item).strip())
|
||||
except Exception:
|
||||
continue
|
||||
if recipient_id not in out:
|
||||
out.append(recipient_id)
|
||||
return out
|
||||
|
||||
|
||||
def day_bounds(report_date: date) -> tuple[datetime, datetime]:
|
||||
start_local = datetime.combine(report_date, time.min, tzinfo=LOCAL_TZ)
|
||||
end_local = start_local + timedelta(days=1)
|
||||
return start_local.astimezone(timezone.utc), end_local.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def fmt_int(value: Any) -> str:
|
||||
try:
|
||||
return f"{int(value):,}".replace(",", " ")
|
||||
except Exception:
|
||||
return "0"
|
||||
|
||||
|
||||
def split_chunks(text: str, limit: int = MESSAGE_LIMIT) -> list[str]:
|
||||
text = str(text or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
chunks: list[str] = []
|
||||
while len(text) > limit:
|
||||
split_at = text.rfind("\n", 0, limit)
|
||||
if split_at < limit // 2:
|
||||
split_at = limit
|
||||
chunks.append(text[:split_at].strip())
|
||||
text = text[split_at:].strip()
|
||||
if text:
|
||||
chunks.append(text)
|
||||
return chunks
|
||||
|
||||
|
||||
class DailyReportWorker:
|
||||
def __init__(self) -> None:
|
||||
self.pool = None
|
||||
self.bot: Bot | None = None
|
||||
self.bot_token = ""
|
||||
self.recipients: list[int] = []
|
||||
self.report_time = "00:04"
|
||||
self.interval_sec = 60
|
||||
self.heartbeat = HeartbeatReporter(WORKER_DAILY_REPORT, 30)
|
||||
|
||||
async def init(self) -> None:
|
||||
self.pool = await get_pool()
|
||||
await self.reload_settings()
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.bot:
|
||||
await self.bot.session.close()
|
||||
|
||||
async def reload_settings(self) -> None:
|
||||
token = (
|
||||
str(await fetch_setting("daily_report_bot_token", "") or "").strip()
|
||||
or str(await fetch_setting("tg_poster_bot_token", "") or "").strip()
|
||||
or settings.tg_bot_token
|
||||
)
|
||||
if not token:
|
||||
raise RuntimeError("daily_report_bot_token, tg_poster_bot_token and TG_BOT_TOKEN are empty")
|
||||
if token != self.bot_token:
|
||||
if self.bot:
|
||||
await self.bot.session.close()
|
||||
self.bot = Bot(token=token)
|
||||
self.bot_token = token
|
||||
self.recipients = parse_recipients(await fetch_setting("daily_report_recipient_ids", [442509142]))
|
||||
if not self.recipients:
|
||||
raise RuntimeError("daily_report_recipient_ids is empty")
|
||||
report_time = str(await fetch_setting("daily_report_time", "00:04") or "00:04").strip()
|
||||
self.report_time = report_time if valid_time(report_time) else "00:04"
|
||||
self.interval_sec = max(10, await fetch_int_setting("daily_report_interval_sec", 60))
|
||||
logger.info("Daily report config: time={} recipients={}", self.report_time, self.recipients)
|
||||
|
||||
async def save_last_sent_date(self, report_date: date) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE app_settings
|
||||
SET value_json=$2::jsonb,
|
||||
updated_at=NOW()
|
||||
WHERE key=$1
|
||||
""",
|
||||
"daily_report_last_sent_date",
|
||||
json.dumps(report_date.isoformat()),
|
||||
)
|
||||
|
||||
async def countval(self, sql: str, *args: Any) -> int:
|
||||
return int(await self.pool.fetchval(sql, *args) or 0)
|
||||
|
||||
async def collect(self, report_date: date) -> dict[str, Any]:
|
||||
start_utc, end_utc = day_bounds(report_date)
|
||||
now_utc = datetime.now(timezone.utc)
|
||||
async with self.pool.acquire() as conn:
|
||||
raw_collected = await conn.fetchval(
|
||||
"SELECT COUNT(*) FROM raw_posts WHERE created_at >= $1 AND created_at < $2",
|
||||
start_utc,
|
||||
end_utc,
|
||||
)
|
||||
storage_ready = await conn.fetchval(
|
||||
"SELECT COUNT(*) FROM raw_posts WHERE status='storage_ready' AND updated_at >= $1 AND updated_at < $2",
|
||||
start_utc,
|
||||
end_utc,
|
||||
)
|
||||
media_failed = await conn.fetchval(
|
||||
"SELECT COUNT(*) FROM raw_post_media WHERE status IN ('failed','link_only') AND updated_at >= $1 AND updated_at < $2",
|
||||
start_utc,
|
||||
end_utc,
|
||||
)
|
||||
qualification_rows = await conn.fetch(
|
||||
"""
|
||||
SELECT COALESCE(qualification_status, 'pending') AS status, COUNT(*) AS count
|
||||
FROM raw_posts
|
||||
WHERE qualified_at >= $1 AND qualified_at < $2
|
||||
GROUP BY COALESCE(qualification_status, 'pending')
|
||||
""",
|
||||
start_utc,
|
||||
end_utc,
|
||||
)
|
||||
writer_ready = await conn.fetchval(
|
||||
"SELECT COUNT(*) FROM raw_posts WHERE rewrite_status='ready' AND rewritten_at >= $1 AND rewritten_at < $2",
|
||||
start_utc,
|
||||
end_utc,
|
||||
)
|
||||
review_now = await conn.fetchval(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM raw_posts
|
||||
WHERE rewrite_status='ready'
|
||||
AND COALESCE(editorial_status, 'review')='review'
|
||||
"""
|
||||
)
|
||||
accepted_today = await conn.fetchval(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM raw_posts
|
||||
WHERE reviewed_at >= $1 AND reviewed_at < $2
|
||||
AND editorial_status='accepted'
|
||||
""",
|
||||
start_utc,
|
||||
end_utc,
|
||||
)
|
||||
rejected_today = await conn.fetchval(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM raw_posts
|
||||
WHERE reviewed_at >= $1 AND reviewed_at < $2
|
||||
AND editorial_status='rejected'
|
||||
""",
|
||||
start_utc,
|
||||
end_utc,
|
||||
)
|
||||
publication_rows = await conn.fetch(
|
||||
"""
|
||||
SELECT platform, status, COUNT(*) AS count
|
||||
FROM post_publications
|
||||
WHERE updated_at >= $1 AND updated_at < $2
|
||||
GROUP BY platform, status
|
||||
ORDER BY platform, status
|
||||
""",
|
||||
start_utc,
|
||||
end_utc,
|
||||
)
|
||||
tg_queue = await conn.fetchval(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM raw_posts rp
|
||||
WHERE rp.status='storage_ready'
|
||||
AND rp.rewrite_status='ready'
|
||||
AND rp.editorial_status='accepted'
|
||||
AND COALESCE(rp.publication_status, 'pending')='pending'
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM raw_post_media m
|
||||
WHERE m.raw_post_id=rp.id
|
||||
AND COALESCE(m.editor_hidden,FALSE)=FALSE
|
||||
AND m.media_type IN ('photo','video')
|
||||
AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') <> ''
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM raw_post_media m
|
||||
WHERE m.raw_post_id=rp.id
|
||||
AND COALESCE(m.editor_hidden,FALSE)=FALSE
|
||||
AND m.media_type IN ('photo','video')
|
||||
AND COALESCE(m.tg_file_id, m.storage_attachment_id, '') = ''
|
||||
)
|
||||
"""
|
||||
)
|
||||
vk_queue = await conn.fetchval(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM raw_posts rp
|
||||
LEFT JOIN post_publications pp ON pp.raw_post_id=rp.id AND pp.platform='vk'
|
||||
WHERE rp.status='storage_ready'
|
||||
AND rp.rewrite_status='ready'
|
||||
AND COALESCE(rp.editorial_status, 'review') IN ('accepted', 'published', 'publish_failed')
|
||||
AND COALESCE(pp.status, 'pending') IN ('pending', 'publish_failed')
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM raw_post_media rpm
|
||||
WHERE rpm.raw_post_id=rp.id
|
||||
AND COALESCE(rpm.editor_hidden, FALSE)=FALSE
|
||||
AND COALESCE(rpm.editor_added, FALSE)=FALSE
|
||||
AND (
|
||||
(rpm.media_type='photo' AND rpm.original_attachment_id LIKE 'photo%')
|
||||
OR (rpm.media_type='video' AND rpm.original_attachment_id LIKE 'video%')
|
||||
)
|
||||
)
|
||||
"""
|
||||
)
|
||||
publish_failed_now = await conn.fetchval(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM raw_posts
|
||||
WHERE rewrite_status='ready'
|
||||
AND COALESCE(editorial_status, '')='publish_failed'
|
||||
"""
|
||||
)
|
||||
workers = await conn.fetch(
|
||||
"""
|
||||
SELECT wc.name, wc.enabled, wh.heartbeat_at, wh.status
|
||||
FROM worker_controls wc
|
||||
LEFT JOIN worker_heartbeats wh ON wh.name=wc.name
|
||||
ORDER BY wc.name
|
||||
"""
|
||||
)
|
||||
top_errors = await conn.fetch(
|
||||
"""
|
||||
SELECT platform, error, COUNT(*) AS count
|
||||
FROM post_publications
|
||||
WHERE status='publish_failed'
|
||||
AND updated_at >= $1 AND updated_at < $2
|
||||
AND COALESCE(error, '') <> ''
|
||||
GROUP BY platform, error
|
||||
ORDER BY COUNT(*) DESC
|
||||
LIMIT 5
|
||||
""",
|
||||
start_utc,
|
||||
end_utc,
|
||||
)
|
||||
return {
|
||||
"date": report_date,
|
||||
"raw_collected": raw_collected,
|
||||
"storage_ready": storage_ready,
|
||||
"media_failed": media_failed,
|
||||
"qualification": {row["status"]: int(row["count"]) for row in qualification_rows},
|
||||
"writer_ready": writer_ready,
|
||||
"review_now": review_now,
|
||||
"accepted_today": accepted_today,
|
||||
"rejected_today": rejected_today,
|
||||
"publications": {(row["platform"], row["status"]): int(row["count"]) for row in publication_rows},
|
||||
"tg_queue": tg_queue,
|
||||
"vk_queue": vk_queue,
|
||||
"publish_failed_now": publish_failed_now,
|
||||
"workers": [dict(row) for row in workers],
|
||||
"top_errors": [dict(row) for row in top_errors],
|
||||
"now_utc": now_utc,
|
||||
}
|
||||
|
||||
def render(self, data: dict[str, Any]) -> str:
|
||||
q = data["qualification"]
|
||||
pubs = data["publications"]
|
||||
workers_lines: list[str] = []
|
||||
for row in data["workers"]:
|
||||
heartbeat_at = row.get("heartbeat_at")
|
||||
is_stale = True
|
||||
if heartbeat_at:
|
||||
is_stale = (data["now_utc"] - heartbeat_at).total_seconds() > 600
|
||||
marker = "OK" if row.get("enabled") and heartbeat_at and not is_stale else ("OFF" if not row.get("enabled") else "STALE")
|
||||
workers_lines.append(f"{marker} {row['name']}: {row.get('status') or '-'}")
|
||||
errors_lines = [
|
||||
f"- {row['platform']}: {row['count']} x {str(row['error'])[:120]}"
|
||||
for row in data["top_errors"]
|
||||
]
|
||||
if not errors_lines:
|
||||
errors_lines = ["- нет"]
|
||||
report_date = data["date"].strftime("%d.%m.%Y")
|
||||
return "\n".join(
|
||||
[
|
||||
f"Ежедневный отчет N8 Parser за {report_date}",
|
||||
"",
|
||||
"Сбор:",
|
||||
f"raw собрано: {fmt_int(data['raw_collected'])}",
|
||||
f"storage ready за день: {fmt_int(data['storage_ready'])}",
|
||||
f"media failed/link_only: {fmt_int(data['media_failed'])}",
|
||||
"",
|
||||
"AI:",
|
||||
f"принято категоризатором: {fmt_int(q.get('accepted', 0))}",
|
||||
f"отклонено категоризатором: {fmt_int(q.get('rejected', 0))}",
|
||||
f"ошибки категоризатора: {fmt_int(q.get('failed', 0))}",
|
||||
f"рерайтов готово: {fmt_int(data['writer_ready'])}",
|
||||
"",
|
||||
"Редакторская:",
|
||||
f"принято редактором за день: {fmt_int(data['accepted_today'])}",
|
||||
f"отклонено редактором за день: {fmt_int(data['rejected_today'])}",
|
||||
f"на проверке сейчас: {fmt_int(data['review_now'])}",
|
||||
f"ошибка публикации сейчас: {fmt_int(data['publish_failed_now'])}",
|
||||
"",
|
||||
"Публикации за день:",
|
||||
f"TG опубликовано: {fmt_int(pubs.get(('tg', 'published'), 0))}",
|
||||
f"VK опубликовано: {fmt_int(pubs.get(('vk', 'published'), 0))}",
|
||||
f"TG ошибки: {fmt_int(pubs.get(('tg', 'publish_failed'), 0))}",
|
||||
f"VK ошибки: {fmt_int(pubs.get(('vk', 'publish_failed'), 0))}",
|
||||
"",
|
||||
"Очередь сейчас:",
|
||||
f"TG кандидатов: {fmt_int(data['tg_queue'])}",
|
||||
f"VK кандидатов: {fmt_int(data['vk_queue'])}",
|
||||
"",
|
||||
"Ошибки дня:",
|
||||
*errors_lines,
|
||||
"",
|
||||
"Воркеры:",
|
||||
*workers_lines,
|
||||
]
|
||||
)
|
||||
|
||||
async def send_text(self, text: str) -> None:
|
||||
if not self.bot:
|
||||
raise RuntimeError("bot is not initialized")
|
||||
for recipient_id in self.recipients:
|
||||
for chunk in split_chunks(text):
|
||||
while True:
|
||||
try:
|
||||
await self.bot.send_message(recipient_id, chunk, disable_web_page_preview=True)
|
||||
break
|
||||
except TelegramRetryAfter as exc:
|
||||
await asyncio.sleep(float(exc.retry_after) + 1)
|
||||
|
||||
async def send_report(self, report_date: date, save_last_sent: bool = False) -> str:
|
||||
await self.reload_settings()
|
||||
data = await self.collect(report_date)
|
||||
text = self.render(data)
|
||||
await self.send_text(text)
|
||||
if save_last_sent:
|
||||
await self.save_last_sent_date(report_date)
|
||||
await self.heartbeat.beat(
|
||||
self.pool,
|
||||
status="sent",
|
||||
meta={"date": report_date.isoformat(), "recipients": self.recipients},
|
||||
force=True,
|
||||
)
|
||||
return text
|
||||
|
||||
async def due_report_date(self) -> date | None:
|
||||
now = datetime.now(LOCAL_TZ)
|
||||
hour, minute = [int(part) for part in self.report_time.split(":", 1)]
|
||||
slot_start = datetime.combine(now.date(), time(hour=hour, minute=minute), tzinfo=LOCAL_TZ)
|
||||
slot_end = slot_start + timedelta(minutes=10)
|
||||
if not (slot_start <= now < slot_end):
|
||||
return None
|
||||
report_date = now.date() - timedelta(days=1)
|
||||
last_sent = str(await fetch_setting("daily_report_last_sent_date", "") or "").strip()
|
||||
if last_sent == report_date.isoformat():
|
||||
return None
|
||||
return report_date
|
||||
|
||||
async def run_once(self) -> bool:
|
||||
await self.reload_settings()
|
||||
enabled = await is_worker_enabled(self.pool, WORKER_DAILY_REPORT)
|
||||
app_enabled = await fetch_bool_setting("daily_report_enabled", True)
|
||||
if not enabled or not app_enabled:
|
||||
await self.heartbeat.beat(self.pool, status="disabled", force=True)
|
||||
return False
|
||||
report_date = await self.due_report_date()
|
||||
if not report_date:
|
||||
await self.heartbeat.beat(self.pool, status="idle")
|
||||
return False
|
||||
await self.send_report(report_date, save_last_sent=True)
|
||||
return True
|
||||
|
||||
async def run_loop(self) -> None:
|
||||
await self.init()
|
||||
logger.info("{} started pid={}", WORKER_DAILY_REPORT, os.getpid())
|
||||
while True:
|
||||
try:
|
||||
await self.run_once()
|
||||
except Exception as exc:
|
||||
logger.exception("Daily report loop error: {}", exc)
|
||||
if self.pool:
|
||||
await self.heartbeat.beat(self.pool, status="error", meta={"error": str(exc)}, force=True)
|
||||
await asyncio.sleep(self.interval_sec)
|
||||
|
||||
|
||||
async def run_send_now(report_date: date | None = None) -> str:
|
||||
worker = DailyReportWorker()
|
||||
await worker.init()
|
||||
try:
|
||||
return await worker.send_report(report_date or datetime.now(LOCAL_TZ).date(), save_last_sent=False)
|
||||
finally:
|
||||
await worker.close()
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--send-now", action="store_true")
|
||||
parser.add_argument("--date", help="Report date YYYY-MM-DD, default today in EKB for --send-now")
|
||||
args = parser.parse_args()
|
||||
logger.remove()
|
||||
logger.add(sys.stdout, level=settings.log_level)
|
||||
if args.send_now:
|
||||
report_date = date.fromisoformat(args.date) if args.date else None
|
||||
text = await run_send_now(report_date)
|
||||
print(text)
|
||||
return
|
||||
worker = DailyReportWorker()
|
||||
await worker.run_loop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,713 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from datetime import datetime, time, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import aiohttp
|
||||
from loguru import logger
|
||||
|
||||
from ..config import settings
|
||||
from ..constants import (
|
||||
PUBLICATION_STATUS_FAILED,
|
||||
PUBLICATION_STATUS_PENDING,
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
WORKER_MAX_POSTER,
|
||||
)
|
||||
from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool
|
||||
from ..heartbeat import HeartbeatReporter
|
||||
from ..jobs import is_worker_enabled
|
||||
from ..text_utils import build_publication_text
|
||||
|
||||
LOCAL_TZ = ZoneInfo("Asia/Yekaterinburg")
|
||||
MAX_MESSAGE_LIMIT = 4000
|
||||
MAX_MEDIA_ITEMS = 10
|
||||
MAX_VIDEO_BYTES = 250 * 1024 * 1024
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
UPLOAD_ROOT = PROJECT_ROOT / "uploads"
|
||||
|
||||
|
||||
class MAXAPIError(RuntimeError):
|
||||
def __init__(self, message: str, status: int | None = None, code: str = "") -> None:
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
|
||||
|
||||
def valid_time(value: str) -> bool:
|
||||
return bool(re.fullmatch(r"(?:[01]\d|2[0-3]):[0-5]\d", str(value or "").strip()))
|
||||
|
||||
|
||||
def normalize_schedule(value: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(value, list):
|
||||
value = []
|
||||
rows: list[dict[str, Any]] = []
|
||||
for idx, item in enumerate(value):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
time_value = str(item.get("time") or "").strip()
|
||||
if not valid_time(time_value):
|
||||
continue
|
||||
count = max(0, min(20, int(item.get("count") or 0)))
|
||||
if count < 1:
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"id": str(item.get("id") or f"slot-{idx + 1}").strip()[:80],
|
||||
"time": time_value,
|
||||
"count": count,
|
||||
"enabled": bool(item.get("enabled", True)),
|
||||
}
|
||||
)
|
||||
return sorted(rows, key=lambda row: row["time"])
|
||||
|
||||
|
||||
def split_message_chunks(text: str, limit: int) -> list[str]:
|
||||
text = str(text or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
chunks: list[str] = []
|
||||
while len(text) > limit:
|
||||
split_at = text.rfind("\n", 0, limit)
|
||||
if split_at < limit // 2:
|
||||
split_at = text.rfind(" ", 0, limit)
|
||||
if split_at < limit // 2:
|
||||
split_at = limit
|
||||
chunks.append(text[:split_at].strip())
|
||||
text = text[split_at:].strip()
|
||||
if text:
|
||||
chunks.append(text)
|
||||
return chunks
|
||||
|
||||
|
||||
def local_upload_path(url: str) -> Path | None:
|
||||
raw = str(url or "").strip()
|
||||
prefix = "/uploads/"
|
||||
if not raw.startswith(prefix):
|
||||
return None
|
||||
path = (UPLOAD_ROOT / raw.removeprefix(prefix)).resolve()
|
||||
try:
|
||||
path.relative_to(UPLOAD_ROOT.resolve())
|
||||
except ValueError:
|
||||
return None
|
||||
return path if path.is_file() else None
|
||||
|
||||
|
||||
class MAXAPIClient:
|
||||
def __init__(self, token: str, base_url: str, timeout_sec: int, max_attempts: int, retry_backoff_max_sec: int) -> None:
|
||||
self.token = token
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.timeout_sec = timeout_sec
|
||||
self.max_attempts = max_attempts
|
||||
self.retry_backoff_max_sec = retry_backoff_max_sec
|
||||
self.session: aiohttp.ClientSession | None = None
|
||||
|
||||
async def __aenter__(self) -> "MAXAPIClient":
|
||||
timeout = aiohttp.ClientTimeout(total=self.timeout_sec)
|
||||
self.session = aiohttp.ClientSession(timeout=timeout, headers={"Authorization": self.token})
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb) -> None:
|
||||
if self.session:
|
||||
await self.session.close()
|
||||
|
||||
async def request(self, method: str, path: str, **kwargs) -> dict[str, Any]:
|
||||
if not self.session:
|
||||
raise RuntimeError("MAX session is not initialized")
|
||||
url = f"{self.base_url}{path}"
|
||||
last_error = ""
|
||||
for attempt in range(1, self.max_attempts + 1):
|
||||
try:
|
||||
async with self.session.request(method, url, **kwargs) as resp:
|
||||
text = await resp.text()
|
||||
try:
|
||||
data = json.loads(text) if text else {}
|
||||
except json.JSONDecodeError:
|
||||
data = {"raw": text}
|
||||
if 200 <= resp.status < 300:
|
||||
return data
|
||||
retry_after = resp.headers.get("Retry-After")
|
||||
message = data.get("message") if isinstance(data, dict) else text
|
||||
code = data.get("code") if isinstance(data, dict) else ""
|
||||
last_error = f"MAX API {resp.status} {code}: {message or text}"
|
||||
if resp.status == 429 and retry_after:
|
||||
await asyncio.sleep(float(retry_after) + 0.5)
|
||||
continue
|
||||
if code == "attachment.not.ready" and attempt < self.max_attempts:
|
||||
await asyncio.sleep(min(2 ** attempt, self.retry_backoff_max_sec))
|
||||
continue
|
||||
if resp.status < 500:
|
||||
raise MAXAPIError(last_error, resp.status, str(code or ""))
|
||||
except Exception as exc:
|
||||
last_error = str(exc)
|
||||
if attempt >= self.max_attempts:
|
||||
raise
|
||||
await asyncio.sleep(min(2 ** attempt, self.retry_backoff_max_sec))
|
||||
raise RuntimeError(last_error or "MAX API request failed")
|
||||
|
||||
async def send_message(self, chat_id: int, text: str, attachments: list[dict[str, Any]] | None = None) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {"text": text, "notify": True}
|
||||
if attachments:
|
||||
payload["attachments"] = attachments
|
||||
return await self.request("POST", f"/messages?chat_id={chat_id}", json=payload)
|
||||
|
||||
async def get_video_info(self, token: str) -> dict[str, Any]:
|
||||
return await self.request("GET", f"/videos/{quote(token, safe='')}")
|
||||
|
||||
async def get_upload_url(self, media_type: str) -> dict[str, Any]:
|
||||
upload_type = "video" if media_type == "video" else "image"
|
||||
return await self.request("POST", f"/uploads?type={upload_type}")
|
||||
|
||||
async def upload_file(self, upload_url: str, path: Path) -> dict[str, Any]:
|
||||
if not self.session:
|
||||
raise RuntimeError("MAX session is not initialized")
|
||||
form = aiohttp.FormData()
|
||||
content_type = mimetypes.guess_type(str(path))[0] or "application/octet-stream"
|
||||
with path.open("rb") as fh:
|
||||
form.add_field("data", fh, filename=path.name, content_type=content_type)
|
||||
async with self.session.post(upload_url, data=form) as resp:
|
||||
text = await resp.text()
|
||||
try:
|
||||
data = json.loads(text) if text else {}
|
||||
except json.JSONDecodeError:
|
||||
data = {"raw": text}
|
||||
if resp.status < 200 or resp.status >= 300:
|
||||
raise RuntimeError(f"MAX upload {resp.status}: {data}")
|
||||
return data
|
||||
|
||||
async def react(self, path_template: str, message_id: str, reaction: str) -> None:
|
||||
path = path_template.format(message_id=message_id)
|
||||
payload = {"reaction": reaction}
|
||||
await self.request("POST", path, json=payload)
|
||||
|
||||
|
||||
class MAXPoster:
|
||||
def __init__(self) -> None:
|
||||
self.pool = None
|
||||
self.heartbeat = HeartbeatReporter(WORKER_MAX_POSTER, 30)
|
||||
self.token = ""
|
||||
self.chat_id = 0
|
||||
self.api_base_url = "https://platform-api2.max.ru"
|
||||
self.schedule: list[dict[str, Any]] = []
|
||||
self.interval_sec = 60
|
||||
self.message_limit = MAX_MESSAGE_LIMIT
|
||||
self.media_max_items = MAX_MEDIA_ITEMS
|
||||
self.media_process_delay_sec = 3.0
|
||||
self.video_ready_attempts = 5
|
||||
self.video_ready_delay_sec = 10.0
|
||||
self.max_attempts = 3
|
||||
self.retry_backoff_max_sec = 30
|
||||
self.send_delay_sec = 1.0
|
||||
self.recent_window = 20
|
||||
self.category_repeat_penalty = 3.0
|
||||
self.source_repeat_penalty = 4.0
|
||||
self.auto_reaction_enabled = True
|
||||
self.auto_reaction = "👍"
|
||||
self.reaction_path_template = "/messages/{message_id}/reactions"
|
||||
self.dry_run = False
|
||||
|
||||
async def init(self) -> None:
|
||||
self.pool = await get_pool()
|
||||
|
||||
async def reload_settings(self) -> None:
|
||||
self.token = str(await fetch_setting("max_poster_bot_token", "") or "").strip()
|
||||
self.chat_id = int(str(await fetch_setting("max_poster_chat_id", "0") or "0").strip())
|
||||
self.api_base_url = str(await fetch_setting("max_poster_api_base_url", self.api_base_url) or self.api_base_url).strip()
|
||||
self.schedule = normalize_schedule(await fetch_setting("max_poster_schedule_json", []))
|
||||
self.interval_sec = max(10, await fetch_int_setting("max_poster_interval_sec", 60))
|
||||
self.message_limit = max(512, min(MAX_MESSAGE_LIMIT, await fetch_int_setting("max_poster_message_limit", MAX_MESSAGE_LIMIT)))
|
||||
self.media_max_items = max(0, min(MAX_MEDIA_ITEMS, await fetch_int_setting("max_poster_media_max_items", MAX_MEDIA_ITEMS)))
|
||||
self.media_process_delay_sec = max(0.0, await fetch_float_setting("max_poster_media_process_delay_sec", 3.0))
|
||||
self.video_ready_attempts = max(1, await fetch_int_setting("max_poster_video_ready_attempts", 5))
|
||||
self.video_ready_delay_sec = max(1.0, await fetch_float_setting("max_poster_video_ready_delay_sec", 10.0))
|
||||
self.max_attempts = max(1, await fetch_int_setting("max_poster_max_attempts", 3))
|
||||
self.retry_backoff_max_sec = max(1, await fetch_int_setting("max_poster_retry_backoff_max_sec", 30))
|
||||
self.send_delay_sec = max(0.0, await fetch_float_setting("max_poster_send_delay_sec", 1.0))
|
||||
self.recent_window = max(1, await fetch_int_setting("max_poster_recent_window", 20))
|
||||
self.category_repeat_penalty = max(0.0, await fetch_float_setting("max_poster_category_repeat_penalty", 3.0))
|
||||
self.source_repeat_penalty = max(0.0, await fetch_float_setting("max_poster_source_repeat_penalty", 4.0))
|
||||
self.auto_reaction_enabled = await fetch_bool_setting("max_poster_auto_reaction_enabled", True)
|
||||
self.auto_reaction = str(await fetch_setting("max_poster_auto_reaction", "👍") or "👍").strip() or "👍"
|
||||
self.reaction_path_template = str(await fetch_setting("max_poster_reaction_path_template", self.reaction_path_template) or "").strip()
|
||||
self.dry_run = await fetch_bool_setting("max_poster_dry_run", False)
|
||||
if not self.token:
|
||||
raise RuntimeError("max_poster_bot_token is empty")
|
||||
if not self.chat_id:
|
||||
raise RuntimeError("max_poster_chat_id is empty")
|
||||
logger.info("MAX poster config: chat={} schedule={} dry_run={}", self.chat_id, self.schedule, self.dry_run)
|
||||
|
||||
async def due_slots(self) -> list[dict[str, Any]]:
|
||||
now = datetime.now(LOCAL_TZ)
|
||||
today = now.date()
|
||||
due: list[dict[str, Any]] = []
|
||||
for row in self.schedule:
|
||||
if not row.get("enabled"):
|
||||
continue
|
||||
hour, minute = [int(part) for part in str(row["time"]).split(":", 1)]
|
||||
slot_start = datetime.combine(today, time(hour=hour, minute=minute), tzinfo=LOCAL_TZ)
|
||||
slot_end = slot_start + timedelta(minutes=10)
|
||||
if not (slot_start <= now < slot_end):
|
||||
continue
|
||||
inserted = await self.pool.fetchrow(
|
||||
"""
|
||||
INSERT INTO publication_runs(poster, schedule_id, scheduled_for, scheduled_time, planned_count, status)
|
||||
VALUES($1, $2, $3, $4, $5, 'started')
|
||||
ON CONFLICT (poster, schedule_id, scheduled_for) DO NOTHING
|
||||
RETURNING id
|
||||
""",
|
||||
"max",
|
||||
row["id"],
|
||||
today,
|
||||
row["time"],
|
||||
int(row["count"]),
|
||||
)
|
||||
if inserted:
|
||||
due.append({**row, "run_id": int(inserted["id"]), "date": today})
|
||||
return due
|
||||
|
||||
async def load_recent(self) -> list[dict[str, Any]]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT rp.final_category_tag, rp.rewrite_category_tag, rp.final_source_tag, rp.rewrite_source_tag
|
||||
FROM post_publications pp
|
||||
JOIN raw_posts rp ON rp.id=pp.raw_post_id
|
||||
WHERE pp.platform='max'
|
||||
AND pp.status=$1
|
||||
AND pp.target_id=$2
|
||||
ORDER BY pp.published_at DESC NULLS LAST, pp.id DESC
|
||||
LIMIT $3::int
|
||||
""",
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
str(self.chat_id),
|
||||
self.recent_window,
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
async def load_candidates(self, limit: int) -> list[dict[str, Any]]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT rp.*, s.name AS source_name, s.tag AS source_tag
|
||||
FROM raw_posts rp
|
||||
JOIN sources s ON s.id=rp.source_id
|
||||
LEFT JOIN post_publications pp ON pp.raw_post_id=rp.id AND pp.platform='max'
|
||||
WHERE rp.status='storage_ready'
|
||||
AND rp.rewrite_status='ready'
|
||||
AND COALESCE(rp.editorial_status, 'review') IN ('accepted', 'published', 'publish_failed')
|
||||
AND COALESCE(pp.status, $1) = $1
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media rpm
|
||||
WHERE rpm.raw_post_id=rp.id
|
||||
AND COALESCE(rpm.editor_hidden, FALSE)=FALSE
|
||||
AND rpm.media_type IN ('photo', 'video')
|
||||
AND COALESCE(rpm.original_url, '') <> ''
|
||||
)
|
||||
ORDER BY COALESCE(rp.reviewed_at, rp.edited_at, rp.rewritten_at, rp.created_at) ASC,
|
||||
rp.id ASC
|
||||
LIMIT $2::int
|
||||
""",
|
||||
PUBLICATION_STATUS_PENDING,
|
||||
max(limit, 50),
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def choose_posts(self, candidates: list[dict[str, Any]], recent: list[dict[str, Any]], count: int) -> list[dict[str, Any]]:
|
||||
selected: list[dict[str, Any]] = []
|
||||
recent_categories = [str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "") for row in recent]
|
||||
recent_sources = [str(row.get("final_source_tag") or row.get("rewrite_source_tag") or "") for row in recent]
|
||||
pool = list(candidates)
|
||||
while pool and len(selected) < count:
|
||||
best_idx = 0
|
||||
selected_categories = [str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "") for row in selected]
|
||||
selected_sources = [str(row.get("final_source_tag") or row.get("rewrite_source_tag") or row.get("source_tag") or "") for row in selected]
|
||||
previous_category = selected_categories[-1] if selected_categories else (recent_categories[0] if recent_categories else "")
|
||||
previous_source = selected_sources[-1] if selected_sources else (recent_sources[0] if recent_sources else "")
|
||||
best_rank: tuple[int, float, int] | None = None
|
||||
for idx, post in enumerate(pool):
|
||||
category = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or "")
|
||||
source = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "")
|
||||
immediate_repeats = int(bool(category) and category == previous_category) + int(bool(source) and source == previous_source)
|
||||
repeat_penalty = (
|
||||
(recent_categories.count(category) + selected_categories.count(category)) * self.category_repeat_penalty
|
||||
+ (recent_sources.count(source) + selected_sources.count(source)) * self.source_repeat_penalty
|
||||
)
|
||||
rank = (immediate_repeats, repeat_penalty, idx)
|
||||
if best_rank is None or rank < best_rank:
|
||||
best_idx = idx
|
||||
best_rank = rank
|
||||
selected.append(pool.pop(best_idx))
|
||||
return selected
|
||||
|
||||
async def load_media(self, raw_post_id: int) -> list[dict[str, Any]]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT *
|
||||
FROM raw_post_media
|
||||
WHERE raw_post_id=$1
|
||||
AND COALESCE(editor_hidden, FALSE)=FALSE
|
||||
AND media_type IN ('photo', 'video')
|
||||
AND COALESCE(original_url, '') <> ''
|
||||
ORDER BY sort_order ASC, id ASC
|
||||
LIMIT $2::int
|
||||
""",
|
||||
raw_post_id,
|
||||
self.media_max_items,
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def build_text(self, post: dict[str, Any]) -> str:
|
||||
category_tag = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or post.get("final_category") or post.get("rewrite_category") or "")
|
||||
source_tag = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "")
|
||||
return build_publication_text(post.get("final_text") or post.get("rewritten_text") or "", category_tag, source_tag)
|
||||
|
||||
async def download_media_to_temp(self, session: aiohttp.ClientSession, item: dict[str, Any]) -> Path | None:
|
||||
media_type = str(item.get("media_type") or "")
|
||||
file_id = str(item.get("tg_file_id") or item.get("storage_attachment_id") or "").strip()
|
||||
if media_type == "video" and file_id:
|
||||
return await self.download_telegram_file_to_temp(session, file_id, ".mp4")
|
||||
url = str(item.get("original_url") or "").strip()
|
||||
local_path = local_upload_path(url)
|
||||
if local_path:
|
||||
return local_path
|
||||
if not url.startswith(("http://", "https://")):
|
||||
return None
|
||||
suffix = Path(url.split("?", 1)[0]).suffix[:12] or (".mp4" if item.get("media_type") == "video" else ".jpg")
|
||||
tmp = tempfile.NamedTemporaryFile(prefix="max-poster-", suffix=suffix, delete=False)
|
||||
path = Path(tmp.name)
|
||||
tmp.close()
|
||||
total = 0
|
||||
try:
|
||||
async with session.get(url) as resp:
|
||||
resp.raise_for_status()
|
||||
with path.open("wb") as fh:
|
||||
async for chunk in resp.content.iter_chunked(1024 * 512):
|
||||
total += len(chunk)
|
||||
if total > MAX_VIDEO_BYTES:
|
||||
raise RuntimeError("media file is larger than MAX video limit")
|
||||
fh.write(chunk)
|
||||
return path
|
||||
except Exception:
|
||||
path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
async def download_telegram_file_to_temp(self, session: aiohttp.ClientSession, file_id: str, suffix: str) -> Path:
|
||||
token = settings.tg_bot_token
|
||||
if not token:
|
||||
raise RuntimeError("TG_BOT_TOKEN is empty; cannot download stored video")
|
||||
api_base = str(await fetch_setting("local_bot_api_url", settings.local_bot_api_url) or "").strip().rstrip("/")
|
||||
if not api_base:
|
||||
api_base = "https://api.telegram.org"
|
||||
get_file_url = f"{api_base}/bot{token}/getFile"
|
||||
async with session.post(get_file_url, json={"file_id": file_id}) as resp:
|
||||
data = await resp.json(content_type=None)
|
||||
if not data.get("ok") or not isinstance(data.get("result"), dict):
|
||||
raise RuntimeError(f"Telegram getFile failed: {data}")
|
||||
file_path = str(data["result"].get("file_path") or "").strip()
|
||||
if not file_path:
|
||||
raise RuntimeError(f"Telegram getFile did not return file_path: {data}")
|
||||
local_file = Path(file_path)
|
||||
if local_file.is_absolute() and local_file.is_file():
|
||||
if local_file.stat().st_size > MAX_VIDEO_BYTES:
|
||||
raise RuntimeError("telegram local video file is larger than MAX video limit")
|
||||
tmp = tempfile.NamedTemporaryFile(prefix="max-poster-tg-", suffix=suffix, delete=False)
|
||||
path = Path(tmp.name)
|
||||
tmp.close()
|
||||
try:
|
||||
shutil.copyfile(local_file, path)
|
||||
return path
|
||||
except Exception:
|
||||
path.unlink(missing_ok=True)
|
||||
raise
|
||||
download_url = f"{api_base}/file/bot{token}/{file_path}"
|
||||
tmp = tempfile.NamedTemporaryFile(prefix="max-poster-tg-", suffix=suffix, delete=False)
|
||||
path = Path(tmp.name)
|
||||
tmp.close()
|
||||
total = 0
|
||||
try:
|
||||
async with session.get(download_url) as resp:
|
||||
resp.raise_for_status()
|
||||
with path.open("wb") as fh:
|
||||
async for chunk in resp.content.iter_chunked(1024 * 1024):
|
||||
total += len(chunk)
|
||||
if total > MAX_VIDEO_BYTES:
|
||||
raise RuntimeError("telegram video file is larger than MAX video limit")
|
||||
fh.write(chunk)
|
||||
return path
|
||||
except Exception:
|
||||
path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
async def upload_media(self, client: MAXAPIClient, item: dict[str, Any]) -> dict[str, Any] | None:
|
||||
media_type = str(item.get("media_type") or "")
|
||||
if media_type not in {"photo", "video"}:
|
||||
return None
|
||||
temp_path: Path | None = None
|
||||
downloaded = False
|
||||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=180)) as session:
|
||||
temp_path = await self.download_media_to_temp(session, item)
|
||||
downloaded = bool(temp_path and not local_upload_path(str(item.get("original_url") or "")))
|
||||
if not temp_path:
|
||||
return None
|
||||
try:
|
||||
upload_info = await client.get_upload_url("video" if media_type == "video" else "image")
|
||||
upload_url = str(upload_info.get("url") or "").strip()
|
||||
if not upload_url:
|
||||
raise RuntimeError(f"MAX upload URL is empty: {upload_info}")
|
||||
payload = await client.upload_file(upload_url, temp_path)
|
||||
if media_type == "video" and upload_info.get("token") and not payload.get("token"):
|
||||
payload["token"] = upload_info["token"]
|
||||
has_image_token = bool(payload.get("photos")) if isinstance(payload.get("photos"), dict) else False
|
||||
if not payload.get("token") and not has_image_token:
|
||||
raise RuntimeError(f"MAX upload did not return token: {payload}")
|
||||
return {"type": "video" if media_type == "video" else "image", "payload": payload}
|
||||
finally:
|
||||
if downloaded and temp_path:
|
||||
temp_path.unlink(missing_ok=True)
|
||||
|
||||
def message_id_from_response(self, data: dict[str, Any]) -> str:
|
||||
candidates = [
|
||||
data.get("id"),
|
||||
data.get("mid"),
|
||||
(data.get("message") or {}).get("id") if isinstance(data.get("message"), dict) else None,
|
||||
(data.get("message") or {}).get("mid") if isinstance(data.get("message"), dict) else None,
|
||||
]
|
||||
message = data.get("message") if isinstance(data.get("message"), dict) else {}
|
||||
body = message.get("body") if isinstance(message.get("body"), dict) else {}
|
||||
candidates.extend([body.get("id"), body.get("mid"), body.get("message_id")])
|
||||
return next((str(value) for value in candidates if value), "")
|
||||
|
||||
async def send_message_waiting_for_media(
|
||||
self,
|
||||
client: MAXAPIClient,
|
||||
text: str,
|
||||
attachments: list[dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
has_video = any(item.get("type") == "video" for item in attachments)
|
||||
attempts = self.video_ready_attempts if has_video else 1
|
||||
delay = self.video_ready_delay_sec
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
return await client.send_message(self.chat_id, text, attachments=attachments or None)
|
||||
except MAXAPIError as exc:
|
||||
if exc.code != "attachment.not.ready" or attempt >= attempts:
|
||||
raise
|
||||
logger.warning(
|
||||
"MAX attachment is not ready, retry send in {}s attempt={}/{}",
|
||||
delay,
|
||||
attempt + 1,
|
||||
attempts,
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
delay = min(delay * 2, self.retry_backoff_max_sec * 4)
|
||||
raise RuntimeError("MAX attachment readiness retry exhausted")
|
||||
|
||||
def video_tokens(self, attachments: list[dict[str, Any]]) -> list[str]:
|
||||
tokens: list[str] = []
|
||||
for item in attachments:
|
||||
if item.get("type") != "video":
|
||||
continue
|
||||
payload = item.get("payload") if isinstance(item.get("payload"), dict) else {}
|
||||
token = str(payload.get("token") or "").strip()
|
||||
if token:
|
||||
tokens.append(token)
|
||||
return tokens
|
||||
|
||||
async def wait_for_videos(self, client: MAXAPIClient, attachments: list[dict[str, Any]]) -> None:
|
||||
tokens = self.video_tokens(attachments)
|
||||
if not tokens:
|
||||
return
|
||||
delay = self.video_ready_delay_sec
|
||||
for attempt in range(1, self.video_ready_attempts + 1):
|
||||
pending: list[str] = []
|
||||
for token in tokens:
|
||||
info = await client.get_video_info(token)
|
||||
if not info.get("urls"):
|
||||
pending.append(token)
|
||||
if not pending:
|
||||
return
|
||||
if attempt >= self.video_ready_attempts:
|
||||
raise RuntimeError(f"MAX video is not ready after {attempt} attempts")
|
||||
logger.warning(
|
||||
"MAX video is not ready, retry info in {}s attempt={}/{} pending={}",
|
||||
delay,
|
||||
attempt + 1,
|
||||
self.video_ready_attempts,
|
||||
len(pending),
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
delay = min(delay * 2, self.retry_backoff_max_sec * 4)
|
||||
|
||||
async def send_post(self, post: dict[str, Any]) -> tuple[list[str], str | None, list[dict[str, Any]], str | None]:
|
||||
text = self.build_text(post)
|
||||
chunks = split_message_chunks(text, self.message_limit)
|
||||
media_rows = await self.load_media(int(post["id"])) if self.media_max_items else []
|
||||
message_ids: list[str] = []
|
||||
attachments: list[dict[str, Any]] = []
|
||||
first_url: str | None = None
|
||||
reaction_error: str | None = None
|
||||
async with MAXAPIClient(
|
||||
self.token,
|
||||
self.api_base_url,
|
||||
timeout_sec=max(30, await fetch_int_setting("max_poster_timeout_sec", 120)),
|
||||
max_attempts=self.max_attempts,
|
||||
retry_backoff_max_sec=self.retry_backoff_max_sec,
|
||||
) as client:
|
||||
for item in media_rows:
|
||||
try:
|
||||
attachment = await self.upload_media(client, item)
|
||||
if attachment:
|
||||
attachments.append(attachment)
|
||||
except Exception as exc:
|
||||
logger.warning("MAX media skipped raw_post={} media_id={}: {}", post["id"], item.get("id"), exc)
|
||||
if len(attachments) >= self.media_max_items:
|
||||
break
|
||||
if self.media_process_delay_sec and attachments:
|
||||
await asyncio.sleep(self.media_process_delay_sec)
|
||||
if self.dry_run:
|
||||
return ["dry-run"], None, attachments, None
|
||||
await self.wait_for_videos(client, attachments)
|
||||
first_text = chunks[0] if chunks else ""
|
||||
first = await self.send_message_waiting_for_media(client, first_text, attachments)
|
||||
message = first.get("message") if isinstance(first.get("message"), dict) else first
|
||||
first_url = message.get("url") if isinstance(message, dict) else None
|
||||
first_message_id = self.message_id_from_response(first)
|
||||
if first_message_id:
|
||||
message_ids.append(first_message_id)
|
||||
if self.auto_reaction_enabled and self.reaction_path_template:
|
||||
try:
|
||||
await client.react(self.reaction_path_template, first_message_id, self.auto_reaction)
|
||||
except Exception as exc:
|
||||
reaction_error = str(exc)
|
||||
logger.warning("MAX auto reaction failed message={}: {}", first_message_id, exc)
|
||||
for chunk in chunks[1:]:
|
||||
if self.send_delay_sec:
|
||||
await asyncio.sleep(self.send_delay_sec)
|
||||
result = await client.send_message(self.chat_id, chunk)
|
||||
message_id = self.message_id_from_response(result)
|
||||
if message_id:
|
||||
message_ids.append(message_id)
|
||||
return message_ids, first_url, attachments, reaction_error
|
||||
|
||||
async def upsert_publication(
|
||||
self,
|
||||
post_id: int,
|
||||
status: str,
|
||||
external_id: str | None = None,
|
||||
url: str | None = None,
|
||||
error: str | None = None,
|
||||
attachments: list[dict[str, Any]] | None = None,
|
||||
) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
INSERT INTO post_publications(
|
||||
raw_post_id, platform, status, target_id, external_id, url, error, attachments_json, published_at, updated_at
|
||||
)
|
||||
VALUES($1, 'max', $2, $3, $4, $5, $6, $7::jsonb, CASE WHEN $2=$8 THEN NOW() ELSE NULL END, NOW())
|
||||
ON CONFLICT (raw_post_id, platform) DO UPDATE
|
||||
SET status=EXCLUDED.status,
|
||||
target_id=EXCLUDED.target_id,
|
||||
external_id=EXCLUDED.external_id,
|
||||
url=EXCLUDED.url,
|
||||
error=EXCLUDED.error,
|
||||
attachments_json=EXCLUDED.attachments_json,
|
||||
published_at=CASE WHEN EXCLUDED.status=$8 THEN COALESCE(EXCLUDED.published_at, NOW()) ELSE post_publications.published_at END,
|
||||
updated_at=NOW()
|
||||
""",
|
||||
post_id,
|
||||
status,
|
||||
str(self.chat_id),
|
||||
external_id,
|
||||
url,
|
||||
error[:1000] if error else None,
|
||||
json.dumps(attachments or [], ensure_ascii=False),
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
)
|
||||
|
||||
async def process_slot(self, slot: dict[str, Any]) -> int:
|
||||
count = int(slot["count"])
|
||||
recent = await self.load_recent()
|
||||
candidates = await self.load_candidates(max(50, count * 10))
|
||||
posts = self.choose_posts(candidates, recent, count)
|
||||
published = 0
|
||||
for post in posts:
|
||||
await self.heartbeat.beat(self.pool, status="publishing", meta={"post_id": int(post["id"]), "slot": slot["id"]}, force=True)
|
||||
try:
|
||||
message_ids, url, attachments, reaction_error = await self.send_post(post)
|
||||
await self.upsert_publication(
|
||||
int(post["id"]),
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
",".join(message_ids),
|
||||
url,
|
||||
error=None,
|
||||
attachments=attachments,
|
||||
)
|
||||
published += 1
|
||||
logger.info("MAX published raw_post={} messages={} attachments={}", post["id"], message_ids, len(attachments))
|
||||
except Exception as exc:
|
||||
await self.upsert_publication(int(post["id"]), PUBLICATION_STATUS_FAILED, error=str(exc))
|
||||
logger.exception("MAX publish failed raw_post={}: {}", post["id"], exc)
|
||||
if self.send_delay_sec:
|
||||
await asyncio.sleep(self.send_delay_sec)
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE publication_runs
|
||||
SET published_count=$2,
|
||||
status=$3,
|
||||
error=$4,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
int(slot["run_id"]),
|
||||
published,
|
||||
"done" if published == count else "partial",
|
||||
None if published == count else f"published {published} of {count}",
|
||||
)
|
||||
return published
|
||||
|
||||
async def run_once(self) -> bool:
|
||||
await self.reload_settings()
|
||||
enabled = await is_worker_enabled(self.pool, WORKER_MAX_POSTER)
|
||||
app_enabled = await fetch_bool_setting("max_poster_enabled", False)
|
||||
if not enabled or not app_enabled:
|
||||
await self.heartbeat.beat(self.pool, status="disabled", force=True)
|
||||
return False
|
||||
slots = await self.due_slots()
|
||||
if not slots:
|
||||
await self.heartbeat.beat(self.pool, status="idle")
|
||||
return False
|
||||
total = 0
|
||||
for slot in slots:
|
||||
total += await self.process_slot(slot)
|
||||
await self.heartbeat.beat(self.pool, status="idle", meta={"published": total}, force=True)
|
||||
return bool(total)
|
||||
|
||||
async def run_loop(self) -> None:
|
||||
await self.init()
|
||||
logger.info("{} started pid={}", WORKER_MAX_POSTER, os.getpid())
|
||||
while True:
|
||||
try:
|
||||
await self.run_once()
|
||||
except Exception as exc:
|
||||
logger.exception("MAX poster loop error: {}", exc)
|
||||
await asyncio.sleep(self.interval_sec)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
logger.remove()
|
||||
logger.add(sys.stdout, level=settings.log_level)
|
||||
worker = MAXPoster()
|
||||
await worker.run_loop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,141 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from ..config import settings
|
||||
from ..constants import WORKER_SITE_POSTER
|
||||
from ..db import fetch_bool_setting, fetch_int_setting, fetch_setting, get_pool
|
||||
from ..heartbeat import HeartbeatReporter
|
||||
from ..jobs import is_worker_enabled
|
||||
|
||||
|
||||
class SitePoster:
|
||||
def __init__(self) -> None:
|
||||
self.pool = None
|
||||
self.heartbeat = HeartbeatReporter(WORKER_SITE_POSTER, 30)
|
||||
self.project_root = Path(__file__).resolve().parents[3]
|
||||
self.interval_sec = 300
|
||||
self.photo_batch_size = 10
|
||||
self.video_batch_size = 1
|
||||
self.ghost_url = "https://f-n8.ru"
|
||||
self.key_file = "/opt/vk-parser/.site-poster-key"
|
||||
|
||||
async def init(self) -> None:
|
||||
self.pool = await get_pool()
|
||||
|
||||
async def reload_settings(self) -> None:
|
||||
self.interval_sec = max(30, await fetch_int_setting("site_poster_interval_sec", 300))
|
||||
self.photo_batch_size = max(0, await fetch_int_setting("site_poster_photo_batch_size", 10))
|
||||
self.video_batch_size = max(0, await fetch_int_setting("site_poster_video_batch_size", 1))
|
||||
self.ghost_url = str(await fetch_setting("site_poster_ghost_url", "https://f-n8.ru") or "https://f-n8.ru").rstrip("/")
|
||||
self.key_file = str(await fetch_setting("site_poster_key_file", "/opt/vk-parser/.site-poster-key") or "").strip()
|
||||
if not self.key_file:
|
||||
raise RuntimeError("site_poster_key_file is empty")
|
||||
|
||||
async def publish_mode(self, flag: str, limit: int) -> tuple[int, int]:
|
||||
if limit < 1:
|
||||
return 0, 0
|
||||
command = [
|
||||
sys.executable,
|
||||
str(self.project_root / "scripts" / "publish_site_batch.py"),
|
||||
flag,
|
||||
"--limit",
|
||||
str(limit),
|
||||
"--ghost-url",
|
||||
self.ghost_url,
|
||||
"--key-file",
|
||||
self.key_file,
|
||||
]
|
||||
env = os.environ.copy()
|
||||
env["PYTHONPATH"] = str(self.project_root / "src")
|
||||
env["PYTHONUNBUFFERED"] = "1"
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*command,
|
||||
cwd=str(self.project_root),
|
||||
env=env,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
)
|
||||
published = 0
|
||||
failed = 0
|
||||
assert process.stdout is not None
|
||||
while True:
|
||||
try:
|
||||
raw_line = await asyncio.wait_for(process.stdout.readline(), timeout=20)
|
||||
except TimeoutError:
|
||||
await self.heartbeat.beat(
|
||||
self.pool,
|
||||
status="publishing",
|
||||
meta={"mode": flag, "published": published, "failed": failed},
|
||||
force=True,
|
||||
)
|
||||
continue
|
||||
if not raw_line:
|
||||
break
|
||||
line = raw_line.decode("utf-8", errors="replace").strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("PUBLISHED "):
|
||||
published += 1
|
||||
logger.info("Site poster: {}", line)
|
||||
elif line.startswith("FAILED "):
|
||||
failed += 1
|
||||
logger.error("Site poster: {}", line)
|
||||
elif line not in {"NO_READY_POSTS", "ALREADY_RUNNING"}:
|
||||
logger.debug("Site poster child: {}", line)
|
||||
await self.heartbeat.beat(
|
||||
self.pool,
|
||||
status="publishing",
|
||||
meta={"mode": flag, "published": published, "failed": failed},
|
||||
)
|
||||
return_code = await process.wait()
|
||||
if return_code != 0:
|
||||
raise RuntimeError(f"site publisher exited with code {return_code} for {flag}")
|
||||
return published, failed
|
||||
|
||||
async def run_once(self) -> bool:
|
||||
await self.reload_settings()
|
||||
worker_enabled = await is_worker_enabled(self.pool, WORKER_SITE_POSTER)
|
||||
app_enabled = await fetch_bool_setting("site_poster_enabled", False)
|
||||
if not worker_enabled or not app_enabled:
|
||||
await self.heartbeat.beat(self.pool, status="disabled", force=True)
|
||||
return False
|
||||
await self.heartbeat.beat(self.pool, status="checking", force=True)
|
||||
photo_published, photo_failed = await self.publish_mode("--all-ready", self.photo_batch_size)
|
||||
video_published, video_failed = await self.publish_mode("--all-ready-videos", self.video_batch_size)
|
||||
published = photo_published + video_published
|
||||
failed = photo_failed + video_failed
|
||||
await self.heartbeat.beat(
|
||||
self.pool,
|
||||
status="idle" if not failed else "error",
|
||||
meta={"published": published, "failed": failed},
|
||||
force=True,
|
||||
)
|
||||
return bool(published)
|
||||
|
||||
async def run_loop(self) -> None:
|
||||
await self.init()
|
||||
logger.info("{} started pid={}", WORKER_SITE_POSTER, os.getpid())
|
||||
while True:
|
||||
try:
|
||||
await self.run_once()
|
||||
except Exception as exc:
|
||||
logger.exception("Site poster loop error: {}", exc)
|
||||
await self.heartbeat.beat(self.pool, status="error", meta={"error": str(exc)[:500]}, force=True)
|
||||
await asyncio.sleep(self.interval_sec)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
logger.remove()
|
||||
logger.add(sys.stdout, level=settings.log_level)
|
||||
worker = SitePoster()
|
||||
await worker.run_loop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,535 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, time, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from aiogram import Bot
|
||||
from aiogram.exceptions import TelegramRetryAfter
|
||||
from aiogram.types import FSInputFile, InputMediaPhoto, InputMediaVideo
|
||||
from loguru import logger
|
||||
|
||||
from ..config import settings
|
||||
from ..constants import (
|
||||
PUBLICATION_STATUS_FAILED,
|
||||
PUBLICATION_STATUS_PENDING,
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
WORKER_TG_POSTER,
|
||||
)
|
||||
from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool
|
||||
from ..heartbeat import HeartbeatReporter
|
||||
from ..jobs import is_worker_enabled
|
||||
from ..text_utils import build_publication_text
|
||||
|
||||
LOCAL_TZ = ZoneInfo("Asia/Yekaterinburg")
|
||||
MAX_MEDIA_GROUP = 10
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
UPLOAD_ROOT = PROJECT_ROOT / "uploads"
|
||||
|
||||
|
||||
def parse_topic(value: str) -> tuple[int, int | None]:
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
raise RuntimeError("tg_poster_chat_id is empty")
|
||||
if ":" in raw:
|
||||
chat_id, thread_id = raw.split(":", 1)
|
||||
return int(chat_id), int(thread_id)
|
||||
return int(raw), None
|
||||
|
||||
|
||||
def valid_time(value: str) -> bool:
|
||||
return bool(re.fullmatch(r"(?:[01]\d|2[0-3]):[0-5]\d", str(value or "").strip()))
|
||||
|
||||
|
||||
def split_message_chunks(text: str, limit: int) -> list[str]:
|
||||
text = str(text or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
chunks: list[str] = []
|
||||
while len(text) > limit:
|
||||
split_at = text.rfind("\n", 0, limit)
|
||||
if split_at < limit // 2:
|
||||
split_at = text.rfind(" ", 0, limit)
|
||||
if split_at < limit // 2:
|
||||
split_at = limit
|
||||
chunks.append(text[:split_at].strip())
|
||||
text = text[split_at:].strip()
|
||||
if text:
|
||||
chunks.append(text)
|
||||
return chunks
|
||||
|
||||
|
||||
def tg_message_url(chat_id: int, message_id: int) -> str:
|
||||
chat = str(abs(int(chat_id)))
|
||||
if chat.startswith("100"):
|
||||
chat = chat[3:]
|
||||
return f"https://t.me/c/{chat}/{message_id}"
|
||||
|
||||
|
||||
def local_upload_path(url: str) -> Path | None:
|
||||
raw = str(url or "").strip()
|
||||
prefix = "/uploads/"
|
||||
if not raw.startswith(prefix):
|
||||
return None
|
||||
path = (UPLOAD_ROOT / raw.removeprefix(prefix)).resolve()
|
||||
try:
|
||||
path.relative_to(UPLOAD_ROOT.resolve())
|
||||
except ValueError:
|
||||
return None
|
||||
return path if path.is_file() else None
|
||||
|
||||
|
||||
class TelegramPoster:
|
||||
def __init__(self) -> None:
|
||||
self.pool = None
|
||||
self.bot: Bot | None = None
|
||||
self.bot_token = ""
|
||||
self.chat_id: int | None = None
|
||||
self.thread_id: int | None = None
|
||||
self.heartbeat = HeartbeatReporter(WORKER_TG_POSTER, 30)
|
||||
self.schedule: list[dict[str, Any]] = []
|
||||
self.media_group_max_items = MAX_MEDIA_GROUP
|
||||
self.caption_limit = 1024
|
||||
self.message_limit = 4096
|
||||
self.max_attempts = 3
|
||||
self.retry_backoff_max_sec = 30
|
||||
self.send_delay_sec = 1.0
|
||||
self.overflow_caption = "⬇️ Описание"
|
||||
self.recent_window = 20
|
||||
self.category_repeat_penalty = 3.0
|
||||
self.source_repeat_penalty = 4.0
|
||||
|
||||
async def init(self) -> None:
|
||||
self.pool = await get_pool()
|
||||
await self.reload_settings()
|
||||
|
||||
async def reload_settings(self) -> None:
|
||||
token = str(await fetch_setting("tg_poster_bot_token", "") or "").strip() or settings.tg_bot_token
|
||||
if not token:
|
||||
raise RuntimeError("tg_poster_bot_token and TG_BOT_TOKEN are empty")
|
||||
if token != self.bot_token:
|
||||
if self.bot:
|
||||
await self.bot.session.close()
|
||||
self.bot = Bot(token=token)
|
||||
self.bot_token = token
|
||||
self.chat_id, self.thread_id = parse_topic(str(await fetch_setting("tg_poster_chat_id", "-1003712724327") or ""))
|
||||
self.schedule = self.normalize_schedule(await fetch_setting("tg_poster_schedule_json", []))
|
||||
self.media_group_max_items = max(1, min(MAX_MEDIA_GROUP, await fetch_int_setting("tg_poster_media_group_max_items", 10)))
|
||||
self.caption_limit = max(128, min(1024, await fetch_int_setting("tg_poster_caption_limit", 1024)))
|
||||
self.message_limit = max(512, min(4096, await fetch_int_setting("tg_poster_message_limit", 4096)))
|
||||
self.max_attempts = max(1, await fetch_int_setting("tg_poster_max_attempts", 3))
|
||||
self.retry_backoff_max_sec = max(1, await fetch_int_setting("tg_poster_retry_backoff_max_sec", 30))
|
||||
self.send_delay_sec = max(0.0, await fetch_float_setting("tg_poster_send_delay_sec", 1.0))
|
||||
self.overflow_caption = str(await fetch_setting("tg_poster_text_overflow_caption", self.overflow_caption) or self.overflow_caption)
|
||||
self.recent_window = max(1, await fetch_int_setting("tg_poster_recent_window", 20))
|
||||
self.category_repeat_penalty = max(0.0, await fetch_float_setting("tg_poster_category_repeat_penalty", 3.0))
|
||||
self.source_repeat_penalty = max(0.0, await fetch_float_setting("tg_poster_source_repeat_penalty", 4.0))
|
||||
logger.info("TG poster config: chat={} schedule={} caption_limit={}", self.chat_id, self.schedule, self.caption_limit)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.bot:
|
||||
await self.bot.session.close()
|
||||
|
||||
def normalize_schedule(self, value: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(value, list):
|
||||
value = []
|
||||
rows: list[dict[str, Any]] = []
|
||||
for idx, item in enumerate(value):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
time_value = str(item.get("time") or "").strip()
|
||||
if not valid_time(time_value):
|
||||
continue
|
||||
count = max(0, min(20, int(item.get("count") or 0)))
|
||||
if count < 1:
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"id": str(item.get("id") or f"slot-{idx + 1}").strip()[:80],
|
||||
"time": time_value,
|
||||
"count": count,
|
||||
"enabled": bool(item.get("enabled", True)),
|
||||
}
|
||||
)
|
||||
return sorted(rows, key=lambda row: row["time"])
|
||||
|
||||
def chat_kwargs(self) -> dict[str, int]:
|
||||
kwargs = {"chat_id": int(self.chat_id)}
|
||||
if self.thread_id:
|
||||
kwargs["message_thread_id"] = int(self.thread_id)
|
||||
return kwargs
|
||||
|
||||
async def tg_retry(self, fn):
|
||||
for attempt in range(1, self.max_attempts + 1):
|
||||
try:
|
||||
return await fn()
|
||||
except TelegramRetryAfter as exc:
|
||||
delay = float(exc.retry_after) + 0.5
|
||||
logger.warning("Telegram flood control, sleep {}s", delay)
|
||||
await asyncio.sleep(delay)
|
||||
except Exception as exc:
|
||||
logger.warning("Telegram post error without retry to avoid duplicate send: {}", exc)
|
||||
raise
|
||||
raise RuntimeError("telegram retry exhausted")
|
||||
|
||||
async def due_slots(self) -> list[dict[str, Any]]:
|
||||
now = datetime.now(LOCAL_TZ)
|
||||
today = now.date()
|
||||
due: list[dict[str, Any]] = []
|
||||
for row in self.schedule:
|
||||
if not row.get("enabled"):
|
||||
continue
|
||||
hour, minute = [int(part) for part in str(row["time"]).split(":", 1)]
|
||||
slot_start = datetime.combine(today, time(hour=hour, minute=minute), tzinfo=LOCAL_TZ)
|
||||
slot_end = slot_start + timedelta(minutes=10)
|
||||
if not (slot_start <= now < slot_end):
|
||||
continue
|
||||
inserted = await self.pool.fetchrow(
|
||||
"""
|
||||
INSERT INTO publication_runs(poster, schedule_id, scheduled_for, scheduled_time, planned_count, status)
|
||||
VALUES($1, $2, $3, $4, $5, 'started')
|
||||
ON CONFLICT (poster, schedule_id, scheduled_for) DO NOTHING
|
||||
RETURNING id
|
||||
""",
|
||||
"tg",
|
||||
row["id"],
|
||||
today,
|
||||
row["time"],
|
||||
int(row["count"]),
|
||||
)
|
||||
if inserted:
|
||||
due.append({**row, "run_id": int(inserted["id"]), "date": today})
|
||||
return due
|
||||
|
||||
async def load_recent(self) -> list[dict[str, Any]]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT final_category_tag, rewrite_category_tag, final_source_tag, rewrite_source_tag
|
||||
FROM raw_posts
|
||||
WHERE publication_status=$1
|
||||
AND tg_publication_chat_id=$2
|
||||
ORDER BY published_at DESC NULLS LAST, id DESC
|
||||
LIMIT $3::int
|
||||
""",
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
self.chat_id,
|
||||
self.recent_window,
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
async def load_candidates(self, limit: int) -> list[dict[str, Any]]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT rp.*, s.name AS source_name, s.tag AS source_tag
|
||||
FROM raw_posts rp
|
||||
JOIN sources s ON s.id=rp.source_id
|
||||
WHERE rp.status='storage_ready'
|
||||
AND rp.rewrite_status='ready'
|
||||
AND rp.editorial_status='accepted'
|
||||
AND COALESCE(rp.publication_status, 'pending') = $1
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media rpm
|
||||
WHERE rpm.raw_post_id=rp.id
|
||||
AND COALESCE(rpm.editor_hidden, FALSE)=FALSE
|
||||
AND rpm.media_type IN ('photo', 'video')
|
||||
AND COALESCE(rpm.tg_file_id, rpm.storage_attachment_id, '') <> ''
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media rpm
|
||||
WHERE rpm.raw_post_id=rp.id
|
||||
AND COALESCE(rpm.editor_hidden, FALSE)=FALSE
|
||||
AND rpm.media_type IN ('photo', 'video')
|
||||
AND COALESCE(rpm.tg_file_id, rpm.storage_attachment_id, '') = ''
|
||||
)
|
||||
ORDER BY COALESCE(rp.reviewed_at, rp.edited_at, rp.rewritten_at, rp.created_at) ASC,
|
||||
rp.id ASC
|
||||
LIMIT $2::int
|
||||
""",
|
||||
PUBLICATION_STATUS_PENDING,
|
||||
max(limit, 50),
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def choose_posts(self, candidates: list[dict[str, Any]], recent: list[dict[str, Any]], count: int) -> list[dict[str, Any]]:
|
||||
selected: list[dict[str, Any]] = []
|
||||
recent_categories = [
|
||||
str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "")
|
||||
for row in recent
|
||||
]
|
||||
recent_sources = [
|
||||
str(row.get("final_source_tag") or row.get("rewrite_source_tag") or "")
|
||||
for row in recent
|
||||
]
|
||||
pool = list(candidates)
|
||||
while pool and len(selected) < count:
|
||||
best_idx = 0
|
||||
selected_categories = [str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "") for row in selected]
|
||||
selected_sources = [str(row.get("final_source_tag") or row.get("rewrite_source_tag") or row.get("source_tag") or "") for row in selected]
|
||||
previous_category = selected_categories[-1] if selected_categories else (recent_categories[0] if recent_categories else "")
|
||||
previous_source = selected_sources[-1] if selected_sources else (recent_sources[0] if recent_sources else "")
|
||||
best_rank: tuple[int, float, int] | None = None
|
||||
for idx, post in enumerate(pool):
|
||||
category = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or "")
|
||||
source = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "")
|
||||
immediate_repeats = int(bool(category) and category == previous_category) + int(bool(source) and source == previous_source)
|
||||
repeat_penalty = (
|
||||
(recent_categories.count(category) + selected_categories.count(category)) * self.category_repeat_penalty
|
||||
+ (recent_sources.count(source) + selected_sources.count(source)) * self.source_repeat_penalty
|
||||
)
|
||||
rank = (immediate_repeats, repeat_penalty, idx)
|
||||
if best_rank is None or rank < best_rank:
|
||||
best_idx = idx
|
||||
best_rank = rank
|
||||
selected.append(pool.pop(best_idx))
|
||||
return selected
|
||||
|
||||
async def load_media(self, raw_post_id: int) -> list[dict[str, Any]]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT *
|
||||
FROM raw_post_media
|
||||
WHERE raw_post_id=$1
|
||||
AND COALESCE(editor_hidden, FALSE)=FALSE
|
||||
AND media_type IN ('photo', 'video')
|
||||
ORDER BY sort_order ASC, id ASC
|
||||
""",
|
||||
raw_post_id,
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def media_value(self, item: dict[str, Any]) -> Any | None:
|
||||
stored = item.get("tg_file_id") or item.get("storage_attachment_id")
|
||||
if stored:
|
||||
return str(stored)
|
||||
url = str(item.get("original_url") or "").strip()
|
||||
path = local_upload_path(url)
|
||||
if path:
|
||||
return FSInputFile(path)
|
||||
return None
|
||||
|
||||
def build_text(self, post: dict[str, Any]) -> str:
|
||||
category_tag = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or post.get("final_category") or post.get("rewrite_category") or "")
|
||||
source_tag = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "")
|
||||
return build_publication_text(post.get("final_text") or post.get("rewritten_text") or "", category_tag, source_tag)
|
||||
|
||||
async def send_text(self, text: str) -> list[int]:
|
||||
message_ids: list[int] = []
|
||||
for chunk in split_message_chunks(text, self.message_limit):
|
||||
message = await self.tg_retry(lambda text=chunk: self.bot.send_message(text=text, **self.chat_kwargs()))
|
||||
message_ids.append(int(message.message_id))
|
||||
if self.send_delay_sec:
|
||||
await asyncio.sleep(self.send_delay_sec)
|
||||
return message_ids
|
||||
|
||||
async def send_media_items(self, items: list[dict[str, Any]], caption: str | None = None) -> list[int]:
|
||||
if not items:
|
||||
return []
|
||||
if len(items) == 1:
|
||||
item = items[0]
|
||||
if item["type"] == "photo":
|
||||
message = await self.tg_retry(
|
||||
lambda: self.bot.send_photo(photo=item["media"], caption=caption, **self.chat_kwargs())
|
||||
)
|
||||
else:
|
||||
message = await self.tg_retry(
|
||||
lambda: self.bot.send_video(video=item["media"], caption=caption, **self.chat_kwargs())
|
||||
)
|
||||
return [int(message.message_id)]
|
||||
group = []
|
||||
for idx, item in enumerate(items):
|
||||
media_caption = caption if idx == 0 else None
|
||||
if item["type"] == "photo":
|
||||
group.append(InputMediaPhoto(media=item["media"], caption=media_caption))
|
||||
else:
|
||||
group.append(InputMediaVideo(media=item["media"], caption=media_caption))
|
||||
messages = await self.tg_retry(lambda: self.bot.send_media_group(media=group, **self.chat_kwargs()))
|
||||
return [int(message.message_id) for message in messages]
|
||||
|
||||
async def send_post(self, post: dict[str, Any]) -> list[int]:
|
||||
text = self.build_text(post)
|
||||
media_rows = await self.load_media(int(post["id"]))
|
||||
media_items = []
|
||||
for item in media_rows:
|
||||
media = self.media_value(item)
|
||||
if media is None:
|
||||
continue
|
||||
media_type = str(item.get("media_type") or "")
|
||||
if media_type == "photo":
|
||||
media_items.append({"type": "photo", "media": media})
|
||||
elif media_type == "video":
|
||||
media_items.append({"type": "video", "media": media})
|
||||
message_ids: list[int] = []
|
||||
if not media_items:
|
||||
return await self.send_text(text)
|
||||
|
||||
first_caption = text if len(text) <= self.caption_limit else self.overflow_caption[: self.caption_limit]
|
||||
first = media_items[: self.media_group_max_items]
|
||||
message_ids.extend(await self.send_media_items(first, first_caption))
|
||||
if self.send_delay_sec:
|
||||
await asyncio.sleep(self.send_delay_sec)
|
||||
|
||||
rest = media_items[self.media_group_max_items :]
|
||||
for start in range(0, len(rest), self.media_group_max_items):
|
||||
chunk = rest[start : start + self.media_group_max_items]
|
||||
message_ids.extend(await self.send_media_items(chunk))
|
||||
if self.send_delay_sec:
|
||||
await asyncio.sleep(self.send_delay_sec)
|
||||
|
||||
if len(text) > self.caption_limit:
|
||||
message_ids.extend(await self.send_text(text))
|
||||
return message_ids
|
||||
|
||||
async def mark_published(self, post_id: int, message_ids: list[int]) -> None:
|
||||
async with self.pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE raw_posts
|
||||
SET publication_status=$2,
|
||||
editorial_status='published',
|
||||
publication_error=NULL,
|
||||
published_at=NOW(),
|
||||
tg_publication_chat_id=$3,
|
||||
tg_publication_thread_id=$4,
|
||||
tg_publication_message_ids=$5,
|
||||
tg_publication_url=$6,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
post_id,
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
self.chat_id,
|
||||
self.thread_id,
|
||||
message_ids,
|
||||
tg_message_url(int(self.chat_id), message_ids[0]) if message_ids else None,
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO post_publications(raw_post_id, platform, status, target_id, external_id, url, attachments_json, published_at, updated_at)
|
||||
VALUES($1, 'tg', $2, $3, $4, $5, $6::jsonb, NOW(), NOW())
|
||||
ON CONFLICT (raw_post_id, platform) DO UPDATE
|
||||
SET status=EXCLUDED.status,
|
||||
target_id=EXCLUDED.target_id,
|
||||
external_id=EXCLUDED.external_id,
|
||||
url=EXCLUDED.url,
|
||||
error=NULL,
|
||||
attachments_json=EXCLUDED.attachments_json,
|
||||
published_at=NOW(),
|
||||
updated_at=NOW()
|
||||
""",
|
||||
post_id,
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
str(self.chat_id),
|
||||
",".join(str(mid) for mid in message_ids),
|
||||
tg_message_url(int(self.chat_id), message_ids[0]) if message_ids else None,
|
||||
json.dumps(message_ids, ensure_ascii=False),
|
||||
)
|
||||
|
||||
async def mark_failed(self, post_id: int, error: str) -> None:
|
||||
async with self.pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE raw_posts
|
||||
SET publication_status=$2,
|
||||
editorial_status='publish_failed',
|
||||
publication_error=$3,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
post_id,
|
||||
PUBLICATION_STATUS_FAILED,
|
||||
error[:1000],
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO post_publications(raw_post_id, platform, status, target_id, error, updated_at)
|
||||
VALUES($1, 'tg', $2, $3, $4, NOW())
|
||||
ON CONFLICT (raw_post_id, platform) DO UPDATE
|
||||
SET status=EXCLUDED.status,
|
||||
target_id=EXCLUDED.target_id,
|
||||
error=EXCLUDED.error,
|
||||
updated_at=NOW()
|
||||
""",
|
||||
post_id,
|
||||
PUBLICATION_STATUS_FAILED,
|
||||
str(self.chat_id),
|
||||
error[:1000],
|
||||
)
|
||||
async def process_slot(self, slot: dict[str, Any]) -> int:
|
||||
count = int(slot["count"])
|
||||
recent = await self.load_recent()
|
||||
candidates = await self.load_candidates(max(50, count * 10))
|
||||
posts = self.choose_posts(candidates, recent, count)
|
||||
published = 0
|
||||
for post in posts:
|
||||
await self.heartbeat.beat(self.pool, status="publishing", meta={"post_id": int(post["id"]), "slot": slot["id"]}, force=True)
|
||||
try:
|
||||
message_ids = await self.send_post(post)
|
||||
await self.mark_published(int(post["id"]), message_ids)
|
||||
published += 1
|
||||
logger.info("TG published raw_post={} messages={}", post["id"], message_ids)
|
||||
except Exception as exc:
|
||||
await self.mark_failed(int(post["id"]), str(exc))
|
||||
logger.exception("TG publish failed raw_post={}: {}", post["id"], exc)
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE publication_runs
|
||||
SET published_count=$2,
|
||||
status=$3,
|
||||
error=$4,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
int(slot["run_id"]),
|
||||
published,
|
||||
"done" if published == count else "partial",
|
||||
None if published == count else f"published {published} of {count}",
|
||||
)
|
||||
return published
|
||||
|
||||
async def run_once(self) -> bool:
|
||||
await self.reload_settings()
|
||||
enabled = await is_worker_enabled(self.pool, WORKER_TG_POSTER)
|
||||
app_enabled = await fetch_bool_setting("tg_poster_enabled", False)
|
||||
if not enabled or not app_enabled:
|
||||
await self.heartbeat.beat(self.pool, status="disabled", force=True)
|
||||
return False
|
||||
slots = await self.due_slots()
|
||||
if not slots:
|
||||
await self.heartbeat.beat(self.pool, status="idle")
|
||||
return False
|
||||
total = 0
|
||||
for slot in slots:
|
||||
total += await self.process_slot(slot)
|
||||
await self.heartbeat.beat(self.pool, status="idle", meta={"published": total}, force=True)
|
||||
return bool(total)
|
||||
|
||||
async def run_loop(self) -> None:
|
||||
await self.init()
|
||||
logger.info("{} started pid={}", WORKER_TG_POSTER, os.getpid())
|
||||
try:
|
||||
while True:
|
||||
await self.run_once()
|
||||
await asyncio.sleep(max(10, await fetch_int_setting("tg_poster_interval_sec", 60)))
|
||||
finally:
|
||||
await self.close()
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
logger.remove()
|
||||
logger.add(sys.stdout, level=settings.log_level)
|
||||
worker = TelegramPoster()
|
||||
await worker.run_loop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,326 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
from loguru import logger
|
||||
|
||||
from ..config import settings
|
||||
from ..constants import PUBLICATION_STATUS_PUBLISHED, WORKER_TG_REACTOR
|
||||
from ..db import fetch_bool_setting, fetch_int_setting, fetch_setting, get_pool
|
||||
from ..heartbeat import HeartbeatReporter
|
||||
from ..jobs import is_worker_enabled
|
||||
|
||||
PLATFORM_TG = "tg"
|
||||
REACTION_STATUS_PENDING = "pending"
|
||||
REACTION_STATUS_IN_PROGRESS = "in_progress"
|
||||
REACTION_STATUS_REACTED = "reacted"
|
||||
REACTION_STATUS_FAILED = "failed"
|
||||
|
||||
|
||||
def split_tokens(value: Any) -> list[str]:
|
||||
if isinstance(value, list):
|
||||
candidates = [str(item or "") for item in value]
|
||||
else:
|
||||
candidates = re.split(r"[\s,;]+", str(value or ""))
|
||||
return [token.strip() for token in candidates if token.strip()]
|
||||
|
||||
|
||||
def normalize_reactions(value: Any) -> list[str]:
|
||||
if not isinstance(value, list):
|
||||
value = []
|
||||
reactions = []
|
||||
for item in value:
|
||||
emoji = str(item or "").strip()
|
||||
if emoji:
|
||||
reactions.append(emoji)
|
||||
return reactions or ["👍"]
|
||||
|
||||
|
||||
def parse_message_ids(row: dict[str, Any]) -> list[str]:
|
||||
raw = row.get("attachments_json")
|
||||
values: list[Any] = []
|
||||
if isinstance(raw, list):
|
||||
values = raw
|
||||
elif isinstance(raw, str):
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
if isinstance(parsed, list):
|
||||
values = parsed
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
if not values and row.get("external_id"):
|
||||
values = str(row["external_id"]).split(",")
|
||||
message_ids = []
|
||||
for value in values:
|
||||
text = str(value or "").strip()
|
||||
if text:
|
||||
message_ids.append(text)
|
||||
return message_ids
|
||||
|
||||
|
||||
class TelegramReactor:
|
||||
def __init__(self) -> None:
|
||||
self.pool = None
|
||||
self.heartbeat = HeartbeatReporter(WORKER_TG_REACTOR, 30)
|
||||
self.bot_tokens: list[str] = []
|
||||
self.reactions: list[str] = ["👍"]
|
||||
self.delay_sec = 60
|
||||
self.interval_sec = 60
|
||||
self.reaction_pause_sec = 2
|
||||
self.react_all_messages = False
|
||||
self.max_attempts = 3
|
||||
self.retry_backoff_max_sec = 30
|
||||
self.since = ""
|
||||
|
||||
async def init(self) -> None:
|
||||
self.pool = await get_pool()
|
||||
await self.reload_settings()
|
||||
|
||||
async def reload_settings(self) -> None:
|
||||
tokens = split_tokens(await fetch_setting("tg_reactor_bot_tokens", ""))
|
||||
if not tokens:
|
||||
fallback = str(await fetch_setting("tg_poster_bot_token", "") or "").strip() or settings.tg_bot_token
|
||||
tokens = split_tokens(fallback)
|
||||
if not tokens:
|
||||
raise RuntimeError("tg_reactor_bot_tokens, tg_poster_bot_token and TG_BOT_TOKEN are empty")
|
||||
self.bot_tokens = tokens
|
||||
self.reactions = normalize_reactions(await fetch_setting("tg_reactor_reactions_json", ["👍"]))
|
||||
self.since = str(await fetch_setting("tg_reactor_since", "") or "").strip()
|
||||
self.delay_sec = max(0, await fetch_int_setting("tg_reactor_delay_sec", 60))
|
||||
self.interval_sec = max(10, await fetch_int_setting("tg_reactor_interval_sec", 60))
|
||||
self.reaction_pause_sec = max(0, await fetch_int_setting("tg_reactor_reaction_pause_sec", 2))
|
||||
self.react_all_messages = await fetch_bool_setting("tg_reactor_react_all_messages", False)
|
||||
self.max_attempts = max(1, await fetch_int_setting("tg_reactor_max_attempts", 3))
|
||||
self.retry_backoff_max_sec = max(1, await fetch_int_setting("tg_reactor_retry_backoff_max_sec", 30))
|
||||
logger.info(
|
||||
"TG reactor config: bots={} reactions={} since={} delay={}s pause={}s all_messages={}",
|
||||
len(self.bot_tokens),
|
||||
self.reactions,
|
||||
self.since or "-",
|
||||
self.delay_sec,
|
||||
self.reaction_pause_sec,
|
||||
self.react_all_messages,
|
||||
)
|
||||
|
||||
async def load_publications(self, limit: int = 50) -> list[dict[str, Any]]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT pp.*
|
||||
FROM post_publications pp
|
||||
WHERE pp.platform=$1
|
||||
AND pp.status=$2
|
||||
AND pp.published_at IS NOT NULL
|
||||
AND pp.published_at <= NOW() - ($3::int * INTERVAL '1 second')
|
||||
AND ($4::text = '' OR pp.published_at >= $4::timestamptz)
|
||||
ORDER BY pp.published_at DESC, pp.id DESC
|
||||
LIMIT $5::int
|
||||
""",
|
||||
PLATFORM_TG,
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
self.delay_sec,
|
||||
self.since,
|
||||
limit,
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
async def ensure_reaction_row(
|
||||
self,
|
||||
publication: dict[str, Any],
|
||||
reactor_key: str,
|
||||
reaction: str,
|
||||
message_id: str,
|
||||
) -> dict[str, Any] | None:
|
||||
row = await self.pool.fetchrow(
|
||||
"""
|
||||
INSERT INTO post_reactions(
|
||||
publication_id, raw_post_id, platform, reactor_key, reaction, target_id, message_id, status, updated_at
|
||||
)
|
||||
VALUES($1, $2, $3, $4, $5, $6, $7, $8, NOW())
|
||||
ON CONFLICT (publication_id, reactor_key, message_id) DO NOTHING
|
||||
RETURNING *
|
||||
""",
|
||||
int(publication["id"]),
|
||||
int(publication["raw_post_id"]),
|
||||
PLATFORM_TG,
|
||||
reactor_key,
|
||||
reaction,
|
||||
str(publication["target_id"]),
|
||||
message_id,
|
||||
REACTION_STATUS_PENDING,
|
||||
)
|
||||
if row:
|
||||
return dict(row)
|
||||
row = await self.pool.fetchrow(
|
||||
"""
|
||||
SELECT *
|
||||
FROM post_reactions
|
||||
WHERE publication_id=$1 AND reactor_key=$2 AND message_id=$3
|
||||
""",
|
||||
int(publication["id"]),
|
||||
reactor_key,
|
||||
message_id,
|
||||
)
|
||||
if not row or row["status"] == REACTION_STATUS_REACTED or int(row["attempts"] or 0) >= self.max_attempts:
|
||||
return None
|
||||
return dict(row)
|
||||
|
||||
async def claim_reaction(self, reaction_id: int) -> dict[str, Any] | None:
|
||||
row = await self.pool.fetchrow(
|
||||
"""
|
||||
UPDATE post_reactions
|
||||
SET status=$2,
|
||||
attempts=attempts + 1,
|
||||
error=NULL,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
AND status IN ($3, $4)
|
||||
AND attempts < $5
|
||||
RETURNING *
|
||||
""",
|
||||
reaction_id,
|
||||
REACTION_STATUS_IN_PROGRESS,
|
||||
REACTION_STATUS_PENDING,
|
||||
REACTION_STATUS_FAILED,
|
||||
self.max_attempts,
|
||||
)
|
||||
return dict(row) if row else None
|
||||
|
||||
async def set_reaction(self, token: str, chat_id: str, message_id: str, reaction: str) -> None:
|
||||
url = f"https://api.telegram.org/bot{token}/setMessageReaction"
|
||||
payload = {
|
||||
"chat_id": int(chat_id),
|
||||
"message_id": int(message_id),
|
||||
"reaction": [{"type": "emoji", "emoji": reaction}],
|
||||
"is_big": False,
|
||||
}
|
||||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=30)) as session:
|
||||
async with session.post(url, json=payload) as resp:
|
||||
data = await resp.json(content_type=None)
|
||||
if not data.get("ok"):
|
||||
description = data.get("description") or data
|
||||
retry_after = ((data.get("parameters") or {}).get("retry_after")) if isinstance(data, dict) else None
|
||||
if retry_after:
|
||||
raise RuntimeError(f"Telegram retry_after={retry_after}: {description}")
|
||||
raise RuntimeError(str(description))
|
||||
|
||||
async def mark_reacted(self, reaction_id: int) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE post_reactions
|
||||
SET status=$2,
|
||||
error=NULL,
|
||||
reacted_at=NOW(),
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
reaction_id,
|
||||
REACTION_STATUS_REACTED,
|
||||
)
|
||||
|
||||
async def mark_failed(self, reaction_id: int, error: str) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE post_reactions
|
||||
SET status=$2,
|
||||
error=$3,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
reaction_id,
|
||||
REACTION_STATUS_FAILED,
|
||||
error[:1000],
|
||||
)
|
||||
|
||||
async def process_publication(self, publication: dict[str, Any]) -> int:
|
||||
message_ids = parse_message_ids(publication)
|
||||
if not message_ids:
|
||||
return 0
|
||||
if not self.react_all_messages:
|
||||
message_ids = message_ids[:1]
|
||||
|
||||
done = 0
|
||||
for message_id in message_ids:
|
||||
for idx, token in enumerate(self.bot_tokens):
|
||||
reaction = self.reactions[idx % len(self.reactions)]
|
||||
reactor_key = f"bot-{idx + 1}:{reaction}"
|
||||
row = await self.ensure_reaction_row(publication, reactor_key, reaction, message_id)
|
||||
if not row:
|
||||
continue
|
||||
claimed = await self.claim_reaction(int(row["id"]))
|
||||
if not claimed:
|
||||
continue
|
||||
try:
|
||||
await self.set_reaction(token, str(publication["target_id"]), message_id, reaction)
|
||||
await self.mark_reacted(int(claimed["id"]))
|
||||
done += 1
|
||||
logger.info(
|
||||
"TG reacted raw_post={} publication={} message={} reactor={}",
|
||||
publication["raw_post_id"],
|
||||
publication["id"],
|
||||
message_id,
|
||||
reactor_key,
|
||||
)
|
||||
if self.reaction_pause_sec:
|
||||
await asyncio.sleep(self.reaction_pause_sec)
|
||||
except Exception as exc:
|
||||
await self.mark_failed(int(claimed["id"]), str(exc))
|
||||
logger.warning(
|
||||
"TG reaction failed raw_post={} publication={} message={} reactor={}: {}",
|
||||
publication["raw_post_id"],
|
||||
publication["id"],
|
||||
message_id,
|
||||
reactor_key,
|
||||
exc,
|
||||
)
|
||||
await asyncio.sleep(min(2 ** int(claimed["attempts"]), self.retry_backoff_max_sec))
|
||||
return done
|
||||
|
||||
async def run_once(self) -> bool:
|
||||
await self.reload_settings()
|
||||
enabled = await is_worker_enabled(self.pool, WORKER_TG_REACTOR)
|
||||
app_enabled = await fetch_bool_setting("tg_reactor_enabled", False)
|
||||
if not enabled or not app_enabled:
|
||||
await self.heartbeat.beat(self.pool, status="disabled", force=True)
|
||||
return False
|
||||
publications = await self.load_publications()
|
||||
if not publications:
|
||||
await self.heartbeat.beat(self.pool, status="idle")
|
||||
return False
|
||||
total = 0
|
||||
for publication in publications:
|
||||
await self.heartbeat.beat(
|
||||
self.pool,
|
||||
status="reacting",
|
||||
meta={"publication_id": int(publication["id"]), "raw_post_id": int(publication["raw_post_id"])},
|
||||
force=True,
|
||||
)
|
||||
total += await self.process_publication(publication)
|
||||
await self.heartbeat.beat(self.pool, status="idle", meta={"reactions": total}, force=True)
|
||||
return bool(total)
|
||||
|
||||
async def run_loop(self) -> None:
|
||||
await self.init()
|
||||
logger.info("{} started pid={}", WORKER_TG_REACTOR, os.getpid())
|
||||
while True:
|
||||
try:
|
||||
await self.run_once()
|
||||
except Exception as exc:
|
||||
logger.exception("TG reactor loop error: {}", exc)
|
||||
await asyncio.sleep(self.interval_sec)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
logger.remove()
|
||||
logger.add(sys.stdout, level=settings.log_level)
|
||||
worker = TelegramReactor()
|
||||
await worker.run_loop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,442 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, time, timedelta, timezone
|
||||
from typing import Any
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import aiohttp
|
||||
from loguru import logger
|
||||
|
||||
from ..config import settings
|
||||
from ..constants import (
|
||||
PUBLICATION_STATUS_FAILED,
|
||||
PUBLICATION_STATUS_PENDING,
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
WORKER_VK_POSTER,
|
||||
)
|
||||
from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool
|
||||
from ..heartbeat import HeartbeatReporter
|
||||
from ..jobs import is_worker_enabled
|
||||
from ..text_utils import build_publication_text
|
||||
from ..vk_api import VKAPIClient, post_vk_url
|
||||
|
||||
LOCAL_TZ = ZoneInfo("Asia/Yekaterinburg")
|
||||
VK_MESSAGE_LIMIT = 16384
|
||||
VK_MAX_ATTACHMENTS = 10
|
||||
|
||||
|
||||
def valid_time(value: str) -> bool:
|
||||
return bool(re.fullmatch(r"(?:[01]\d|2[0-3]):[0-5]\d", str(value or "").strip()))
|
||||
|
||||
|
||||
def normalize_schedule(value: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(value, list):
|
||||
value = []
|
||||
rows: list[dict[str, Any]] = []
|
||||
for idx, item in enumerate(value):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
time_value = str(item.get("time") or "").strip()
|
||||
if not valid_time(time_value):
|
||||
continue
|
||||
count = max(0, min(20, int(item.get("count") or 0)))
|
||||
if count < 1:
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"id": str(item.get("id") or f"slot-{idx + 1}").strip()[:80],
|
||||
"time": time_value,
|
||||
"count": count,
|
||||
"enabled": bool(item.get("enabled", True)),
|
||||
}
|
||||
)
|
||||
return sorted(rows, key=lambda row: row["time"])
|
||||
|
||||
|
||||
class VKPoster:
|
||||
def __init__(self) -> None:
|
||||
self.pool = None
|
||||
self.heartbeat = HeartbeatReporter(WORKER_VK_POSTER, 30)
|
||||
self.token = ""
|
||||
self.owner_id = 0
|
||||
self.from_group = True
|
||||
self.schedule: list[dict[str, Any]] = []
|
||||
self.interval_sec = 60
|
||||
self.send_delay_sec = 1.0
|
||||
self.recent_window = 20
|
||||
self.category_repeat_penalty = 3.0
|
||||
self.source_repeat_penalty = 4.0
|
||||
self.dry_run = False
|
||||
|
||||
async def init(self) -> None:
|
||||
self.pool = await get_pool()
|
||||
|
||||
def token_expires_soon(self, value: Any) -> bool:
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
return False
|
||||
try:
|
||||
expires_at = datetime.fromisoformat(raw)
|
||||
if expires_at.tzinfo is None:
|
||||
expires_at = expires_at.replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
return False
|
||||
return expires_at <= datetime.now(timezone.utc) + timedelta(minutes=15)
|
||||
|
||||
async def save_setting(self, key: str, value: Any) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE app_settings
|
||||
SET value_json=$2::jsonb,
|
||||
updated_at=NOW()
|
||||
WHERE key=$1
|
||||
""",
|
||||
key,
|
||||
json.dumps(value, ensure_ascii=False),
|
||||
)
|
||||
|
||||
async def refresh_access_token_if_needed(self) -> None:
|
||||
expires_at = await fetch_setting("vk_poster_token_expires_at", "")
|
||||
if not self.token_expires_soon(expires_at):
|
||||
return
|
||||
refresh_token = str(await fetch_setting("vk_poster_refresh_token", "") or "").strip()
|
||||
client_id = str(await fetch_setting("vk_poster_app_id", "") or "").strip()
|
||||
client_secret = str(await fetch_setting("vk_poster_client_secret", "") or "").strip()
|
||||
device_id = str(await fetch_setting("vk_poster_token_device_id", "") or "").strip()
|
||||
if not refresh_token or not client_id:
|
||||
return
|
||||
data = {
|
||||
"grant_type": "refresh_token",
|
||||
"refresh_token": refresh_token,
|
||||
"client_id": client_id,
|
||||
"device_id": device_id,
|
||||
"state": "vk-poster-refresh",
|
||||
}
|
||||
if client_secret:
|
||||
data["client_secret"] = client_secret
|
||||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=30)) as session:
|
||||
async with session.post("https://id.vk.ru/oauth2/auth", data=data) as resp:
|
||||
payload = await resp.json(content_type=None)
|
||||
if payload.get("error"):
|
||||
raise RuntimeError(f"VK token refresh failed: {payload.get('error')}: {payload.get('error_description')}")
|
||||
access_token = str(payload.get("access_token") or "").strip()
|
||||
if not access_token:
|
||||
raise RuntimeError(f"VK token refresh did not return access_token: {payload}")
|
||||
new_refresh = str(payload.get("refresh_token") or "").strip() or refresh_token
|
||||
new_device_id = str(payload.get("device_id") or "").strip() or device_id
|
||||
expires_in = payload.get("expires_in")
|
||||
expires_at_value = ""
|
||||
try:
|
||||
expires_at_value = (datetime.now(timezone.utc) + timedelta(seconds=int(expires_in))).isoformat()
|
||||
except Exception:
|
||||
pass
|
||||
await self.save_setting("vk_poster_access_token", access_token)
|
||||
await self.save_setting("vk_poster_refresh_token", new_refresh)
|
||||
await self.save_setting("vk_poster_token_device_id", new_device_id)
|
||||
await self.save_setting("vk_poster_token_expires_at", expires_at_value)
|
||||
self.token = access_token
|
||||
logger.info("VK access token refreshed; expires_at={}", expires_at_value)
|
||||
|
||||
async def reload_settings(self) -> None:
|
||||
self.token = (
|
||||
str(await fetch_setting("vk_poster_access_token", "") or "").strip()
|
||||
or settings.vk_group_access_token
|
||||
or settings.vk_access_token
|
||||
)
|
||||
await self.refresh_access_token_if_needed()
|
||||
self.owner_id = int(await fetch_int_setting("vk_poster_owner_id", settings.vk_storage_owner_id if settings.vk_storage_group_id else 0))
|
||||
self.from_group = await fetch_bool_setting("vk_poster_from_group", True)
|
||||
self.schedule = normalize_schedule(await fetch_setting("vk_poster_schedule_json", []))
|
||||
self.interval_sec = max(10, await fetch_int_setting("vk_poster_interval_sec", 60))
|
||||
self.send_delay_sec = max(0.0, await fetch_float_setting("vk_poster_send_delay_sec", 1.0))
|
||||
self.recent_window = max(1, await fetch_int_setting("vk_poster_recent_window", 20))
|
||||
self.category_repeat_penalty = max(0.0, await fetch_float_setting("vk_poster_category_repeat_penalty", 3.0))
|
||||
self.source_repeat_penalty = max(0.0, await fetch_float_setting("vk_poster_source_repeat_penalty", 4.0))
|
||||
self.dry_run = await fetch_bool_setting("vk_poster_dry_run", False)
|
||||
if not self.token:
|
||||
raise RuntimeError("vk_poster_access_token, VK_GROUP_ACCESS_TOKEN and VK_ACCESS_TOKEN are empty")
|
||||
if not self.owner_id:
|
||||
raise RuntimeError("vk_poster_owner_id is empty")
|
||||
logger.info("VK poster config: owner={} schedule={} dry_run={}", self.owner_id, self.schedule, self.dry_run)
|
||||
|
||||
async def due_slots(self) -> list[dict[str, Any]]:
|
||||
now = datetime.now(LOCAL_TZ)
|
||||
today = now.date()
|
||||
due: list[dict[str, Any]] = []
|
||||
for row in self.schedule:
|
||||
if not row.get("enabled"):
|
||||
continue
|
||||
hour, minute = [int(part) for part in str(row["time"]).split(":", 1)]
|
||||
slot_start = datetime.combine(today, time(hour=hour, minute=minute), tzinfo=LOCAL_TZ)
|
||||
slot_end = slot_start + timedelta(minutes=10)
|
||||
if not (slot_start <= now < slot_end):
|
||||
continue
|
||||
inserted = await self.pool.fetchrow(
|
||||
"""
|
||||
INSERT INTO publication_runs(poster, schedule_id, scheduled_for, scheduled_time, planned_count, status)
|
||||
VALUES($1, $2, $3, $4, $5, 'started')
|
||||
ON CONFLICT (poster, schedule_id, scheduled_for) DO NOTHING
|
||||
RETURNING id
|
||||
""",
|
||||
"vk",
|
||||
row["id"],
|
||||
today,
|
||||
row["time"],
|
||||
int(row["count"]),
|
||||
)
|
||||
if inserted:
|
||||
due.append({**row, "run_id": int(inserted["id"]), "date": today})
|
||||
return due
|
||||
|
||||
async def load_recent(self) -> list[dict[str, Any]]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT rp.final_category_tag, rp.rewrite_category_tag, rp.final_source_tag, rp.rewrite_source_tag
|
||||
FROM post_publications pp
|
||||
JOIN raw_posts rp ON rp.id=pp.raw_post_id
|
||||
WHERE pp.platform='vk'
|
||||
AND pp.status=$1
|
||||
AND pp.target_id=$2
|
||||
ORDER BY pp.published_at DESC NULLS LAST, pp.id DESC
|
||||
LIMIT $3::int
|
||||
""",
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
str(self.owner_id),
|
||||
self.recent_window,
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
async def load_candidates(self, limit: int) -> list[dict[str, Any]]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT rp.*, s.name AS source_name, s.tag AS source_tag
|
||||
FROM raw_posts rp
|
||||
JOIN sources s ON s.id=rp.source_id
|
||||
LEFT JOIN post_publications pp ON pp.raw_post_id=rp.id AND pp.platform='vk'
|
||||
WHERE rp.status='storage_ready'
|
||||
AND rp.rewrite_status='ready'
|
||||
AND COALESCE(rp.editorial_status, 'review') IN ('accepted', 'published', 'publish_failed')
|
||||
AND COALESCE(pp.status, $1) = $1
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM raw_post_media rpm
|
||||
WHERE rpm.raw_post_id=rp.id
|
||||
AND COALESCE(rpm.editor_hidden, FALSE)=FALSE
|
||||
AND COALESCE(rpm.editor_added, FALSE)=FALSE
|
||||
AND (
|
||||
(rpm.media_type='photo' AND rpm.original_attachment_id LIKE 'photo%')
|
||||
OR (
|
||||
rpm.media_type='video'
|
||||
AND rpm.original_attachment_id LIKE 'video%'
|
||||
AND COALESCE(rpm.status, '') <> 'link_only'
|
||||
)
|
||||
)
|
||||
)
|
||||
ORDER BY COALESCE(rp.reviewed_at, rp.edited_at, rp.rewritten_at, rp.created_at) ASC,
|
||||
rp.id ASC
|
||||
LIMIT $2::int
|
||||
""",
|
||||
PUBLICATION_STATUS_PENDING,
|
||||
max(limit, 50),
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def choose_posts(self, candidates: list[dict[str, Any]], recent: list[dict[str, Any]], count: int) -> list[dict[str, Any]]:
|
||||
selected: list[dict[str, Any]] = []
|
||||
recent_categories = [str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "") for row in recent]
|
||||
recent_sources = [str(row.get("final_source_tag") or row.get("rewrite_source_tag") or "") for row in recent]
|
||||
pool = list(candidates)
|
||||
while pool and len(selected) < count:
|
||||
best_idx = 0
|
||||
selected_categories = [str(row.get("final_category_tag") or row.get("rewrite_category_tag") or "") for row in selected]
|
||||
selected_sources = [str(row.get("final_source_tag") or row.get("rewrite_source_tag") or row.get("source_tag") or "") for row in selected]
|
||||
previous_category = selected_categories[-1] if selected_categories else (recent_categories[0] if recent_categories else "")
|
||||
previous_source = selected_sources[-1] if selected_sources else (recent_sources[0] if recent_sources else "")
|
||||
best_rank: tuple[int, float, int] | None = None
|
||||
for idx, post in enumerate(pool):
|
||||
category = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or "")
|
||||
source = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "")
|
||||
immediate_repeats = int(bool(category) and category == previous_category) + int(bool(source) and source == previous_source)
|
||||
repeat_penalty = (
|
||||
(recent_categories.count(category) + selected_categories.count(category)) * self.category_repeat_penalty
|
||||
+ (recent_sources.count(source) + selected_sources.count(source)) * self.source_repeat_penalty
|
||||
)
|
||||
rank = (immediate_repeats, repeat_penalty, idx)
|
||||
if best_rank is None or rank < best_rank:
|
||||
best_idx = idx
|
||||
best_rank = rank
|
||||
selected.append(pool.pop(best_idx))
|
||||
return selected
|
||||
|
||||
async def load_media(self, raw_post_id: int) -> list[dict[str, Any]]:
|
||||
rows = await self.pool.fetch(
|
||||
"""
|
||||
SELECT *
|
||||
FROM raw_post_media
|
||||
WHERE raw_post_id=$1
|
||||
AND COALESCE(editor_hidden, FALSE)=FALSE
|
||||
AND media_type IN ('photo', 'video')
|
||||
ORDER BY sort_order ASC, id ASC
|
||||
LIMIT $2::int
|
||||
""",
|
||||
raw_post_id,
|
||||
VK_MAX_ATTACHMENTS,
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def build_text(self, post: dict[str, Any]) -> str:
|
||||
category_tag = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or post.get("final_category") or post.get("rewrite_category") or "")
|
||||
source_tag = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "")
|
||||
return build_publication_text(post.get("final_text") or post.get("rewritten_text") or "", category_tag, source_tag)[:VK_MESSAGE_LIMIT]
|
||||
|
||||
async def media_attachment(self, client: VKAPIClient, item: dict[str, Any]) -> str | None:
|
||||
attachment_id = str(item.get("original_attachment_id") or "").strip()
|
||||
media_type = str(item.get("media_type") or "").strip()
|
||||
if media_type == "video":
|
||||
if str(item.get("status") or "").strip() == "link_only":
|
||||
return None
|
||||
return attachment_id if attachment_id.startswith("video") else None
|
||||
if media_type != "photo":
|
||||
return None
|
||||
if attachment_id.startswith("photo") and not item.get("editor_added"):
|
||||
return attachment_id
|
||||
return None
|
||||
|
||||
async def send_post(self, post: dict[str, Any]) -> tuple[int, str, list[str]]:
|
||||
text = self.build_text(post)
|
||||
media_rows = await self.load_media(int(post["id"]))
|
||||
async with VKAPIClient(
|
||||
token=self.token,
|
||||
rps=2,
|
||||
timeout_total_sec=max(30, await fetch_int_setting("vk_poster_timeout_sec", 90)),
|
||||
retry_attempts=max(1, await fetch_int_setting("vk_poster_max_attempts", 3)),
|
||||
retry_min_delay_sec=2,
|
||||
retry_max_delay_sec=max(2, await fetch_int_setting("vk_poster_retry_backoff_max_sec", 30)),
|
||||
) as client:
|
||||
attachments: list[str] = []
|
||||
for item in media_rows:
|
||||
try:
|
||||
attachment = await self.media_attachment(client, item)
|
||||
except Exception as exc:
|
||||
logger.warning("VK media skipped raw_post={} media_id={}: {}", post["id"], item.get("id"), exc)
|
||||
continue
|
||||
if attachment:
|
||||
attachments.append(attachment)
|
||||
if len(attachments) >= VK_MAX_ATTACHMENTS:
|
||||
break
|
||||
if self.dry_run:
|
||||
return 0, "", attachments
|
||||
post_id = await client.create_wall_post(self.owner_id, text, attachments, from_group=self.from_group)
|
||||
return post_id, post_vk_url(self.owner_id, post_id), attachments
|
||||
|
||||
async def upsert_publication(
|
||||
self,
|
||||
post_id: int,
|
||||
status: str,
|
||||
external_id: str | None = None,
|
||||
url: str | None = None,
|
||||
error: str | None = None,
|
||||
attachments: list[str] | None = None,
|
||||
) -> None:
|
||||
await self.pool.execute(
|
||||
"""
|
||||
INSERT INTO post_publications(
|
||||
raw_post_id, platform, status, target_id, external_id, url, error, attachments_json, published_at, updated_at
|
||||
)
|
||||
VALUES($1, 'vk', $2, $3, $4, $5, $6, $7::jsonb, CASE WHEN $2=$8 THEN NOW() ELSE NULL END, NOW())
|
||||
ON CONFLICT (raw_post_id, platform) DO UPDATE
|
||||
SET status=EXCLUDED.status,
|
||||
target_id=EXCLUDED.target_id,
|
||||
external_id=EXCLUDED.external_id,
|
||||
url=EXCLUDED.url,
|
||||
error=EXCLUDED.error,
|
||||
attachments_json=EXCLUDED.attachments_json,
|
||||
published_at=CASE WHEN EXCLUDED.status=$8 THEN COALESCE(EXCLUDED.published_at, NOW()) ELSE post_publications.published_at END,
|
||||
updated_at=NOW()
|
||||
""",
|
||||
post_id,
|
||||
status,
|
||||
str(self.owner_id),
|
||||
external_id,
|
||||
url,
|
||||
error[:1000] if error else None,
|
||||
json.dumps(attachments or [], ensure_ascii=False),
|
||||
PUBLICATION_STATUS_PUBLISHED,
|
||||
)
|
||||
|
||||
async def process_slot(self, slot: dict[str, Any]) -> int:
|
||||
count = int(slot["count"])
|
||||
recent = await self.load_recent()
|
||||
candidates = await self.load_candidates(max(50, count * 10))
|
||||
posts = self.choose_posts(candidates, recent, count)
|
||||
published = 0
|
||||
for post in posts:
|
||||
await self.heartbeat.beat(self.pool, status="publishing", meta={"post_id": int(post["id"]), "slot": slot["id"]}, force=True)
|
||||
try:
|
||||
vk_post_id, url, attachments = await self.send_post(post)
|
||||
await self.upsert_publication(int(post["id"]), PUBLICATION_STATUS_PUBLISHED, str(vk_post_id), url, attachments=attachments)
|
||||
published += 1
|
||||
logger.info("VK published raw_post={} post_id={} attachments={}", post["id"], vk_post_id, len(attachments))
|
||||
except Exception as exc:
|
||||
await self.upsert_publication(int(post["id"]), PUBLICATION_STATUS_FAILED, error=str(exc))
|
||||
logger.exception("VK publish failed raw_post={}: {}", post["id"], exc)
|
||||
if self.send_delay_sec:
|
||||
await asyncio.sleep(self.send_delay_sec)
|
||||
await self.pool.execute(
|
||||
"""
|
||||
UPDATE publication_runs
|
||||
SET published_count=$2,
|
||||
status=$3,
|
||||
error=$4,
|
||||
updated_at=NOW()
|
||||
WHERE id=$1
|
||||
""",
|
||||
int(slot["run_id"]),
|
||||
published,
|
||||
"done" if published == count else "partial",
|
||||
None if published == count else f"published {published} of {count}",
|
||||
)
|
||||
return published
|
||||
|
||||
async def run_once(self) -> bool:
|
||||
await self.reload_settings()
|
||||
enabled = await is_worker_enabled(self.pool, WORKER_VK_POSTER)
|
||||
app_enabled = await fetch_bool_setting("vk_poster_enabled", False)
|
||||
if not enabled or not app_enabled:
|
||||
await self.heartbeat.beat(self.pool, status="disabled", force=True)
|
||||
return False
|
||||
slots = await self.due_slots()
|
||||
if not slots:
|
||||
await self.heartbeat.beat(self.pool, status="idle")
|
||||
return False
|
||||
total = 0
|
||||
for slot in slots:
|
||||
total += await self.process_slot(slot)
|
||||
await self.heartbeat.beat(self.pool, status="idle", meta={"published": total}, force=True)
|
||||
return bool(total)
|
||||
|
||||
async def run_loop(self) -> None:
|
||||
await self.init()
|
||||
logger.info("{} started pid={}", WORKER_VK_POSTER, os.getpid())
|
||||
while True:
|
||||
try:
|
||||
await self.run_once()
|
||||
except Exception as exc:
|
||||
logger.exception("VK poster loop error: {}", exc)
|
||||
await asyncio.sleep(self.interval_sec)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
logger.remove()
|
||||
logger.add(sys.stdout, level=settings.log_level)
|
||||
worker = VKPoster()
|
||||
await worker.run_loop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -91,6 +91,7 @@ class TelegramStorageUploader:
|
||||
self.text_overflow_marker = "Продолжение следующим сообщением."
|
||||
self.download_timeout_sec = 45
|
||||
self.video_max_size_mb = 49
|
||||
self.video_max_height = 720
|
||||
self.video_max_duration_sec = 300
|
||||
self.yt_dlp_timeout_sec = 300
|
||||
self.max_media_attempts = 3
|
||||
@@ -126,6 +127,7 @@ class TelegramStorageUploader:
|
||||
self.text_overflow_marker = str(await fetch_setting("telegram_text_overflow_marker", self.text_overflow_marker))
|
||||
self.download_timeout_sec = max(5, await fetch_int_setting("uploader_download_timeout_sec", 45))
|
||||
self.video_max_size_mb = max(1, await fetch_int_setting("video_max_size_mb", 49))
|
||||
self.video_max_height = max(144, await fetch_int_setting("video_max_height", 720))
|
||||
self.video_max_duration_sec = max(1, await fetch_int_setting("video_max_duration_sec", 300))
|
||||
self.yt_dlp_timeout_sec = max(30, await fetch_int_setting("uploader_yt_dlp_timeout_sec", 300))
|
||||
self.max_media_attempts = max(1, await fetch_int_setting("uploader_max_media_attempts", 3))
|
||||
@@ -192,6 +194,64 @@ class TelegramStorageUploader:
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def media_blocking_error(self, media: list[dict], prepared_media_ids: set[int] | None = None) -> str | None:
|
||||
prepared_media_ids = prepared_media_ids or set()
|
||||
visible_media = [
|
||||
item
|
||||
for item in media
|
||||
if str(item.get("media_type") or "") in ("photo", "video")
|
||||
and not bool(item.get("editor_hidden"))
|
||||
]
|
||||
if not visible_media:
|
||||
return "no publishable media"
|
||||
pending = [
|
||||
item
|
||||
for item in visible_media
|
||||
if str(item.get("status") or "") == "pending"
|
||||
and int(item["id"]) not in prepared_media_ids
|
||||
]
|
||||
if pending:
|
||||
details = ", ".join(
|
||||
f"{item.get('media_type')}#{item.get('id')}: {item.get('error') or 'pending'}"
|
||||
for item in pending[:5]
|
||||
)
|
||||
return f"media still pending: {details}"
|
||||
failed = [
|
||||
item
|
||||
for item in visible_media
|
||||
if str(item.get("status") or "") == MEDIA_STATUS_FAILED
|
||||
]
|
||||
if failed:
|
||||
details = ", ".join(
|
||||
f"{item.get('media_type')}#{item.get('id')}: {item.get('error') or 'failed'}"
|
||||
for item in failed[:5]
|
||||
)
|
||||
return f"media failed: {details}"
|
||||
link_only = [
|
||||
item
|
||||
for item in visible_media
|
||||
if str(item.get("status") or "") == MEDIA_STATUS_LINK_ONLY
|
||||
]
|
||||
if link_only:
|
||||
details = ", ".join(
|
||||
f"{item.get('media_type')}#{item.get('id')}: {item.get('error') or 'link only'}"
|
||||
for item in link_only[:5]
|
||||
)
|
||||
return f"media link only: {details}"
|
||||
missing_upload = [
|
||||
item
|
||||
for item in visible_media
|
||||
if not (item.get("tg_file_id") or item.get("storage_attachment_id"))
|
||||
and int(item["id"]) not in prepared_media_ids
|
||||
]
|
||||
if missing_upload:
|
||||
details = ", ".join(
|
||||
f"{item.get('media_type')}#{item.get('id')}: missing tg_file_id"
|
||||
for item in missing_upload[:5]
|
||||
)
|
||||
return f"media not uploaded: {details}"
|
||||
return None
|
||||
|
||||
async def download_bytes(self, session: aiohttp.ClientSession, url: str) -> bytes | None:
|
||||
try:
|
||||
async with session.get(url, timeout=aiohttp.ClientTimeout(total=self.download_timeout_sec)) as response:
|
||||
@@ -271,7 +331,13 @@ class TelegramStorageUploader:
|
||||
output_path,
|
||||
"--no-playlist",
|
||||
"-f",
|
||||
f"best[filesize<{max_size}]/bestvideo[filesize<{max_size}]+bestaudio/best",
|
||||
(
|
||||
f"best[height<={self.video_max_height}][filesize<{max_size}]"
|
||||
f"/best[height<={self.video_max_height}]"
|
||||
f"/bestvideo[height<={self.video_max_height}][filesize<{max_size}]+bestaudio/best"
|
||||
f"/bestvideo[height<={self.video_max_height}]+bestaudio/best"
|
||||
f"/best[filesize<{max_size}]"
|
||||
),
|
||||
"--quiet",
|
||||
"--no-warnings",
|
||||
]
|
||||
@@ -498,6 +564,25 @@ class TelegramStorageUploader:
|
||||
media = await self.load_media(raw_post_id)
|
||||
prepared = await self.prepare_media(raw_post_id, media)
|
||||
media = await self.load_media(raw_post_id)
|
||||
prepared_media_ids = {int(item["media_id"]) for item in prepared}
|
||||
blocking_error = self.media_blocking_error(media, prepared_media_ids)
|
||||
if blocking_error:
|
||||
for item in prepared:
|
||||
tmp_path = item.get("tmp_path")
|
||||
if tmp_path:
|
||||
try:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
except Exception:
|
||||
pass
|
||||
if not blocking_error.startswith("media still pending:"):
|
||||
await self.mark_post_failed(raw_post_id, blocking_error)
|
||||
await ack_done(self.pool, job_id)
|
||||
logger.warning("Telegram storage blocked raw_post={}: {}", raw_post_id, blocking_error)
|
||||
return
|
||||
await ack_retry(self.pool, job, blocking_error, delay_sec=120)
|
||||
logger.warning("Telegram storage retry raw_post={}: {}", raw_post_id, blocking_error)
|
||||
return
|
||||
try:
|
||||
message_ids, meta_message_id = await self.send_storage_post(post, prepared, media)
|
||||
finally:
|
||||
|
||||
Reference in New Issue
Block a user