exostring/new_vk_parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add project common publication tags

Browse Source
This commit is contained in:
Your Name
2026-08-03 21:43:56 +05:00
parent e8445c811e
commit 8e0b390e6c
10 changed files with 143 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
MULTI_PROJECT.md
+13
View File
@@ -31,6 +31,19 @@ Goal: keep one codebase and run separate deployments for each editorial project.
Sources, target groups/channels, poster tokens, schedules, prompts, categories, and enabled workers should stay in `app_settings`, `sources`, `content_categories`, and `worker_controls`.
Common publication hashtags are also per-project database settings:
- key: `publication_common_tags`
- FN-8: `#новости_снаряжения`
- RAA: `#raa_news #страйкбол`
- The app posts these tags before category/source tags in TG/VK/MAX social posters.
RAA current social targets:
- raw media Telegram channel: `tg_media_channel_id=-1004312015562`
- final Telegram publication channel: `tg_poster_chat_id=-1001303630155`
- VK owner: `vk_poster_owner_id=-36860851`, `vk_poster_from_group=true`
## Poster modules
The site poster is selected by `site_poster_provider`. A project can disable the worker or leave the provider empty when there is no website poster yet. New website integrations should be added as provider modules behind the same interface instead of branching the worker by project name.
STATE.md
+22 -1
View File
@@ -48,6 +48,25 @@ Current runtime is local infrastructure, not the old VPS.
- New installs default `site-poster` to disabled and must explicitly set provider,
base URL, secret, and `site_poster_enabled=true`.
## RAA Duplicate
As of 2026-08-03, the RAA duplicate is a separate Coolify app/database using the
same codebase shape.
- Local copy path requested by user: `D:\DEVELOPMENT\raa-parser_poster`.
- Gitea repo: `http://192.168.1.135:3000/exostring/raa-parser_poster.git`.
- Coolify app UUID: `korokrhpoyqxf0nw2y8vk7lp` (`RAA admin`).
- Public admin URL: `https://raa.panel.f-n8.ru`.
- RAA Postgres container: `m9v14hzfh5mk9kelxzcqvhc6`, DB `raa_parser`, user `raa_parser_user`.
- Raw Telegram media channel setting: `tg_media_channel_id=-1004312015562` (`VK RAW RAA`).
- Final Telegram publication chat setting: `tg_poster_chat_id=-1001303630155` (`Red Airsoft | Страйкбол`).
- VK publication owner: `vk_poster_owner_id=-36860851`, `vk_poster_from_group=true`.
- `TG_BOT_TOKEN`, `TELEGRAM_API_ID`, and `TELEGRAM_API_HASH` were copied from FN-8 Coolify env.
- Local Bot API app setting: `local_bot_api_url=http://127.0.0.1:8081`.
- As last checked, publication workers were intentionally disabled:
`vk-storage-uploader=false`, `tg-poster=false`, `vk-poster=false`,
`tg_poster_enabled=false`, `vk_poster_enabled=false`.
## Important Text Rules
- AI writer output text must be clean: no physical hashtags at the end.
@@ -58,7 +77,9 @@ Current runtime is local infrastructure, not the old VPS.
- Publication preview may show hashtags by composing:
- clean text
- blank line
- `#category #source`
- optional project-wide common tags first, then `#category #source`
- Project-wide common tags live in DB setting `publication_common_tags`.
FN-8 value: `#новости_снаряжения`. RAA value: `#raa_news #страйкбол`.
- The editor textarea must show only clean editable text.
- The list preview in `/editor` should show the publication preview, including composed hashtags.
- `build_publication_text()` in `src/vk_parser_app/text_utils.py` is display/composition helper only. Do not use it before writing `final_text` to DB.
db/migrations/041_publication_common_tags_and_category_descriptions.sql
+13
View File
@@ -0,0 +1,13 @@
ALTER TABLE content_categories
ADD COLUMN IF NOT EXISTS description TEXT NOT NULL DEFAULT '';
INSERT INTO app_settings(key, value_json, value_type, title, description, category)
VALUES (
'publication_common_tags',
'""'::jsonb,
'text',
'Общие хэштеги публикаций',
'Один или несколько хэштегов, которые добавляются первыми в публикациях соцсетей. Можно писать через пробел, запятую или с новой строки.',
'Publishing'
)
ON CONFLICT (key) DO NOTHING;
src/vk_parser_app/admin.py
+30 -13
View File
@@ -1325,13 +1325,21 @@ async def writer_category_payload() -> list[dict[str, Any]]:
pool = await get_pool()
rows = await pool.fetch(
"""
SELECT sort_order AS id, name, tag
SELECT sort_order AS id, name, tag, COALESCE(description, '') AS description
FROM content_categories
WHERE is_active=TRUE
ORDER BY sort_order, name
"""
)
categories = [{"id": int(row["id"]), "name": str(row["name"]), "tag": str(row["tag"])} for row in rows]
categories = [
{
"id": int(row["id"]),
"name": str(row["name"]),
"tag": str(row["tag"]),
"description": str(row["description"] or ""),
}
for row in rows
]
if categories:
return categories
return [
@@ -1344,7 +1352,8 @@ async def category_rows() -> list[dict[str, Any]]:
pool = await get_pool()
rows = await pool.fetch(
"""
SELECT id, name, tag, site_name, site_slug, site_enabled,
SELECT id, name, tag, COALESCE(description, '') AS description,
site_name, site_slug, site_enabled,
is_active, sort_order, created_at, updated_at
FROM content_categories
ORDER BY is_active DESC, sort_order, name
@@ -3630,6 +3639,7 @@ async def category_create(
csrf_token: str = Form(...),
name: str = Form(...),
tag: str = Form(""),
description: str = Form(""),
site_name: str = Form(...),
site_slug: str = Form(...),
site_enabled: str | None = Form(None),
@@ -3642,6 +3652,7 @@ async def category_create(
if not name:
return redirect("/workers")
tag = normalize_hash_tag(tag or name, "category")
description = description.strip()
site_name = site_name.strip()
site_slug = re.sub(r"[^a-z0-9-]+", "-", site_slug.strip().lower().replace("_", "-")).strip("-")
if not site_name or not site_slug:
@@ -3650,24 +3661,26 @@ async def category_create(
sort_order = int(await pool.fetchval("SELECT COALESCE(MAX(sort_order), 0) + 1 FROM content_categories") or 1)
await pool.execute(
"""
INSERT INTO content_categories(name, tag, sort_order, site_name, site_slug, site_enabled)
VALUES($1, $2, $3, $4, $5, $6)
INSERT INTO content_categories(name, tag, sort_order, description, site_name, site_slug, site_enabled)
VALUES($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (name) DO UPDATE
SET tag=$2,
site_name=$4,
site_slug=$5,
site_enabled=$6,
description=$4,
site_name=$5,
site_slug=$6,
site_enabled=$7,
is_active=TRUE,
updated_at=NOW()
""",
name,
tag,
sort_order,
description,
site_name,
site_slug,
site_enabled is not None,
)
await audit(user["id"], "category.create", "content_category", None, {"name": name, "tag": tag, "site_name": site_name, "site_slug": site_slug, "site_enabled": site_enabled is not None})
await audit(user["id"], "category.create", "content_category", None, {"name": name, "tag": tag, "description": description, "site_name": site_name, "site_slug": site_slug, "site_enabled": site_enabled is not None})
return redirect("/workers")
@@ -3678,6 +3691,7 @@ async def category_update(
csrf_token: str = Form(...),
name: str = Form(...),
tag: str = Form(""),
description: str = Form(""),
site_name: str = Form(...),
site_slug: str = Form(...),
site_enabled: str | None = Form(None),
@@ -3690,6 +3704,7 @@ async def category_update(
if not name:
return redirect("/workers")
tag = normalize_hash_tag(tag or name, "category")
description = description.strip()
site_name = site_name.strip()
site_slug = re.sub(r"[^a-z0-9-]+", "-", site_slug.strip().lower().replace("_", "-")).strip("-")
if not site_name or not site_slug:
@@ -3700,20 +3715,22 @@ async def category_update(
UPDATE content_categories
SET name=$2,
tag=$3,
site_name=$4,
site_slug=$5,
site_enabled=$6,
description=$4,
site_name=$5,
site_slug=$6,
site_enabled=$7,
updated_at=NOW()
WHERE id=$1
""",
category_id,
name,
tag,
description,
site_name,
site_slug,
site_enabled is not None,
)
await audit(user["id"], "category.update", "content_category", category_id, {"name": name, "tag": tag, "site_name": site_name, "site_slug": site_slug, "site_enabled": site_enabled is not None})
await audit(user["id"], "category.update", "content_category", category_id, {"name": name, "tag": tag, "description": description, "site_name": site_name, "site_slug": site_slug, "site_enabled": site_enabled is not None})
return redirect("/workers")
src/vk_parser_app/templates/workers.html
+6 -1
View File
@@ -73,7 +73,7 @@
</summary>
<div class="p-4 flex flex-col gap-6 bg-app-bg/30">
<div class="text-sm text-app-textMuted">
Название описывает категорию для AI, тэг используется в соцсетях.
Название и описание помогают AI выбрать категорию, тэг используется в соцсетях.
</div>
<form method="post" action="/categories/create" class="flex flex-col gap-4 bg-app-surface p-4 rounded-xl border border-app-border">
@@ -88,6 +88,10 @@
<label class="block text-[10px] uppercase font-bold text-app-textMuted mb-1">Тэг (для хэштегов)</label>
<input name="tag" class="input w-full">
</div>
<div class="md:col-span-2">
<label class="block text-[10px] uppercase font-bold text-app-textMuted mb-1">Описание (для AI)</label>
<textarea name="description" class="textarea w-full min-h-20"></textarea>
</div>
<div>
<label class="block text-[10px] uppercase font-bold text-app-textMuted mb-1">На сайте</label>
<input name="site_name" class="input w-full" required>
@@ -127,6 +131,7 @@
<div class="flex flex-col gap-1.5">
<input form="category-update-{{ c.id }}" name="name" value="{{ c.name }}" class="input input-sm w-full bg-app-bg text-white h-7">
<input form="category-update-{{ c.id }}" name="tag" value="{{ c.tag }}" class="input input-sm w-full bg-app-bg text-blue-400 font-mono h-7">
<textarea form="category-update-{{ c.id }}" name="description" class="textarea textarea-sm w-full bg-app-bg text-app-textMain min-h-16">{{ c.description }}</textarea>
</div>
</td>
<td class="px-3 py-2">
src/vk_parser_app/text_utils.py
+20 -3
View File
@@ -23,8 +23,24 @@ def strip_trailing_hashtag_line(text: str) -> str:
return "\n".join(lines).rstrip()
def publication_hashtags(category_tag: str, source_tag: str) -> str:
def parse_hash_tags(value: object) -> list[str]:
if isinstance(value, list):
raw_items = [str(item) for item in value]
else:
raw_items = re.split(r"[\s,|]+", str(value or ""))
tags: list[str] = []
seen: set[str] = set()
for item in raw_items:
tag = normalize_hash_tag(item, "")
if tag and tag not in seen:
tags.append(tag)
seen.add(tag)
return tags
def publication_hashtags(category_tag: str, source_tag: str, common_tags: object = "") -> str:
tags = [
*parse_hash_tags(common_tags),
normalize_hash_tag(category_tag, "category"),
normalize_hash_tag(source_tag, "source"),
]
@@ -38,6 +54,7 @@ def build_publication_text(
text: str,
category_tag: str,
source_tag: str,
common_tags: object = "",
format_title: bool = False,
parse_mode: str | None = None,
) -> str:
@@ -59,7 +76,7 @@ def build_publication_text(
break
if title_idx is None:
return publication_hashtags(category_tag, source_tag)
return publication_hashtags(category_tag, source_tag, common_tags)
formatted_lines = []
for idx, line in enumerate(cleaned_lines):
@@ -84,7 +101,7 @@ def build_publication_text(
raw_body = "\n".join(formatted_lines)
# Collapse multiple consecutive blank lines to at most two newlines (\n\n)
normalized_body = re.sub(r"\n{3,}", "\n\n", raw_body).strip()
hashtags = publication_hashtags(category_tag, source_tag)
hashtags = publication_hashtags(category_tag, source_tag, common_tags)
if hashtags:
return f"{normalized_body}\n\n{hashtags}" if normalized_body else hashtags
return normalized_body
src/vk_parser_app/workers/ai_writer.py
+13 -5
View File
@@ -16,7 +16,7 @@ from ..constants import WORKER_AI_WRITER
from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool
from ..heartbeat import HeartbeatReporter
from ..jobs import is_worker_enabled
from ..text_utils import build_publication_text, normalize_hash_tag, parse_categories
from ..text_utils import normalize_hash_tag, parse_categories, strip_trailing_hashtag_line
from .ai_alerts import send_ai_worker_error_alert
@@ -53,7 +53,7 @@ OUTPUT SCHEMA (return array matching input order):
Rules:
- Input is a JSON object with key "posts" containing accepted posts.
- Input has key "categories" with objects: id, name, tag.
- Input has key "categories" with objects: id, name, tag, description.
- Each post includes producer_name and producer_tag. producer_name can be a manufacturer, shop, or publishing source.
- The "text" field for every rewrite must explicitly mention producer_name at least once. Do not satisfy this only through producer_tag, hashtags, notes, or metadata.
- Pick exactly one category from the categories list in input and return its numeric id as category_id.
@@ -208,7 +208,7 @@ async def load_writer_categories(pool) -> list[dict[str, Any]]:
try:
rows = await pool.fetch(
"""
SELECT sort_order AS id, name, tag
SELECT sort_order AS id, name, tag, COALESCE(description, '') AS description
FROM content_categories
WHERE is_active=TRUE
ORDER BY sort_order, name
@@ -216,7 +216,15 @@ async def load_writer_categories(pool) -> list[dict[str, Any]]:
)
except Exception:
rows = []
categories = [{"id": int(row["id"]), "name": str(row["name"]), "tag": str(row["tag"])} for row in rows]
categories = [
{
"id": int(row["id"]),
"name": str(row["name"]),
"tag": str(row["tag"]),
"description": str(row["description"] or ""),
}
for row in rows
]
if categories:
return categories
legacy = parse_categories(await fetch_setting("ai_writer_categories", []))
@@ -399,7 +407,7 @@ class AIWriterWorker:
async def apply_rewrites(self, batch_id: int, rewrites: list[dict], model: str, prompt: str) -> None:
for item in rewrites:
final_text = build_publication_text(item["text"], item["category_tag"], item["source_tag"])
final_text = strip_trailing_hashtag_line(item["text"])
editorial_status = "rejected" if item.get("reject_by_category") else "review"
editor_notes = "Отклонено AI: Не целевой контент" if item.get("reject_by_category") else None
await self.pool.execute(
src/vk_parser_app/workers/max_poster.py
+8 -1
View File
@@ -218,6 +218,7 @@ class MAXPoster:
self.auto_reaction = "👍"
self.reaction_path_template = "/messages/{message_id}/reactions"
self.dry_run = False
self.common_tags = ""
async def init(self) -> None:
self.pool = await get_pool()
@@ -243,6 +244,7 @@ class MAXPoster:
self.auto_reaction = str(await fetch_setting("max_poster_auto_reaction", "👍") or "👍").strip() or "👍"
self.reaction_path_template = str(await fetch_setting("max_poster_reaction_path_template", self.reaction_path_template) or "").strip()
self.dry_run = await fetch_bool_setting("max_poster_dry_run", False)
self.common_tags = str(await fetch_setting("publication_common_tags", "") or "").strip()
if not self.token:
raise RuntimeError("max_poster_bot_token is empty")
if not self.chat_id:
@@ -371,7 +373,12 @@ class MAXPoster:
def build_text(self, post: dict[str, Any]) -> str:
category_tag = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or post.get("final_category") or post.get("rewrite_category") or "")
source_tag = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "")
return build_publication_text(post.get("final_text") or post.get("rewritten_text") or "", category_tag, source_tag)
return build_publication_text(
post.get("final_text") or post.get("rewritten_text") or "",
category_tag,
source_tag,
self.common_tags,
)
async def download_media_to_temp(self, session: aiohttp.ClientSession, item: dict[str, Any]) -> Path | None:
media_type = str(item.get("media_type") or "")
src/vk_parser_app/workers/tg_poster.py
+10 -1
View File
@@ -111,6 +111,7 @@ class TelegramPoster:
self.recent_window = 20
self.category_repeat_penalty = 3.0
self.source_repeat_penalty = 4.0
self.common_tags = ""
async def init(self) -> None:
self.pool = await get_pool()
@@ -137,6 +138,7 @@ class TelegramPoster:
self.recent_window = max(1, await fetch_int_setting("tg_poster_recent_window", 20))
self.category_repeat_penalty = max(0.0, await fetch_float_setting("tg_poster_category_repeat_penalty", 3.0))
self.source_repeat_penalty = max(0.0, await fetch_float_setting("tg_poster_source_repeat_penalty", 4.0))
self.common_tags = str(await fetch_setting("publication_common_tags", "") or "").strip()
logger.info("TG poster config: chat={} schedule={} caption_limit={}", self.chat_id, self.schedule, self.caption_limit)
async def close(self) -> None:
@@ -325,7 +327,14 @@ class TelegramPoster:
def build_text(self, post: dict[str, Any]) -> str:
category_tag = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or post.get("final_category") or post.get("rewrite_category") or "")
source_tag = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "")
return build_publication_text(post.get("final_text") or post.get("rewritten_text") or "", category_tag, source_tag, format_title=True, parse_mode="html")
return build_publication_text(
post.get("final_text") or post.get("rewritten_text") or "",
category_tag,
source_tag,
self.common_tags,
format_title=True,
parse_mode="html",
)
async def send_text(self, text: str) -> list[int]:
message_ids: list[int] = []
src/vk_parser_app/workers/vk_poster.py
+8 -1
View File
@@ -72,6 +72,7 @@ class VKPoster:
self.category_repeat_penalty = 3.0
self.source_repeat_penalty = 4.0
self.dry_run = False
self.common_tags = ""
async def init(self) -> None:
self.pool = await get_pool()
@@ -158,6 +159,7 @@ class VKPoster:
self.category_repeat_penalty = max(0.0, await fetch_float_setting("vk_poster_category_repeat_penalty", 3.0))
self.source_repeat_penalty = max(0.0, await fetch_float_setting("vk_poster_source_repeat_penalty", 4.0))
self.dry_run = await fetch_bool_setting("vk_poster_dry_run", False)
self.common_tags = str(await fetch_setting("publication_common_tags", "") or "").strip()
if not self.token:
raise RuntimeError("vk_poster_access_token, VK_GROUP_ACCESS_TOKEN and VK_ACCESS_TOKEN are empty")
if not self.owner_id:
@@ -292,7 +294,12 @@ class VKPoster:
def build_text(self, post: dict[str, Any]) -> str:
category_tag = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or post.get("final_category") or post.get("rewrite_category") or "")
source_tag = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "")
return build_publication_text(post.get("final_text") or post.get("rewritten_text") or "", category_tag, source_tag)[:VK_MESSAGE_LIMIT]
return build_publication_text(
post.get("final_text") or post.get("rewritten_text") or "",
category_tag,
source_tag,
self.common_tags,
)[:VK_MESSAGE_LIMIT]
async def media_attachment(self, client: VKAPIClient, item: dict[str, Any]) -> str | None:
attachment_id = str(item.get("original_attachment_id") or "").strip()