exostring/raa-parser_poster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add project common publication tags

Browse Source
This commit is contained in:
Your Name
2026-08-03 21:43:56 +05:00
parent 2bcf5f38bf
commit a19cc82168
10 changed files with 143 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/vk_parser_app/admin.py
+30 -13
View File
@@ -1325,13 +1325,21 @@ async def writer_category_payload() -> list[dict[str, Any]]:
pool = await get_pool()
rows = await pool.fetch(
"""
SELECT sort_order AS id, name, tag
SELECT sort_order AS id, name, tag, COALESCE(description, '') AS description
FROM content_categories
WHERE is_active=TRUE
ORDER BY sort_order, name
"""
)
categories = [{"id": int(row["id"]), "name": str(row["name"]), "tag": str(row["tag"])} for row in rows]
categories = [
{
"id": int(row["id"]),
"name": str(row["name"]),
"tag": str(row["tag"]),
"description": str(row["description"] or ""),
}
for row in rows
]
if categories:
return categories
return [
@@ -1344,7 +1352,8 @@ async def category_rows() -> list[dict[str, Any]]:
pool = await get_pool()
rows = await pool.fetch(
"""
SELECT id, name, tag, site_name, site_slug, site_enabled,
SELECT id, name, tag, COALESCE(description, '') AS description,
site_name, site_slug, site_enabled,
is_active, sort_order, created_at, updated_at
FROM content_categories
ORDER BY is_active DESC, sort_order, name
@@ -3630,6 +3639,7 @@ async def category_create(
csrf_token: str = Form(...),
name: str = Form(...),
tag: str = Form(""),
description: str = Form(""),
site_name: str = Form(...),
site_slug: str = Form(...),
site_enabled: str | None = Form(None),
@@ -3642,6 +3652,7 @@ async def category_create(
if not name:
return redirect("/workers")
tag = normalize_hash_tag(tag or name, "category")
description = description.strip()
site_name = site_name.strip()
site_slug = re.sub(r"[^a-z0-9-]+", "-", site_slug.strip().lower().replace("_", "-")).strip("-")
if not site_name or not site_slug:
@@ -3650,24 +3661,26 @@ async def category_create(
sort_order = int(await pool.fetchval("SELECT COALESCE(MAX(sort_order), 0) + 1 FROM content_categories") or 1)
await pool.execute(
"""
INSERT INTO content_categories(name, tag, sort_order, site_name, site_slug, site_enabled)
VALUES($1, $2, $3, $4, $5, $6)
INSERT INTO content_categories(name, tag, sort_order, description, site_name, site_slug, site_enabled)
VALUES($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (name) DO UPDATE
SET tag=$2,
site_name=$4,
site_slug=$5,
site_enabled=$6,
description=$4,
site_name=$5,
site_slug=$6,
site_enabled=$7,
is_active=TRUE,
updated_at=NOW()
""",
name,
tag,
sort_order,
description,
site_name,
site_slug,
site_enabled is not None,
)
await audit(user["id"], "category.create", "content_category", None, {"name": name, "tag": tag, "site_name": site_name, "site_slug": site_slug, "site_enabled": site_enabled is not None})
await audit(user["id"], "category.create", "content_category", None, {"name": name, "tag": tag, "description": description, "site_name": site_name, "site_slug": site_slug, "site_enabled": site_enabled is not None})
return redirect("/workers")
@@ -3678,6 +3691,7 @@ async def category_update(
csrf_token: str = Form(...),
name: str = Form(...),
tag: str = Form(""),
description: str = Form(""),
site_name: str = Form(...),
site_slug: str = Form(...),
site_enabled: str | None = Form(None),
@@ -3690,6 +3704,7 @@ async def category_update(
if not name:
return redirect("/workers")
tag = normalize_hash_tag(tag or name, "category")
description = description.strip()
site_name = site_name.strip()
site_slug = re.sub(r"[^a-z0-9-]+", "-", site_slug.strip().lower().replace("_", "-")).strip("-")
if not site_name or not site_slug:
@@ -3700,20 +3715,22 @@ async def category_update(
UPDATE content_categories
SET name=$2,
tag=$3,
site_name=$4,
site_slug=$5,
site_enabled=$6,
description=$4,
site_name=$5,
site_slug=$6,
site_enabled=$7,
updated_at=NOW()
WHERE id=$1
""",
category_id,
name,
tag,
description,
site_name,
site_slug,
site_enabled is not None,
)
await audit(user["id"], "category.update", "content_category", category_id, {"name": name, "tag": tag, "site_name": site_name, "site_slug": site_slug, "site_enabled": site_enabled is not None})
await audit(user["id"], "category.update", "content_category", category_id, {"name": name, "tag": tag, "description": description, "site_name": site_name, "site_slug": site_slug, "site_enabled": site_enabled is not None})
return redirect("/workers")
src/vk_parser_app/templates/workers.html
+6 -1
View File
@@ -73,7 +73,7 @@
</summary>
<div class="p-4 flex flex-col gap-6 bg-app-bg/30">
<div class="text-sm text-app-textMuted">
Название описывает категорию для AI, тэг используется в соцсетях.
Название и описание помогают AI выбрать категорию, тэг используется в соцсетях.
</div>
<form method="post" action="/categories/create" class="flex flex-col gap-4 bg-app-surface p-4 rounded-xl border border-app-border">
@@ -88,6 +88,10 @@
<label class="block text-[10px] uppercase font-bold text-app-textMuted mb-1">Тэг (для хэштегов)</label>
<input name="tag" class="input w-full">
</div>
<div class="md:col-span-2">
<label class="block text-[10px] uppercase font-bold text-app-textMuted mb-1">Описание (для AI)</label>
<textarea name="description" class="textarea w-full min-h-20"></textarea>
</div>
<div>
<label class="block text-[10px] uppercase font-bold text-app-textMuted mb-1">На сайте</label>
<input name="site_name" class="input w-full" required>
@@ -127,6 +131,7 @@
<div class="flex flex-col gap-1.5">
<input form="category-update-{{ c.id }}" name="name" value="{{ c.name }}" class="input input-sm w-full bg-app-bg text-white h-7">
<input form="category-update-{{ c.id }}" name="tag" value="{{ c.tag }}" class="input input-sm w-full bg-app-bg text-blue-400 font-mono h-7">
<textarea form="category-update-{{ c.id }}" name="description" class="textarea textarea-sm w-full bg-app-bg text-app-textMain min-h-16">{{ c.description }}</textarea>
</div>
</td>
<td class="px-3 py-2">
src/vk_parser_app/text_utils.py
+20 -3
View File
@@ -23,8 +23,24 @@ def strip_trailing_hashtag_line(text: str) -> str:
return "\n".join(lines).rstrip()
def publication_hashtags(category_tag: str, source_tag: str) -> str:
def parse_hash_tags(value: object) -> list[str]:
if isinstance(value, list):
raw_items = [str(item) for item in value]
else:
raw_items = re.split(r"[\s,|]+", str(value or ""))
tags: list[str] = []
seen: set[str] = set()
for item in raw_items:
tag = normalize_hash_tag(item, "")
if tag and tag not in seen:
tags.append(tag)
seen.add(tag)
return tags
def publication_hashtags(category_tag: str, source_tag: str, common_tags: object = "") -> str:
tags = [
*parse_hash_tags(common_tags),
normalize_hash_tag(category_tag, "category"),
normalize_hash_tag(source_tag, "source"),
]
@@ -38,6 +54,7 @@ def build_publication_text(
text: str,
category_tag: str,
source_tag: str,
common_tags: object = "",
format_title: bool = False,
parse_mode: str | None = None,
) -> str:
@@ -59,7 +76,7 @@ def build_publication_text(
break
if title_idx is None:
return publication_hashtags(category_tag, source_tag)
return publication_hashtags(category_tag, source_tag, common_tags)
formatted_lines = []
for idx, line in enumerate(cleaned_lines):
@@ -84,7 +101,7 @@ def build_publication_text(
raw_body = "\n".join(formatted_lines)
# Collapse multiple consecutive blank lines to at most two newlines (\n\n)
normalized_body = re.sub(r"\n{3,}", "\n\n", raw_body).strip()
hashtags = publication_hashtags(category_tag, source_tag)
hashtags = publication_hashtags(category_tag, source_tag, common_tags)
if hashtags:
return f"{normalized_body}\n\n{hashtags}" if normalized_body else hashtags
return normalized_body
src/vk_parser_app/workers/ai_writer.py
+13 -5
View File
@@ -16,7 +16,7 @@ from ..constants import WORKER_AI_WRITER
from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool
from ..heartbeat import HeartbeatReporter
from ..jobs import is_worker_enabled
from ..text_utils import build_publication_text, normalize_hash_tag, parse_categories
from ..text_utils import normalize_hash_tag, parse_categories, strip_trailing_hashtag_line
from .ai_alerts import send_ai_worker_error_alert
@@ -53,7 +53,7 @@ OUTPUT SCHEMA (return array matching input order):
Rules:
- Input is a JSON object with key "posts" containing accepted posts.
- Input has key "categories" with objects: id, name, tag.
- Input has key "categories" with objects: id, name, tag, description.
- Each post includes producer_name and producer_tag. producer_name can be a manufacturer, shop, or publishing source.
- The "text" field for every rewrite must explicitly mention producer_name at least once. Do not satisfy this only through producer_tag, hashtags, notes, or metadata.
- Pick exactly one category from the categories list in input and return its numeric id as category_id.
@@ -208,7 +208,7 @@ async def load_writer_categories(pool) -> list[dict[str, Any]]:
try:
rows = await pool.fetch(
"""
SELECT sort_order AS id, name, tag
SELECT sort_order AS id, name, tag, COALESCE(description, '') AS description
FROM content_categories
WHERE is_active=TRUE
ORDER BY sort_order, name
@@ -216,7 +216,15 @@ async def load_writer_categories(pool) -> list[dict[str, Any]]:
)
except Exception:
rows = []
categories = [{"id": int(row["id"]), "name": str(row["name"]), "tag": str(row["tag"])} for row in rows]
categories = [
{
"id": int(row["id"]),
"name": str(row["name"]),
"tag": str(row["tag"]),
"description": str(row["description"] or ""),
}
for row in rows
]
if categories:
return categories
legacy = parse_categories(await fetch_setting("ai_writer_categories", []))
@@ -399,7 +407,7 @@ class AIWriterWorker:
async def apply_rewrites(self, batch_id: int, rewrites: list[dict], model: str, prompt: str) -> None:
for item in rewrites:
final_text = build_publication_text(item["text"], item["category_tag"], item["source_tag"])
final_text = strip_trailing_hashtag_line(item["text"])
editorial_status = "rejected" if item.get("reject_by_category") else "review"
editor_notes = "Отклонено AI: Не целевой контент" if item.get("reject_by_category") else None
await self.pool.execute(
src/vk_parser_app/workers/max_poster.py
+8 -1
View File
@@ -218,6 +218,7 @@ class MAXPoster:
self.auto_reaction = "👍"
self.reaction_path_template = "/messages/{message_id}/reactions"
self.dry_run = False
self.common_tags = ""
async def init(self) -> None:
self.pool = await get_pool()
@@ -243,6 +244,7 @@ class MAXPoster:
self.auto_reaction = str(await fetch_setting("max_poster_auto_reaction", "👍") or "👍").strip() or "👍"
self.reaction_path_template = str(await fetch_setting("max_poster_reaction_path_template", self.reaction_path_template) or "").strip()
self.dry_run = await fetch_bool_setting("max_poster_dry_run", False)
self.common_tags = str(await fetch_setting("publication_common_tags", "") or "").strip()
if not self.token:
raise RuntimeError("max_poster_bot_token is empty")
if not self.chat_id:
@@ -371,7 +373,12 @@ class MAXPoster:
def build_text(self, post: dict[str, Any]) -> str:
category_tag = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or post.get("final_category") or post.get("rewrite_category") or "")
source_tag = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "")
return build_publication_text(post.get("final_text") or post.get("rewritten_text") or "", category_tag, source_tag)
return build_publication_text(
post.get("final_text") or post.get("rewritten_text") or "",
category_tag,
source_tag,
self.common_tags,
)
async def download_media_to_temp(self, session: aiohttp.ClientSession, item: dict[str, Any]) -> Path | None:
media_type = str(item.get("media_type") or "")
src/vk_parser_app/workers/tg_poster.py
+10 -1
View File
@@ -111,6 +111,7 @@ class TelegramPoster:
self.recent_window = 20
self.category_repeat_penalty = 3.0
self.source_repeat_penalty = 4.0
self.common_tags = ""
async def init(self) -> None:
self.pool = await get_pool()
@@ -137,6 +138,7 @@ class TelegramPoster:
self.recent_window = max(1, await fetch_int_setting("tg_poster_recent_window", 20))
self.category_repeat_penalty = max(0.0, await fetch_float_setting("tg_poster_category_repeat_penalty", 3.0))
self.source_repeat_penalty = max(0.0, await fetch_float_setting("tg_poster_source_repeat_penalty", 4.0))
self.common_tags = str(await fetch_setting("publication_common_tags", "") or "").strip()
logger.info("TG poster config: chat={} schedule={} caption_limit={}", self.chat_id, self.schedule, self.caption_limit)
async def close(self) -> None:
@@ -325,7 +327,14 @@ class TelegramPoster:
def build_text(self, post: dict[str, Any]) -> str:
category_tag = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or post.get("final_category") or post.get("rewrite_category") or "")
source_tag = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "")
return build_publication_text(post.get("final_text") or post.get("rewritten_text") or "", category_tag, source_tag, format_title=True, parse_mode="html")
return build_publication_text(
post.get("final_text") or post.get("rewritten_text") or "",
category_tag,
source_tag,
self.common_tags,
format_title=True,
parse_mode="html",
)
async def send_text(self, text: str) -> list[int]:
message_ids: list[int] = []
src/vk_parser_app/workers/vk_poster.py
+8 -1
View File
@@ -72,6 +72,7 @@ class VKPoster:
self.category_repeat_penalty = 3.0
self.source_repeat_penalty = 4.0
self.dry_run = False
self.common_tags = ""
async def init(self) -> None:
self.pool = await get_pool()
@@ -158,6 +159,7 @@ class VKPoster:
self.category_repeat_penalty = max(0.0, await fetch_float_setting("vk_poster_category_repeat_penalty", 3.0))
self.source_repeat_penalty = max(0.0, await fetch_float_setting("vk_poster_source_repeat_penalty", 4.0))
self.dry_run = await fetch_bool_setting("vk_poster_dry_run", False)
self.common_tags = str(await fetch_setting("publication_common_tags", "") or "").strip()
if not self.token:
raise RuntimeError("vk_poster_access_token, VK_GROUP_ACCESS_TOKEN and VK_ACCESS_TOKEN are empty")
if not self.owner_id:
@@ -292,7 +294,12 @@ class VKPoster:
def build_text(self, post: dict[str, Any]) -> str:
category_tag = str(post.get("final_category_tag") or post.get("rewrite_category_tag") or post.get("final_category") or post.get("rewrite_category") or "")
source_tag = str(post.get("final_source_tag") or post.get("rewrite_source_tag") or post.get("source_tag") or "")
return build_publication_text(post.get("final_text") or post.get("rewritten_text") or "", category_tag, source_tag)[:VK_MESSAGE_LIMIT]
return build_publication_text(
post.get("final_text") or post.get("rewritten_text") or "",
category_tag,
source_tag,
self.common_tags,
)[:VK_MESSAGE_LIMIT]
async def media_attachment(self, client: VKAPIClient, item: dict[str, Any]) -> str | None:
attachment_id = str(item.get("original_attachment_id") or "").strip()