diff --git a/PROJECT.md b/PROJECT.md index 66f59cd..47b9bb9 100644 --- a/PROJECT.md +++ b/PROJECT.md @@ -61,7 +61,7 @@ │ ├── templates/ # HTML-шаблоны │ └── workers/ │ ├── parser.py # VK-парсер -│ ├── vk_storage_uploader.py# Telegram media storage uploader +│ ├── media_uploader.py# Telegram media storage uploader │ ├── ai_qualifier.py # AI-квалификатор │ └── ai_writer.py # AI-райтер ├── original_project/ # старый проект как справочник @@ -338,7 +338,7 @@ AI-райтер: Сейчас основной тип: -- `vk.storage.copy` +- `media.storage.copy` Название историческое: сначала планировался VK storage, потом медиа-сторедж вернулся в Telegram. Тип задачи пока не переименован. @@ -357,7 +357,7 @@ AI-райтер: Имена воркеров: - `vk-parser` -- `vk-storage-uploader` +- `media-uploader` - `ai-qualifier` - `ai-writer` - `tg-poster` @@ -556,7 +556,7 @@ Retention: записи старше 30 дней удаляются при ст 6. При включенном `parser_dedupe_content_hash` также убирает дубли по `content_hash`. 7. Применяет политику мусорных постов. 8. Сохраняет raw post и media. -9. Создаёт job `vk.storage.copy`. +9. Создаёт job `media.storage.copy`. ### 7.2. Политика мусорных постов @@ -615,13 +615,13 @@ Retention: записи старше 30 дней удаляются при ст ## 8. Media storage uploader -Файл: `src/vk_parser_app/workers/vk_storage_uploader.py`. +Файл: `src/vk_parser_app/workers/media_uploader.py`. Название файла историческое: сейчас фактическое хранилище медиа - Telegram, а не VK. ### 8.1. Что делает -1. Забирает job типа `vk.storage.copy`. +1. Забирает job типа `media.storage.copy`. 2. Загружает `raw_post` и media. 3. Скачивает изображения/видео. 4. Отправляет в Telegram storage channel. @@ -1247,7 +1247,7 @@ uvicorn vk_parser_app.admin:app --host 0.0.0.0 --port 8080 ```powershell $env:PYTHONPATH="src" python -m vk_parser_app.workers.parser -python -m vk_parser_app.workers.vk_storage_uploader +python -m vk_parser_app.workers.media_uploader python -m vk_parser_app.workers.ai_qualifier python -m vk_parser_app.workers.ai_writer python -m vk_parser_app.workers.tg_poster @@ -1376,16 +1376,10 @@ systemctl start ai-writer.service Они не влияют на текущую очередь, но могут путать в админке. Их стоит аккуратно пометить как failed/cancelled или скрывать служебные тесты. -### 20.2. Название `vk_storage_uploader.py` +### 20.2. Media uploader -Файл исторически называется VK storage uploader, но фактически загружает в Telegram. Можно позже переименовать: - -```text -vk_storage_uploader.py -> tg_storage_uploader.py -JOB_TYPE_VK_STORAGE_COPY -> tg.storage.copy -``` - -Делать только миграцией и аккуратно, чтобы не потерять jobs. +Uploader переименован в `media_uploader.py`, worker — в `media-uploader`, а jobs — в +`media.storage.copy`. Миграция сохраняет уже созданные задания. ### 20.3. Настройки Telegram uploader @@ -1449,7 +1443,7 @@ JOB_TYPE_VK_STORAGE_COPY -> tg.storage.copy sources -> parser -> raw_posts + raw_post_media - -> jobs(vk.storage.copy) + -> jobs(media.storage.copy) -> Telegram storage uploader -> status=storage_ready -> AI qualifier @@ -1485,7 +1479,7 @@ admin_sessions ```text src/vk_parser_app/admin.py src/vk_parser_app/workers/parser.py -src/vk_parser_app/workers/vk_storage_uploader.py +src/vk_parser_app/workers/media_uploader.py src/vk_parser_app/workers/ai_qualifier.py src/vk_parser_app/workers/ai_writer.py src/vk_parser_app/workers/tg_poster.py diff --git a/RAA.md b/RAA.md index 1a52dc8..e7f05ef 100644 --- a/RAA.md +++ b/RAA.md @@ -70,7 +70,7 @@ Database settings: As last checked on 2026-08-03: - `vk-parser=true` -- `vk-storage-uploader=true` +- `media-uploader=true` - `ai-qualifier=false` - `ai-writer=false` - `tg-poster=false` diff --git a/README.md b/README.md index ed925a4..6559c96 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,9 @@ https://vk.com/wall-239548476_123 "access": "cloudflare", "max_items": 20, "follow_links": true, - "content_selector": "article.full" + "content_selector": "article.full", + "text_selector": ".field--name-body", + "min_text_length": 50 } ``` @@ -162,7 +164,7 @@ PYTHONPATH=src .venv/bin/python -m vk_parser_app.workers.parser VK storage uploader: ```bash -PYTHONPATH=src .venv/bin/python -m vk_parser_app.workers.vk_storage_uploader +PYTHONPATH=src .venv/bin/python -m vk_parser_app.workers.media_uploader ``` ## systemd units @@ -203,17 +205,17 @@ RestartSec=10 WantedBy=multi-user.target ``` -`vk-storage-uploader.service`: +`media-uploader.service`: ```ini [Unit] -Description=VK Storage Uploader Worker +Description=Media Uploader Worker After=network.target postgresql.service [Service] WorkingDirectory=/opt/vk-parser Environment=PYTHONPATH=/opt/vk-parser/src -ExecStart=/opt/vk-parser/.venv/bin/python -m vk_parser_app.workers.vk_storage_uploader +ExecStart=/opt/vk-parser/.venv/bin/python -m vk_parser_app.workers.media_uploader Restart=always RestartSec=10 @@ -224,13 +226,13 @@ WantedBy=multi-user.target ## Текущий pipeline ```text -sources(vk) +sources(vk/site) -> vk-parser -> raw_posts + raw_post_media - -> jobs(type='vk.storage.copy') - -> vk-storage-uploader - -> wall.post в storage-группе + -> jobs(type='media.storage.copy') + -> media-uploader + -> локальный Telegram Bot API -> raw_posts.storage_post_url ``` -Фото копируются в storage-группу. Видео в первой версии сохраняются как `link_only`/исходное VK-вложение, чтобы сначала стабилизировать основной контур. +Фото и видео загружаются в Telegram media storage; исходные URL остаются в `raw_post_media`. diff --git a/STATE.md b/STATE.md index c48f02e..fbd6548 100644 --- a/STATE.md +++ b/STATE.md @@ -27,7 +27,7 @@ Current runtime is local infrastructure, not the old VPS. - Templates: `src/vk_parser_app/templates` - Workers: - VK parser: `src/vk_parser_app/workers/parser.py` - - Telegram media/storage uploader: `src/vk_parser_app/workers/vk_storage_uploader.py` + - Telegram media/storage uploader: `src/vk_parser_app/workers/media_uploader.py` - AI qualifier: `src/vk_parser_app/workers/ai_qualifier.py` - AI writer: `src/vk_parser_app/workers/ai_writer.py` - DB migrations: `db/migrations` @@ -66,7 +66,7 @@ canonical git repository as FN-8. Code work for both projects must happen in - VK publication owner: `vk_poster_owner_id=-36860851`, `vk_poster_from_group=true`. - `TG_BOT_TOKEN`, `TELEGRAM_API_ID`, and `TELEGRAM_API_HASH` were copied from FN-8 Coolify env. - Local Bot API app setting: `local_bot_api_url=http://127.0.0.1:8081`. -- As last checked, RAA has `vk-parser=true`, `vk-storage-uploader=true`. +- As last checked, RAA has `vk-parser=true`, `media-uploader=true`. AI and publication workers are intentionally disabled until explicitly started: `ai-qualifier=false`, `ai-writer=false`, `tg-poster=false`, `vk-poster=false`, `site-poster=false`, `tg_poster_enabled=false`, `vk_poster_enabled=false`. diff --git a/db/migrations/050_site_parser_text_threshold.sql b/db/migrations/050_site_parser_text_threshold.sql new file mode 100644 index 0000000..d6649ef --- /dev/null +++ b/db/migrations/050_site_parser_text_threshold.sql @@ -0,0 +1,10 @@ +INSERT INTO app_settings(key, value_json, value_type, title, description, category) +VALUES ( + 'site_parser_min_text_length', + '50'::jsonb, + 'int', + 'Минимальная длина текста сайта', + 'Материалы сайта с более коротким телом не попадают в raw и media uploader.', + 'Site Parser' +) +ON CONFLICT (key) DO NOTHING; diff --git a/db/migrations/051_rename_media_uploader.sql b/db/migrations/051_rename_media_uploader.sql new file mode 100644 index 0000000..87e14b9 --- /dev/null +++ b/db/migrations/051_rename_media_uploader.sql @@ -0,0 +1,14 @@ +UPDATE jobs +SET type='media.storage.copy', + updated_at=NOW() +WHERE type='vk.storage.copy'; + +UPDATE worker_controls +SET name='media-uploader', + updated_at=NOW() +WHERE name='vk-storage-uploader'; + +UPDATE worker_heartbeats +SET name='media-uploader', + updated_at=NOW() +WHERE name='vk-storage-uploader'; diff --git a/scripts/backfill_pending_videos.py b/scripts/backfill_pending_videos.py index 5b0813d..2b4eee2 100644 --- a/scripts/backfill_pending_videos.py +++ b/scripts/backfill_pending_videos.py @@ -11,14 +11,14 @@ from aiogram.types import FSInputFile from vk_parser_app.constants import MEDIA_STATUS_FAILED, MEDIA_STATUS_LINK_ONLY from vk_parser_app.db import get_pool -from vk_parser_app.workers.vk_storage_uploader import TMP_DIR, TMP_PREFIX, TelegramStorageUploader +from vk_parser_app.workers.media_uploader import TMP_DIR, TMP_PREFIX, MediaUploader def jlog(event: str, **payload: Any) -> None: print(json.dumps({"event": event, **payload}, ensure_ascii=False, default=str), flush=True) -async def load_pending_videos(worker: TelegramStorageUploader, limit: int) -> list[dict[str, Any]]: +async def load_pending_videos(worker: MediaUploader, limit: int) -> list[dict[str, Any]]: rows = await worker.pool.fetch( """ SELECT m.*, rp.original_url AS post_url @@ -38,7 +38,7 @@ async def load_pending_videos(worker: TelegramStorageUploader, limit: int) -> li return [dict(row) for row in rows] -async def send_video_to_storage(worker: TelegramStorageUploader, media: dict[str, Any], path: str) -> None: +async def send_video_to_storage(worker: MediaUploader, media: dict[str, Any], path: str) -> None: caption = ( f"Backfill video media #{int(media['id'])} for raw_post #{int(media['raw_post_id'])}\n" f'source post' @@ -58,7 +58,7 @@ async def send_video_to_storage(worker: TelegramStorageUploader, media: dict[str async def main() -> None: limit = int(os.getenv("BACKFILL_LIMIT", "200")) - worker = TelegramStorageUploader() + worker = MediaUploader() await worker.init() videos = await load_pending_videos(worker, limit) jlog("start", count=len(videos), limit=limit) diff --git a/site_parser_worker/app.py b/site_parser_worker/app.py index c0c5310..902659a 100644 --- a/site_parser_worker/app.py +++ b/site_parser_worker/app.py @@ -167,6 +167,7 @@ async def enrich_items_in_browser( rucaptcha_token: str, browser_state: dict[str, Any] | None, content_selector: str, + text_selector: str, ) -> tuple[list[ParsedItem], dict[str, Any]]: async with browser_lock: async with async_playwright() as playwright: @@ -194,9 +195,11 @@ async def enrich_items_in_browser( except Exception as exc: raise RuntimeError(f"Item content not found: {item.url} ({content_selector})") from exc html, text, media = clean_html(await content.inner_html(), item.url) - if text: - item.html = html - item.text = text + if text_selector: + text_content = content.locator(text_selector).first + text = clean_html(await text_content.inner_html())[1] if await text_content.count() else "" + item.html = html + item.text = text item.media = list({entry["url"]: entry for entry in [*item.media, *media]}.values()) state = await context.storage_state() await browser.close() @@ -232,6 +235,7 @@ async def parse_source( if not isinstance(follow_links, bool): raise HTTPException(status_code=422, detail="config.follow_links must be true or false") content_selector = str(config.get("content_selector") or "").strip() + text_selector = str(config.get("text_selector") or "").strip() if follow_links and not content_selector: raise HTTPException(status_code=422, detail="config.content_selector is required when follow_links=true") url = str(request.url) @@ -273,6 +277,7 @@ async def parse_source( request.rucaptcha_token.get_secret_value(), state, content_selector, + text_selector, ) except Exception as exc: raise HTTPException(status_code=502, detail=str(exc)) from exc diff --git a/src/vk_parser_app/admin.py b/src/vk_parser_app/admin.py index e4bbcf5..4610112 100644 --- a/src/vk_parser_app/admin.py +++ b/src/vk_parser_app/admin.py @@ -37,7 +37,7 @@ from .workers.site_poster import SitePoster from .workers.tg_poster import TelegramPoster from .workers.tg_reactor import TelegramReactor from .workers.vk_poster import VKPoster -from .workers.vk_storage_uploader import TelegramStorageUploader +from .workers.media_uploader import MediaUploader COOKIE_NAME = "vk_parser_admin" VK_OAUTH_VERIFIER_COOKIE = "vk_oauth_verifier" @@ -372,6 +372,7 @@ SETTING_ORDER = { "site_parser_rucaptcha_token", "site_parser_timeout_sec", "site_parser_interval_minutes", + "site_parser_min_text_length", ], "VK": [ "vk_requests_per_second", @@ -1820,7 +1821,7 @@ async def startup() -> None: ("tg-poster", TelegramPoster()), ("tg-reactor", TelegramReactor()), ("vk-poster", VKPoster()), - ("vk-storage-uploader", TelegramStorageUploader()), + ("media-uploader", MediaUploader()), ] for name, worker in workers: asyncio.create_task(start_worker_task(worker, name)) diff --git a/src/vk_parser_app/constants.py b/src/vk_parser_app/constants.py index a8b99df..6d19627 100644 --- a/src/vk_parser_app/constants.py +++ b/src/vk_parser_app/constants.py @@ -27,10 +27,10 @@ JOB_STATUS_RETRY = "retry" JOB_STATUS_DONE = "done" JOB_STATUS_DEAD = "dead" -JOB_TYPE_VK_STORAGE_COPY = "vk.storage.copy" +JOB_TYPE_MEDIA_STORAGE_COPY = "media.storage.copy" WORKER_PARSER = "vk-parser" -WORKER_STORAGE_UPLOADER = "vk-storage-uploader" +WORKER_MEDIA_UPLOADER = "media-uploader" WORKER_AI_QUALIFIER = "ai-qualifier" WORKER_AI_WRITER = "ai-writer" WORKER_TG_POSTER = "tg-poster" diff --git a/src/vk_parser_app/source_adapters.py b/src/vk_parser_app/source_adapters.py index 722005e..afabe72 100644 --- a/src/vk_parser_app/source_adapters.py +++ b/src/vk_parser_app/source_adapters.py @@ -37,6 +37,10 @@ class SourceItem: media: list[SourceMedia] = field(default_factory=list) raw: dict[str, Any] = field(default_factory=dict) + @property + def body_text(self) -> str: + return str(self.raw.get("text") or "").strip() + def validate_source_config(platform: str, config: dict[str, Any]) -> None: if platform == PLATFORM_VK: @@ -60,6 +64,12 @@ def validate_source_config(platform: str, config: dict[str, Any]) -> None: raise ValueError("follow_links должен быть true или false") if follow_links and not str(config.get("content_selector") or "").strip(): raise ValueError("При follow_links=true нужен content_selector") + try: + min_text_length = int(config.get("min_text_length", 0)) + except (TypeError, ValueError) as exc: + raise ValueError("min_text_length должен быть целым числом") from exc + if not 0 <= min_text_length <= 100_000: + raise ValueError("min_text_length должен быть от 0 до 100000") def _posted_at(value: Any) -> datetime: diff --git a/src/vk_parser_app/templates/source_form.html b/src/vk_parser_app/templates/source_form.html index a79c643..154911e 100644 --- a/src/vk_parser_app/templates/source_form.html +++ b/src/vk_parser_app/templates/source_form.html @@ -52,7 +52,8 @@ "max_items": 20 }

access: auto сначала пробует обычный запрос и при Cloudflare использует RuCaptcha; http запрещает браузер; cloudflare сразу запускает браузер.

-

Если RSS не содержит текст или медиа, добавьте "follow_links": true и CSS-селектор содержимого страницы, например "content_selector": "article.full".

+

Если RSS не содержит текст или медиа, добавьте "follow_links": true, селектор материала "content_selector": "article.full" и при необходимости отдельный селектор текста "text_selector": ".field--name-body".

+

min_text_length переопределяет общий порог Site Parser только для этого источника.

diff --git a/src/vk_parser_app/workers/vk_storage_uploader.py b/src/vk_parser_app/workers/media_uploader.py similarity index 98% rename from src/vk_parser_app/workers/vk_storage_uploader.py rename to src/vk_parser_app/workers/media_uploader.py index abfb20a..d3c56bf 100644 --- a/src/vk_parser_app/workers/vk_storage_uploader.py +++ b/src/vk_parser_app/workers/media_uploader.py @@ -19,13 +19,13 @@ from loguru import logger from ..config import settings from ..constants import ( - JOB_TYPE_VK_STORAGE_COPY, + JOB_TYPE_MEDIA_STORAGE_COPY, MEDIA_STATUS_FAILED, MEDIA_STATUS_LINK_ONLY, MEDIA_STATUS_UPLOADED, POST_STATUS_FAILED, POST_STATUS_STORAGE_READY, - WORKER_STORAGE_UPLOADER, + WORKER_MEDIA_UPLOADER, ) from ..db import fetch_float_setting, fetch_int_setting, fetch_setting, get_pool from ..heartbeat import HeartbeatReporter @@ -90,13 +90,13 @@ def original_link(post: dict) -> str: return f'#{int(post["id"])}' if url else f'#{int(post["id"])}' -class TelegramStorageUploader: +class MediaUploader: def __init__(self) -> None: self.pool = None self.bot: Bot | None = None self.storage_chat_id: int | None = None self.storage_thread_id: int | None = None - self.heartbeat = HeartbeatReporter(WORKER_STORAGE_UPLOADER, 30) + self.heartbeat = HeartbeatReporter(WORKER_MEDIA_UPLOADER, 30) self.media_group_max_items = MAX_MEDIA_GROUP self.media_upload_delay_sec = 1.0 self.tg_retry_max_attempts = 4 @@ -115,7 +115,7 @@ class TelegramStorageUploader: async def init(self) -> None: self.pool = await get_pool() - recovered = await recover_stale_jobs(self.pool, JOB_TYPE_VK_STORAGE_COPY, stale_minutes=20) + recovered = await recover_stale_jobs(self.pool, JOB_TYPE_MEDIA_STORAGE_COPY, stale_minutes=20) if recovered: logger.warning("Recovered stale storage jobs: {}", recovered) @@ -660,12 +660,12 @@ class TelegramStorageUploader: logger.info("Telegram storage done: raw_post={} messages={}", raw_post_id, message_ids) async def run_once(self, worker_id: str) -> bool: - enabled = await is_worker_enabled(self.pool, WORKER_STORAGE_UPLOADER) + enabled = await is_worker_enabled(self.pool, WORKER_MEDIA_UPLOADER) if not enabled: await self.heartbeat.beat(self.pool, status="disabled", force=True) return False - job = await claim_job(self.pool, JOB_TYPE_VK_STORAGE_COPY, worker_id) + job = await claim_job(self.pool, JOB_TYPE_MEDIA_STORAGE_COPY, worker_id) if not job: await self.heartbeat.beat(self.pool, status="idle") return False @@ -688,7 +688,7 @@ class TelegramStorageUploader: async def run_loop(self) -> None: await self.init() - worker_id = f"{WORKER_STORAGE_UPLOADER}:{os.getpid()}" + worker_id = f"{WORKER_MEDIA_UPLOADER}:{os.getpid()}" logger.info("{} started", worker_id) try: while True: @@ -705,7 +705,7 @@ class TelegramStorageUploader: async def main() -> None: logger.remove() logger.add(sys.stdout, level=settings.log_level) - worker = TelegramStorageUploader() + worker = MediaUploader() await worker.run_loop() diff --git a/src/vk_parser_app/workers/parser.py b/src/vk_parser_app/workers/parser.py index 5c421cb..b07c3eb 100644 --- a/src/vk_parser_app/workers/parser.py +++ b/src/vk_parser_app/workers/parser.py @@ -10,7 +10,7 @@ from loguru import logger from ..config import settings from ..constants import ( - JOB_TYPE_VK_STORAGE_COPY, + JOB_TYPE_MEDIA_STORAGE_COPY, PLATFORM_SITE, PLATFORM_VK, POST_STATUS_SKIPPED, @@ -22,7 +22,7 @@ from ..constants import ( from ..db import fetch_bool_setting, fetch_float_setting, fetch_int_setting, fetch_setting, get_pool from ..heartbeat import HeartbeatReporter from ..jobs import is_worker_enabled -from ..source_adapters import SiteParserClient, SourceItem +from ..source_adapters import SiteParserClient, SourceItem, json_object from ..vk_api import ( VKAPIClient, VKAPIError, @@ -223,7 +223,7 @@ class VKParserWorker: VALUES($1, 'raw_post', $2, '{}'::jsonb, 'pending') ON CONFLICT DO NOTHING """, - JOB_TYPE_VK_STORAGE_COPY, + JOB_TYPE_MEDIA_STORAGE_COPY, raw_post_id, ) return int(raw_post_id) @@ -246,7 +246,9 @@ class VKParserWorker: recent = [item for item in fetched if item.posted_at > since_dt] known = await self.known_post_ids(source_id, [item.external_id for item in recent]) candidates = [item for item in recent if item.external_id not in known] - min_text_length = max(0, await fetch_int_setting("parser_min_text_length", 0)) + default_min_text_length = max(0, await fetch_int_setting("site_parser_min_text_length", 50)) + source_config = json_object(source.get("settings_json")) + min_text_length = max(0, int(source_config.get("min_text_length", default_min_text_length))) skip_empty_text = await fetch_bool_setting("parser_skip_empty_text", True) skip_no_media = await fetch_bool_setting("parser_skip_no_media", True) skip_short_text = await fetch_bool_setting("parser_skip_text_too_short", True) @@ -260,11 +262,11 @@ class VKParserWorker: if dedupe_content_hash and content_hash in known_hashes: continue skip_reason = None - if skip_empty_text and not item.text: + if skip_empty_text and not item.body_text: skip_reason = "empty_text" elif skip_no_media and not item.media: skip_reason = "no_media" - elif skip_short_text and len(item.text) < min_text_length: + elif skip_short_text and len(item.body_text) < min_text_length: skip_reason = "text_too_short" if skip_reason and not store_skipped: continue @@ -389,7 +391,7 @@ class VKParserWorker: VALUES($1, 'raw_post', $2, '{}'::jsonb, 'pending') ON CONFLICT DO NOTHING """, - JOB_TYPE_VK_STORAGE_COPY, + JOB_TYPE_MEDIA_STORAGE_COPY, raw_post_id, ) return int(raw_post_id) diff --git a/tests/test_source_adapters.py b/tests/test_source_adapters.py index 00d235e..dc253c7 100644 --- a/tests/test_source_adapters.py +++ b/tests/test_source_adapters.py @@ -35,6 +35,13 @@ class FollowedPageResponse(FakeResponse): return data +class EmptyBodyResponse(FakeResponse): + async def json(self, **_kwargs): + data = await super().json(**_kwargs) + data["items"][0]["text"] = "" + return data + + class FakeSession: def post(self, *_args, **_kwargs): return FakeResponse() @@ -45,6 +52,11 @@ class FollowedPageSession(FakeSession): return FollowedPageResponse() +class EmptyBodySession(FakeSession): + def post(self, *_args, **_kwargs): + return EmptyBodyResponse() + + class SourceAdapterTests(unittest.IsolatedAsyncioTestCase): async def test_worker_response_is_normalized(self) -> None: client = SiteParserClient(FakeSession(), "http://worker", "token", "captcha", 30) @@ -66,6 +78,17 @@ class SourceAdapterTests(unittest.IsolatedAsyncioTestCase): "runtime_state_json": "{}", }) self.assertEqual(items[0].text, "Title\nAuthor") + self.assertEqual(items[0].body_text, "Title\nAuthor") + + async def test_title_is_not_counted_as_site_body(self) -> None: + client = SiteParserClient(EmptyBodySession(), "http://worker", "token", "captcha", 30) + items, _ = await client.fetch({ + "url": "https://example.test/rss.xml", + "settings_json": '{"format":"rss"}', + "runtime_state_json": "{}", + }) + self.assertEqual(items[0].text, "Title") + self.assertEqual(items[0].body_text, "") def test_site_config_is_required(self) -> None: with self.assertRaisesRegex(ValueError, "нужен конфиг"): @@ -75,6 +98,10 @@ class SourceAdapterTests(unittest.IsolatedAsyncioTestCase): with self.assertRaisesRegex(ValueError, "content_selector"): validate_source_config("site", {"format": "rss", "follow_links": True}) + def test_site_min_text_length_is_validated(self) -> None: + with self.assertRaisesRegex(ValueError, "min_text_length"): + validate_source_config("site", {"format": "rss", "min_text_length": "many"}) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_storage_uploader.py b/tests/test_storage_uploader.py index 93aae9c..a7ddae4 100644 --- a/tests/test_storage_uploader.py +++ b/tests/test_storage_uploader.py @@ -1,7 +1,7 @@ import asyncio from pathlib import Path -from vk_parser_app.workers.vk_storage_uploader import TelegramStorageUploader, video_provider +from vk_parser_app.workers.media_uploader import MediaUploader, video_provider class FakeContent: @@ -35,7 +35,7 @@ def test_video_provider() -> None: async def test_http_video_download() -> None: path = "/tmp/vkparser_tg_media_http_test.mp4" - result = await TelegramStorageUploader().download_http_video(FakeSession(), "https://example.test/a.mp4", path) + result = await MediaUploader().download_http_video(FakeSession(), "https://example.test/a.mp4", path) assert result["size_bytes"] == 5 assert Path(path).read_bytes() == b"video" Path(path).unlink()