From 48f50d846c5ef54b55da66b976b620ade3741382 Mon Sep 17 00:00:00 2001 From: exostring Date: Sat, 5 Sep 2026 00:20:30 +0500 Subject: [PATCH] Re-encode oversized video downloads to fit instead of guessing wait times Reverts the guessed long-retry-budget approach (no documented VK transcoding SLA exists to tune it against, and it would've blocked other routes in the same poll cycle for minutes). Instead, when a download comes out over the platform size limit - whether genuinely oversized or VK having briefly served an unfinished-transcode master on a height-capped URL - re-encode it locally with ffmpeg to a bitrate computed from its own real duration (ffprobe), so it fits deterministically rather than by retrying and hoping. Co-Authored-By: Claude Sonnet 5 --- src/config.py | 6 --- src/media_processor.py | 114 +++++++++++++++++++++++++++++++---------- 2 files changed, 88 insertions(+), 32 deletions(-) diff --git a/src/config.py b/src/config.py index 8f5ccf7..7dcd68c 100644 --- a/src/config.py +++ b/src/config.py @@ -83,12 +83,6 @@ class Settings(BaseSettings): # Extra attempts after a transient failure (stall/timeout/network hiccup) before # falling back to a link-only post. Permanent failures (private/removed/etc.) never retry. yt_dlp_retry_attempts: int = 2 - # A fresh upload's lower-quality renditions can still be mid-transcode on VK's side, - # so a height-capped URL briefly serves the full-res master and trips our size check - - # confirmed on a real post still happening 8 minutes after publish. Needs a separate, - # much longer retry budget than the generic stall/timeout case above. - yt_dlp_size_retry_attempts: int = 5 - yt_dlp_size_retry_delay_sec: float = 90.0 # Text Styling & Decoration header_text: str = "" diff --git a/src/media_processor.py b/src/media_processor.py index d293627..f424991 100644 --- a/src/media_processor.py +++ b/src/media_processor.py @@ -102,39 +102,93 @@ class MediaProcessor: async def download_video_ytdlp( self, url: str, prefix: str = "video_" ) -> tuple[Optional[Path], Optional[str], bool]: - """Retries transient failures before giving up - a single flaky attempt used to fall - straight through to the link-only fallback even though the video was perfectly - downloadable. Stalls/timeouts/network hiccups get a couple of quick retries; a - size-exceeded result (likely VK still transcoding lower renditions) switches to a - much longer, more patient retry budget - see yt_dlp_size_retry_* in config.py.""" + """Retries transient failures (stalls, timeouts, one-off yt-dlp/network hiccups) + a couple of times before giving up - a single flaky attempt used to fall straight + through to the link-only fallback even though the video was perfectly downloadable. + An over-the-limit download is handled separately, by re-encoding down to size (see + _reencode_to_fit) rather than by retrying - VK publishes no signal for when a + fresher upload's lower-quality renditions finish transcoding, so waiting on a guess + isn't reliable, and re-encoding the file we already have is.""" last_result: tuple[Optional[Path], Optional[str], bool] = (None, "video download failed", False) - attempt = 0 - max_attempts = settings.yt_dlp_retry_attempts + 1 - delay = 3.0 - while attempt < max_attempts: - attempt += 1 + for attempt in range(1, settings.yt_dlp_retry_attempts + 2): path, err, is_permanent = await self._download_video_ytdlp_attempt(url, prefix) if path or is_permanent: return path, err, is_permanent last_result = (path, err, is_permanent) - # A fresh upload's lower-quality renditions can still be mid-transcode on - # VK's side - the "720p" URL then briefly serves the full-res master until - # the ladder catches up, which trips our size cap. Confirmed on a real post - # still happening 8 minutes after publish, so this needs a much longer and - # more patient retry budget than a stalled connection does - switch to it - # (and restart the attempt count) the first time we see this specific error. - if "exceeds size limit" in (err or "") and max_attempts != settings.yt_dlp_size_retry_attempts + 1: - max_attempts = settings.yt_dlp_size_retry_attempts + 1 - delay = settings.yt_dlp_size_retry_delay_sec - attempt = 0 - if attempt < max_attempts: + if attempt <= settings.yt_dlp_retry_attempts: logger.warning( - "yt-dlp attempt {}/{} failed transiently for {}: {} - retrying in {}s", - attempt, max_attempts, url, err, delay, + "yt-dlp attempt {}/{} failed transiently for {}: {} - retrying", + attempt, settings.yt_dlp_retry_attempts + 1, url, err, ) - await asyncio.sleep(delay) + await asyncio.sleep(3.0) return last_result + async def _ffprobe_duration_sec(self, path: Path) -> Optional[float]: + try: + proc = await asyncio.create_subprocess_exec( + "ffprobe", "-v", "error", "-show_entries", "format=duration", + "-of", "default=noprint_wrappers=1:nokey=1", str(path), + stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL, + ) + out, _ = await proc.communicate() + return float(out.decode().strip()) + except (OSError, ValueError): + return None + + async def _reencode_to_fit(self, input_path: Path, max_size: int) -> Optional[Path]: + """Re-encodes an over-the-limit download down under max_size instead of discarding + a perfectly watchable video - handles both a genuinely oversized source and VK + having briefly served an unfinished-transcode master on a height-capped URL. Target + bitrate is computed from the file's own real duration (ffprobe), not from VK's + metadata, so it's exact regardless of why the file came out this size.""" + duration_sec = await self._ffprobe_duration_sec(input_path) + if not duration_sec or duration_sec <= 0: + return None + audio_bitrate = 128_000 + # 10% headroom: encoders overshoot their target average bitrate somewhat, and we'd + # rather re-encode a touch smaller than land just over the limit again. + target_total_bitrate = int((max_size * 8) / duration_sec * 0.90) + target_video_bitrate = target_total_bitrate - audio_bitrate + if target_video_bitrate < 300_000: + # Duration this long vs. size this small leaves no sane quality to encode at - + # a real oversized-content case, not a transcode-timing artifact. + return None + + output_path = input_path.with_name(f"{input_path.stem}_fit.mp4") + await register_active_path(output_path) + cmd = [ + "ffmpeg", "-y", "-i", str(input_path), + "-vf", f"scale=-2:min({settings.video_max_height}\\,ih)", + "-c:v", "libx264", "-preset", "veryfast", + "-b:v", str(target_video_bitrate), "-maxrate", str(int(target_video_bitrate * 1.2)), + "-bufsize", str(target_video_bitrate * 2), + "-c:a", "aac", "-b:a", str(audio_bitrate), + str(output_path), + ] + proc = None + try: + proc = await asyncio.create_subprocess_exec( + *cmd, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL, + ) + await asyncio.wait_for(proc.wait(), timeout=settings.yt_dlp_timeout_sec) + except (OSError, asyncio.TimeoutError): + if proc and proc.returncode is None: + proc.kill() + await proc.communicate() + output_path.unlink(missing_ok=True) + await unregister_active_path(output_path) + return None + + if proc.returncode != 0 or not output_path.exists() or output_path.stat().st_size == 0: + output_path.unlink(missing_ok=True) + await unregister_active_path(output_path) + return None + if output_path.stat().st_size > max_size: + output_path.unlink(missing_ok=True) + await unregister_active_path(output_path) + return None + return output_path + async def _download_video_ytdlp_attempt( self, url: str, prefix: str = "video_" ) -> tuple[Optional[Path], Optional[str], bool]: @@ -229,13 +283,21 @@ class MediaProcessor: size = output_path.stat().st_size if size > max_size: logger.warning( - "Downloaded video size {} MB exceeds limit {} MB", + "Downloaded video size {} MB exceeds limit {} MB - re-encoding to fit", round(size / (1024 * 1024), 2), round(max_size / (1024 * 1024), 2), ) + fitted = await self._reencode_to_fit(output_path, max_size) output_path.unlink(missing_ok=True) await unregister_active_path(output_path) - return None, "video exceeds size limit", False + if fitted: + logger.info( + "Re-encoded to {} MB, under the {} MB limit", + round(fitted.stat().st_size / (1024 * 1024), 2), + round(max_size / (1024 * 1024), 2), + ) + return fitted, None, False + return None, "video exceeds size limit even after re-encoding", True return output_path, None, False