From bbbb9bc3a21b296f2ab8d3f689e41baed08d0a50 Mon Sep 17 00:00:00 2001 From: exostring Date: Sun, 6 Sep 2026 16:50:48 +0500 Subject: [PATCH] Select video format by known size before downloading, no more re-encoding Replaces both prior fixes for the oversized-video bug (a blind long retry wait, then an ffmpeg re-encode pass) with the actual root-cause fix: the old "-f" selector's [filesize --- src/media_processor.py | 163 ++++++++++++++++++----------------------- 1 file changed, 71 insertions(+), 92 deletions(-) diff --git a/src/media_processor.py b/src/media_processor.py index f424991..b9e238d 100644 --- a/src/media_processor.py +++ b/src/media_processor.py @@ -11,6 +11,7 @@ from dataclasses import dataclass from pathlib import Path from typing import Any, Optional, Set import aiohttp +import yt_dlp try: from .config import settings except (ImportError, ValueError): @@ -102,13 +103,10 @@ class MediaProcessor: async def download_video_ytdlp( self, url: str, prefix: str = "video_" ) -> tuple[Optional[Path], Optional[str], bool]: - """Retries transient failures (stalls, timeouts, one-off yt-dlp/network hiccups) - a couple of times before giving up - a single flaky attempt used to fall straight - through to the link-only fallback even though the video was perfectly downloadable. - An over-the-limit download is handled separately, by re-encoding down to size (see - _reencode_to_fit) rather than by retrying - VK publishes no signal for when a - fresher upload's lower-quality renditions finish transcoding, so waiting on a guess - isn't reliable, and re-encoding the file we already have is.""" + """Retries transient failures (stalls, timeouts, one-off yt-dlp/network hiccups, + format metadata not settled yet) a couple of times before giving up - a single + flaky attempt used to fall straight through to the link-only fallback even though + the video was perfectly downloadable.""" last_result: tuple[Optional[Path], Optional[str], bool] = (None, "video download failed", False) for attempt in range(1, settings.yt_dlp_retry_attempts + 2): path, err, is_permanent = await self._download_video_ytdlp_attempt(url, prefix) @@ -123,71 +121,54 @@ class MediaProcessor: await asyncio.sleep(3.0) return last_result - async def _ffprobe_duration_sec(self, path: Path) -> Optional[float]: + async def _select_format( + self, url: str, max_size: int, netrc_path: Optional[str] + ) -> tuple[Optional[str], Optional[str]]: + """Looks up format metadata only (no download) and picks the best-quality format + at or under video_max_height whose size is actually known (filesize, or tbr * + duration) and fits max_size. Never gambles on a format with unknown size - that's + what let VK's progressive "urlNNN" links (no reported bitrate at all, sometimes + briefly serving an unfinished-transcode master) blow past the limit undetected + after a full download. Returns (format_selector, error).""" + ydl_opts: dict[str, Any] = {"quiet": True, "no_warnings": True, "skip_download": True} + if netrc_path: + ydl_opts["netrc_location"] = netrc_path + loop = asyncio.get_running_loop() try: - proc = await asyncio.create_subprocess_exec( - "ffprobe", "-v", "error", "-show_entries", "format=duration", - "-of", "default=noprint_wrappers=1:nokey=1", str(path), - stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL, + info = await loop.run_in_executor( + None, lambda: yt_dlp.YoutubeDL(ydl_opts).extract_info(url, download=False) ) - out, _ = await proc.communicate() - return float(out.decode().strip()) - except (OSError, ValueError): - return None + except Exception as exc: + return None, f"format lookup failed: {exc}" - async def _reencode_to_fit(self, input_path: Path, max_size: int) -> Optional[Path]: - """Re-encodes an over-the-limit download down under max_size instead of discarding - a perfectly watchable video - handles both a genuinely oversized source and VK - having briefly served an unfinished-transcode master on a height-capped URL. Target - bitrate is computed from the file's own real duration (ffprobe), not from VK's - metadata, so it's exact regardless of why the file came out this size.""" - duration_sec = await self._ffprobe_duration_sec(input_path) - if not duration_sec or duration_sec <= 0: - return None - audio_bitrate = 128_000 - # 10% headroom: encoders overshoot their target average bitrate somewhat, and we'd - # rather re-encode a touch smaller than land just over the limit again. - target_total_bitrate = int((max_size * 8) / duration_sec * 0.90) - target_video_bitrate = target_total_bitrate - audio_bitrate - if target_video_bitrate < 300_000: - # Duration this long vs. size this small leaves no sane quality to encode at - - # a real oversized-content case, not a transcode-timing artifact. - return None + duration = info.get("duration") or 0 + best_combined: Optional[tuple[float, str]] = None + best_video_only: Optional[tuple[float, str]] = None + for f in info.get("formats") or []: + height = f.get("height") + if height is None or height > settings.video_max_height: + continue + if f.get("vcodec") in (None, "none"): + continue + size = f.get("filesize") or f.get("filesize_approx") + if not size and f.get("tbr") and duration: + size = f["tbr"] * 1000 / 8 * duration + if not size or size > max_size: + continue + entry = (size, f["format_id"]) + acodec = f.get("acodec") + if acodec and acodec != "none": + if not best_combined or size > best_combined[0]: + best_combined = entry + else: + if not best_video_only or size > best_video_only[0]: + best_video_only = entry - output_path = input_path.with_name(f"{input_path.stem}_fit.mp4") - await register_active_path(output_path) - cmd = [ - "ffmpeg", "-y", "-i", str(input_path), - "-vf", f"scale=-2:min({settings.video_max_height}\\,ih)", - "-c:v", "libx264", "-preset", "veryfast", - "-b:v", str(target_video_bitrate), "-maxrate", str(int(target_video_bitrate * 1.2)), - "-bufsize", str(target_video_bitrate * 2), - "-c:a", "aac", "-b:a", str(audio_bitrate), - str(output_path), - ] - proc = None - try: - proc = await asyncio.create_subprocess_exec( - *cmd, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL, - ) - await asyncio.wait_for(proc.wait(), timeout=settings.yt_dlp_timeout_sec) - except (OSError, asyncio.TimeoutError): - if proc and proc.returncode is None: - proc.kill() - await proc.communicate() - output_path.unlink(missing_ok=True) - await unregister_active_path(output_path) - return None - - if proc.returncode != 0 or not output_path.exists() or output_path.stat().st_size == 0: - output_path.unlink(missing_ok=True) - await unregister_active_path(output_path) - return None - if output_path.stat().st_size > max_size: - output_path.unlink(missing_ok=True) - await unregister_active_path(output_path) - return None - return output_path + if best_combined: + return best_combined[1], None + if best_video_only: + return f"{best_video_only[1]}+bestaudio", None + return None, "no format under the size limit with a known size" async def _download_video_ytdlp_attempt( self, url: str, prefix: str = "video_" @@ -197,32 +178,32 @@ class MediaProcessor: await register_active_path(output_path) max_size = self.max_video_bytes netrc_path = None - - cmd = [sys.executable, "-m", "yt_dlp"] if "vk.com" in url and settings.vk_access_token: netrc_path = f"{output_path}.netrc" try: with open(netrc_path, "w", encoding="utf-8") as fh: fh.write(f"machine vk.com login vk_token password {settings.vk_access_token}\n") - cmd.extend(["--netrc-location", netrc_path]) except Exception as exc: logger.warning("Failed to write netrc for yt-dlp: {}", exc) + netrc_path = None - cmd.extend([ + format_selector, select_err = await self._select_format(url, max_size, netrc_path) + if not format_selector: + if netrc_path: + Path(netrc_path).unlink(missing_ok=True) + await unregister_active_path(output_path) + # Not permanent - metadata for a very fresh upload can still be settling, + # so a short retry (see download_video_ytdlp) is worth trying before giving up. + return None, select_err or "no suitable format found", False + + cmd = [ + sys.executable, "-m", "yt_dlp", url, "-o", str(output_path), "--no-playlist", "--match-filter", f"duration <= {settings.video_max_duration_sec}", "--merge-output-format", "mp4", - # Every fallback keeps the height cap: without it yt-dlp can pull an - # arbitrarily large/high-res stream only to have it discarded afterwards - # by the size check below, burning bandwidth and the whole timeout budget. - "-f", ( - f"best[height<={settings.video_max_height}][filesize<{max_size}]" - f"/best[height<={settings.video_max_height}]" - f"/bestvideo[height<={settings.video_max_height}][filesize<{max_size}]+bestaudio" - f"/bestvideo[height<={settings.video_max_height}]+bestaudio" - ), + "-f", format_selector, # No --quiet: we need yt-dlp's own progress lines to tell a slow-but-alive # download (fine, however long it takes) apart from a genuinely hung one - # a flat wall-clock timeout can't tell those apart and was killing large @@ -230,7 +211,9 @@ class MediaProcessor: # timeout). --newline makes each progress update its own line instead of # overwriting via \r, so we can read it with readline(). "--newline", "--no-warnings", - ]) + ] + if netrc_path: + cmd.extend(["--netrc-location", netrc_path]) proc = None output_lines: list[str] = [] @@ -282,22 +265,18 @@ class MediaProcessor: size = output_path.stat().st_size if size > max_size: + # Shouldn't happen - _select_format only hands back formats whose known size + # (filesize or tbr * duration) already fit under max_size. If it does, the + # estimate was off rather than the video having gotten bigger mid-download, + # so retrying won't help. logger.warning( - "Downloaded video size {} MB exceeds limit {} MB - re-encoding to fit", + "Downloaded video size {} MB exceeds limit {} MB despite format selection", round(size / (1024 * 1024), 2), round(max_size / (1024 * 1024), 2), ) - fitted = await self._reencode_to_fit(output_path, max_size) output_path.unlink(missing_ok=True) await unregister_active_path(output_path) - if fitted: - logger.info( - "Re-encoded to {} MB, under the {} MB limit", - round(fitted.stat().st_size / (1024 * 1024), 2), - round(max_size / (1024 * 1024), 2), - ) - return fitted, None, False - return None, "video exceeds size limit even after re-encoding", True + return None, "video exceeds size limit", True return output_path, None, False