Select video format by known size before downloading, no more re-encoding

Replaces both prior fixes for the oversized-video bug (a blind long retry
wait, then an ffmpeg re-encode pass) with the actual root-cause fix: the old
"-f" selector's [filesize<X] filter only checks the literal filesize field,
which VK's progressive "urlNNN" links never report - so the filter silently
never matched and yt-dlp fell back to picking a format with no known size at
all, occasionally a not-yet-transcoded master far above the height cap.

Now format metadata is fetched up front (yt-dlp's own extract_info, no
download) and the best-quality format at/under the height cap whose size is
actually known (filesize or tbr * duration) and fits the limit is selected
explicitly before a single byte is downloaded. No guessing at VK's
transcoding timing, no re-encoding, no quality loss, no wasted bandwidth on
oversized pulls. Verified against a real 33-minute post that previously
failed: now resolves straight to a 720p/183MB format and downloads exactly
that.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

src/media_processor.py

@@ -11,6 +11,7 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, Optional, Set
 import aiohttp
+import yt_dlp
 try:
     from .config import settings
 except (ImportError, ValueError):
@@ -102,13 +103,10 @@ class MediaProcessor:
     async def download_video_ytdlp(
         self, url: str, prefix: str = "video_"
     ) -> tuple[Optional[Path], Optional[str], bool]:
-        """Retries transient failures (stalls, timeouts, one-off yt-dlp/network hiccups)
+        """Retries transient failures (stalls, timeouts, one-off yt-dlp/network hiccups,
-        a couple of times before giving up - a single flaky attempt used to fall straight
+        format metadata not settled yet) a couple of times before giving up - a single
-        through to the link-only fallback even though the video was perfectly downloadable.
+        flaky attempt used to fall straight through to the link-only fallback even though
-        An over-the-limit download is handled separately, by re-encoding down to size (see
+        the video was perfectly downloadable."""
-        _reencode_to_fit) rather than by retrying - VK publishes no signal for when a
-        fresher upload's lower-quality renditions finish transcoding, so waiting on a guess
-        isn't reliable, and re-encoding the file we already have is."""
         last_result: tuple[Optional[Path], Optional[str], bool] = (None, "video download failed", False)
         for attempt in range(1, settings.yt_dlp_retry_attempts + 2):
             path, err, is_permanent = await self._download_video_ytdlp_attempt(url, prefix)
@@ -123,71 +121,54 @@ class MediaProcessor:
                 await asyncio.sleep(3.0)
         return last_result
 
-    async def _ffprobe_duration_sec(self, path: Path) -> Optional[float]:
+    async def _select_format(
+        self, url: str, max_size: int, netrc_path: Optional[str]
+    ) -> tuple[Optional[str], Optional[str]]:
+        """Looks up format metadata only (no download) and picks the best-quality format
+        at or under video_max_height whose size is actually known (filesize, or tbr *
+        duration) and fits max_size. Never gambles on a format with unknown size - that's
+        what let VK's progressive "urlNNN" links (no reported bitrate at all, sometimes
+        briefly serving an unfinished-transcode master) blow past the limit undetected
+        after a full download. Returns (format_selector, error)."""
+        ydl_opts: dict[str, Any] = {"quiet": True, "no_warnings": True, "skip_download": True}
+        if netrc_path:
+            ydl_opts["netrc_location"] = netrc_path
+        loop = asyncio.get_running_loop()
         try:
-            proc = await asyncio.create_subprocess_exec(
+            info = await loop.run_in_executor(
-                "ffprobe", "-v", "error", "-show_entries", "format=duration",
+                None, lambda: yt_dlp.YoutubeDL(ydl_opts).extract_info(url, download=False)
-                "-of", "default=noprint_wrappers=1:nokey=1", str(path),
-                stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL,
             )
-            out, _ = await proc.communicate()
+        except Exception as exc:
-            return float(out.decode().strip())
+            return None, f"format lookup failed: {exc}"
-        except (OSError, ValueError):
-            return None
 
-    async def _reencode_to_fit(self, input_path: Path, max_size: int) -> Optional[Path]:
+        duration = info.get("duration") or 0
-        """Re-encodes an over-the-limit download down under max_size instead of discarding
+        best_combined: Optional[tuple[float, str]] = None
-        a perfectly watchable video - handles both a genuinely oversized source and VK
+        best_video_only: Optional[tuple[float, str]] = None
-        having briefly served an unfinished-transcode master on a height-capped URL. Target
+        for f in info.get("formats") or []:
-        bitrate is computed from the file's own real duration (ffprobe), not from VK's
+            height = f.get("height")
-        metadata, so it's exact regardless of why the file came out this size."""
+            if height is None or height > settings.video_max_height:
-        duration_sec = await self._ffprobe_duration_sec(input_path)
+                continue
-        if not duration_sec or duration_sec <= 0:
+            if f.get("vcodec") in (None, "none"):
-            return None
+                continue
-        audio_bitrate = 128_000
+            size = f.get("filesize") or f.get("filesize_approx")
-        # 10% headroom: encoders overshoot their target average bitrate somewhat, and we'd
+            if not size and f.get("tbr") and duration:
-        # rather re-encode a touch smaller than land just over the limit again.
+                size = f["tbr"] * 1000 / 8 * duration
-        target_total_bitrate = int((max_size * 8) / duration_sec * 0.90)
+            if not size or size > max_size:
-        target_video_bitrate = target_total_bitrate - audio_bitrate
+                continue
-        if target_video_bitrate < 300_000:
+            entry = (size, f["format_id"])
-            # Duration this long vs. size this small leaves no sane quality to encode at -
+            acodec = f.get("acodec")
-            # a real oversized-content case, not a transcode-timing artifact.
+            if acodec and acodec != "none":
-            return None
+                if not best_combined or size > best_combined[0]:
+                    best_combined = entry
+            else:
+                if not best_video_only or size > best_video_only[0]:
+                    best_video_only = entry
 
-        output_path = input_path.with_name(f"{input_path.stem}_fit.mp4")
+        if best_combined:
-        await register_active_path(output_path)
+            return best_combined[1], None
-        cmd = [
+        if best_video_only:
-            "ffmpeg", "-y", "-i", str(input_path),
+            return f"{best_video_only[1]}+bestaudio", None
-            "-vf", f"scale=-2:min({settings.video_max_height}\\,ih)",
+        return None, "no format under the size limit with a known size"
-            "-c:v", "libx264", "-preset", "veryfast",
-            "-b:v", str(target_video_bitrate), "-maxrate", str(int(target_video_bitrate * 1.2)),
-            "-bufsize", str(target_video_bitrate * 2),
-            "-c:a", "aac", "-b:a", str(audio_bitrate),
-            str(output_path),
-        ]
-        proc = None
-        try:
-            proc = await asyncio.create_subprocess_exec(
-                *cmd, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL,
-            )
-            await asyncio.wait_for(proc.wait(), timeout=settings.yt_dlp_timeout_sec)
-        except (OSError, asyncio.TimeoutError):
-            if proc and proc.returncode is None:
-                proc.kill()
-                await proc.communicate()
-            output_path.unlink(missing_ok=True)
-            await unregister_active_path(output_path)
-            return None
-
-        if proc.returncode != 0 or not output_path.exists() or output_path.stat().st_size == 0:
-            output_path.unlink(missing_ok=True)
-            await unregister_active_path(output_path)
-            return None
-        if output_path.stat().st_size > max_size:
-            output_path.unlink(missing_ok=True)
-            await unregister_active_path(output_path)
-            return None
-        return output_path
 
     async def _download_video_ytdlp_attempt(
         self, url: str, prefix: str = "video_"
@@ -197,32 +178,32 @@ class MediaProcessor:
         await register_active_path(output_path)
         max_size = self.max_video_bytes
         netrc_path = None
-
-        cmd = [sys.executable, "-m", "yt_dlp"]
         if "vk.com" in url and settings.vk_access_token:
             netrc_path = f"{output_path}.netrc"
             try:
                 with open(netrc_path, "w", encoding="utf-8") as fh:
                     fh.write(f"machine vk.com login vk_token password {settings.vk_access_token}\n")
-                cmd.extend(["--netrc-location", netrc_path])
             except Exception as exc:
                 logger.warning("Failed to write netrc for yt-dlp: {}", exc)
+                netrc_path = None
 
-        cmd.extend([
+        format_selector, select_err = await self._select_format(url, max_size, netrc_path)
+        if not format_selector:
+            if netrc_path:
+                Path(netrc_path).unlink(missing_ok=True)
+            await unregister_active_path(output_path)
+            # Not permanent - metadata for a very fresh upload can still be settling,
+            # so a short retry (see download_video_ytdlp) is worth trying before giving up.
+            return None, select_err or "no suitable format found", False
+
+        cmd = [
+            sys.executable, "-m", "yt_dlp",
             url,
             "-o", str(output_path),
             "--no-playlist",
             "--match-filter", f"duration <= {settings.video_max_duration_sec}",
             "--merge-output-format", "mp4",
-            # Every fallback keeps the height cap: without it yt-dlp can pull an
+            "-f", format_selector,
-            # arbitrarily large/high-res stream only to have it discarded afterwards
-            # by the size check below, burning bandwidth and the whole timeout budget.
-            "-f", (
-                f"best[height<={settings.video_max_height}][filesize<{max_size}]"
-                f"/best[height<={settings.video_max_height}]"
-                f"/bestvideo[height<={settings.video_max_height}][filesize<{max_size}]+bestaudio"
-                f"/bestvideo[height<={settings.video_max_height}]+bestaudio"
-            ),
             # No --quiet: we need yt-dlp's own progress lines to tell a slow-but-alive
             # download (fine, however long it takes) apart from a genuinely hung one -
             # a flat wall-clock timeout can't tell those apart and was killing large
@@ -230,7 +211,9 @@ class MediaProcessor:
             # timeout). --newline makes each progress update its own line instead of
             # overwriting via \r, so we can read it with readline().
             "--newline", "--no-warnings",
-        ])
+        ]
+        if netrc_path:
+            cmd.extend(["--netrc-location", netrc_path])
 
         proc = None
         output_lines: list[str] = []
@@ -282,22 +265,18 @@ class MediaProcessor:
 
         size = output_path.stat().st_size
         if size > max_size:
+            # Shouldn't happen - _select_format only hands back formats whose known size
+            # (filesize or tbr * duration) already fit under max_size. If it does, the
+            # estimate was off rather than the video having gotten bigger mid-download,
+            # so retrying won't help.
             logger.warning(
-                "Downloaded video size {} MB exceeds limit {} MB - re-encoding to fit",
+                "Downloaded video size {} MB exceeds limit {} MB despite format selection",
                 round(size / (1024 * 1024), 2),
                 round(max_size / (1024 * 1024), 2),
             )
-            fitted = await self._reencode_to_fit(output_path, max_size)
             output_path.unlink(missing_ok=True)
             await unregister_active_path(output_path)
-            if fitted:
+            return None, "video exceeds size limit", True
-                logger.info(
-                    "Re-encoded to {} MB, under the {} MB limit",
-                    round(fitted.stat().st_size / (1024 * 1024), 2),
-                    round(max_size / (1024 * 1024), 2),
-                )
-                return fitted, None, False
-            return None, "video exceeds size limit even after re-encoding", True
 
         return output_path, None, False