Select video format by known size before downloading, no more re-encoding
Replaces both prior fixes for the oversized-video bug (a blind long retry wait, then an ffmpeg re-encode pass) with the actual root-cause fix: the old "-f" selector's [filesize<X] filter only checks the literal filesize field, which VK's progressive "urlNNN" links never report - so the filter silently never matched and yt-dlp fell back to picking a format with no known size at all, occasionally a not-yet-transcoded master far above the height cap. Now format metadata is fetched up front (yt-dlp's own extract_info, no download) and the best-quality format at/under the height cap whose size is actually known (filesize or tbr * duration) and fits the limit is selected explicitly before a single byte is downloaded. No guessing at VK's transcoding timing, no re-encoding, no quality loss, no wasted bandwidth on oversized pulls. Verified against a real 33-minute post that previously failed: now resolves straight to a 720p/183MB format and downloads exactly that. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
+71
-92
@@ -11,6 +11,7 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, Set
|
||||
import aiohttp
|
||||
import yt_dlp
|
||||
try:
|
||||
from .config import settings
|
||||
except (ImportError, ValueError):
|
||||
@@ -102,13 +103,10 @@ class MediaProcessor:
|
||||
async def download_video_ytdlp(
|
||||
self, url: str, prefix: str = "video_"
|
||||
) -> tuple[Optional[Path], Optional[str], bool]:
|
||||
"""Retries transient failures (stalls, timeouts, one-off yt-dlp/network hiccups)
|
||||
a couple of times before giving up - a single flaky attempt used to fall straight
|
||||
through to the link-only fallback even though the video was perfectly downloadable.
|
||||
An over-the-limit download is handled separately, by re-encoding down to size (see
|
||||
_reencode_to_fit) rather than by retrying - VK publishes no signal for when a
|
||||
fresher upload's lower-quality renditions finish transcoding, so waiting on a guess
|
||||
isn't reliable, and re-encoding the file we already have is."""
|
||||
"""Retries transient failures (stalls, timeouts, one-off yt-dlp/network hiccups,
|
||||
format metadata not settled yet) a couple of times before giving up - a single
|
||||
flaky attempt used to fall straight through to the link-only fallback even though
|
||||
the video was perfectly downloadable."""
|
||||
last_result: tuple[Optional[Path], Optional[str], bool] = (None, "video download failed", False)
|
||||
for attempt in range(1, settings.yt_dlp_retry_attempts + 2):
|
||||
path, err, is_permanent = await self._download_video_ytdlp_attempt(url, prefix)
|
||||
@@ -123,71 +121,54 @@ class MediaProcessor:
|
||||
await asyncio.sleep(3.0)
|
||||
return last_result
|
||||
|
||||
async def _ffprobe_duration_sec(self, path: Path) -> Optional[float]:
|
||||
async def _select_format(
|
||||
self, url: str, max_size: int, netrc_path: Optional[str]
|
||||
) -> tuple[Optional[str], Optional[str]]:
|
||||
"""Looks up format metadata only (no download) and picks the best-quality format
|
||||
at or under video_max_height whose size is actually known (filesize, or tbr *
|
||||
duration) and fits max_size. Never gambles on a format with unknown size - that's
|
||||
what let VK's progressive "urlNNN" links (no reported bitrate at all, sometimes
|
||||
briefly serving an unfinished-transcode master) blow past the limit undetected
|
||||
after a full download. Returns (format_selector, error)."""
|
||||
ydl_opts: dict[str, Any] = {"quiet": True, "no_warnings": True, "skip_download": True}
|
||||
if netrc_path:
|
||||
ydl_opts["netrc_location"] = netrc_path
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"ffprobe", "-v", "error", "-show_entries", "format=duration",
|
||||
"-of", "default=noprint_wrappers=1:nokey=1", str(path),
|
||||
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL,
|
||||
info = await loop.run_in_executor(
|
||||
None, lambda: yt_dlp.YoutubeDL(ydl_opts).extract_info(url, download=False)
|
||||
)
|
||||
out, _ = await proc.communicate()
|
||||
return float(out.decode().strip())
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
except Exception as exc:
|
||||
return None, f"format lookup failed: {exc}"
|
||||
|
||||
async def _reencode_to_fit(self, input_path: Path, max_size: int) -> Optional[Path]:
|
||||
"""Re-encodes an over-the-limit download down under max_size instead of discarding
|
||||
a perfectly watchable video - handles both a genuinely oversized source and VK
|
||||
having briefly served an unfinished-transcode master on a height-capped URL. Target
|
||||
bitrate is computed from the file's own real duration (ffprobe), not from VK's
|
||||
metadata, so it's exact regardless of why the file came out this size."""
|
||||
duration_sec = await self._ffprobe_duration_sec(input_path)
|
||||
if not duration_sec or duration_sec <= 0:
|
||||
return None
|
||||
audio_bitrate = 128_000
|
||||
# 10% headroom: encoders overshoot their target average bitrate somewhat, and we'd
|
||||
# rather re-encode a touch smaller than land just over the limit again.
|
||||
target_total_bitrate = int((max_size * 8) / duration_sec * 0.90)
|
||||
target_video_bitrate = target_total_bitrate - audio_bitrate
|
||||
if target_video_bitrate < 300_000:
|
||||
# Duration this long vs. size this small leaves no sane quality to encode at -
|
||||
# a real oversized-content case, not a transcode-timing artifact.
|
||||
return None
|
||||
duration = info.get("duration") or 0
|
||||
best_combined: Optional[tuple[float, str]] = None
|
||||
best_video_only: Optional[tuple[float, str]] = None
|
||||
for f in info.get("formats") or []:
|
||||
height = f.get("height")
|
||||
if height is None or height > settings.video_max_height:
|
||||
continue
|
||||
if f.get("vcodec") in (None, "none"):
|
||||
continue
|
||||
size = f.get("filesize") or f.get("filesize_approx")
|
||||
if not size and f.get("tbr") and duration:
|
||||
size = f["tbr"] * 1000 / 8 * duration
|
||||
if not size or size > max_size:
|
||||
continue
|
||||
entry = (size, f["format_id"])
|
||||
acodec = f.get("acodec")
|
||||
if acodec and acodec != "none":
|
||||
if not best_combined or size > best_combined[0]:
|
||||
best_combined = entry
|
||||
else:
|
||||
if not best_video_only or size > best_video_only[0]:
|
||||
best_video_only = entry
|
||||
|
||||
output_path = input_path.with_name(f"{input_path.stem}_fit.mp4")
|
||||
await register_active_path(output_path)
|
||||
cmd = [
|
||||
"ffmpeg", "-y", "-i", str(input_path),
|
||||
"-vf", f"scale=-2:min({settings.video_max_height}\\,ih)",
|
||||
"-c:v", "libx264", "-preset", "veryfast",
|
||||
"-b:v", str(target_video_bitrate), "-maxrate", str(int(target_video_bitrate * 1.2)),
|
||||
"-bufsize", str(target_video_bitrate * 2),
|
||||
"-c:a", "aac", "-b:a", str(audio_bitrate),
|
||||
str(output_path),
|
||||
]
|
||||
proc = None
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
await asyncio.wait_for(proc.wait(), timeout=settings.yt_dlp_timeout_sec)
|
||||
except (OSError, asyncio.TimeoutError):
|
||||
if proc and proc.returncode is None:
|
||||
proc.kill()
|
||||
await proc.communicate()
|
||||
output_path.unlink(missing_ok=True)
|
||||
await unregister_active_path(output_path)
|
||||
return None
|
||||
|
||||
if proc.returncode != 0 or not output_path.exists() or output_path.stat().st_size == 0:
|
||||
output_path.unlink(missing_ok=True)
|
||||
await unregister_active_path(output_path)
|
||||
return None
|
||||
if output_path.stat().st_size > max_size:
|
||||
output_path.unlink(missing_ok=True)
|
||||
await unregister_active_path(output_path)
|
||||
return None
|
||||
return output_path
|
||||
if best_combined:
|
||||
return best_combined[1], None
|
||||
if best_video_only:
|
||||
return f"{best_video_only[1]}+bestaudio", None
|
||||
return None, "no format under the size limit with a known size"
|
||||
|
||||
async def _download_video_ytdlp_attempt(
|
||||
self, url: str, prefix: str = "video_"
|
||||
@@ -197,32 +178,32 @@ class MediaProcessor:
|
||||
await register_active_path(output_path)
|
||||
max_size = self.max_video_bytes
|
||||
netrc_path = None
|
||||
|
||||
cmd = [sys.executable, "-m", "yt_dlp"]
|
||||
if "vk.com" in url and settings.vk_access_token:
|
||||
netrc_path = f"{output_path}.netrc"
|
||||
try:
|
||||
with open(netrc_path, "w", encoding="utf-8") as fh:
|
||||
fh.write(f"machine vk.com login vk_token password {settings.vk_access_token}\n")
|
||||
cmd.extend(["--netrc-location", netrc_path])
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to write netrc for yt-dlp: {}", exc)
|
||||
netrc_path = None
|
||||
|
||||
cmd.extend([
|
||||
format_selector, select_err = await self._select_format(url, max_size, netrc_path)
|
||||
if not format_selector:
|
||||
if netrc_path:
|
||||
Path(netrc_path).unlink(missing_ok=True)
|
||||
await unregister_active_path(output_path)
|
||||
# Not permanent - metadata for a very fresh upload can still be settling,
|
||||
# so a short retry (see download_video_ytdlp) is worth trying before giving up.
|
||||
return None, select_err or "no suitable format found", False
|
||||
|
||||
cmd = [
|
||||
sys.executable, "-m", "yt_dlp",
|
||||
url,
|
||||
"-o", str(output_path),
|
||||
"--no-playlist",
|
||||
"--match-filter", f"duration <= {settings.video_max_duration_sec}",
|
||||
"--merge-output-format", "mp4",
|
||||
# Every fallback keeps the height cap: without it yt-dlp can pull an
|
||||
# arbitrarily large/high-res stream only to have it discarded afterwards
|
||||
# by the size check below, burning bandwidth and the whole timeout budget.
|
||||
"-f", (
|
||||
f"best[height<={settings.video_max_height}][filesize<{max_size}]"
|
||||
f"/best[height<={settings.video_max_height}]"
|
||||
f"/bestvideo[height<={settings.video_max_height}][filesize<{max_size}]+bestaudio"
|
||||
f"/bestvideo[height<={settings.video_max_height}]+bestaudio"
|
||||
),
|
||||
"-f", format_selector,
|
||||
# No --quiet: we need yt-dlp's own progress lines to tell a slow-but-alive
|
||||
# download (fine, however long it takes) apart from a genuinely hung one -
|
||||
# a flat wall-clock timeout can't tell those apart and was killing large
|
||||
@@ -230,7 +211,9 @@ class MediaProcessor:
|
||||
# timeout). --newline makes each progress update its own line instead of
|
||||
# overwriting via \r, so we can read it with readline().
|
||||
"--newline", "--no-warnings",
|
||||
])
|
||||
]
|
||||
if netrc_path:
|
||||
cmd.extend(["--netrc-location", netrc_path])
|
||||
|
||||
proc = None
|
||||
output_lines: list[str] = []
|
||||
@@ -282,22 +265,18 @@ class MediaProcessor:
|
||||
|
||||
size = output_path.stat().st_size
|
||||
if size > max_size:
|
||||
# Shouldn't happen - _select_format only hands back formats whose known size
|
||||
# (filesize or tbr * duration) already fit under max_size. If it does, the
|
||||
# estimate was off rather than the video having gotten bigger mid-download,
|
||||
# so retrying won't help.
|
||||
logger.warning(
|
||||
"Downloaded video size {} MB exceeds limit {} MB - re-encoding to fit",
|
||||
"Downloaded video size {} MB exceeds limit {} MB despite format selection",
|
||||
round(size / (1024 * 1024), 2),
|
||||
round(max_size / (1024 * 1024), 2),
|
||||
)
|
||||
fitted = await self._reencode_to_fit(output_path, max_size)
|
||||
output_path.unlink(missing_ok=True)
|
||||
await unregister_active_path(output_path)
|
||||
if fitted:
|
||||
logger.info(
|
||||
"Re-encoded to {} MB, under the {} MB limit",
|
||||
round(fitted.stat().st_size / (1024 * 1024), 2),
|
||||
round(max_size / (1024 * 1024), 2),
|
||||
)
|
||||
return fitted, None, False
|
||||
return None, "video exceeds size limit even after re-encoding", True
|
||||
return None, "video exceeds size limit", True
|
||||
|
||||
return output_path, None, False
|
||||
|
||||
|
||||
Reference in New Issue
Block a user