From 50df345b5b08cab16e417c9a8e8185bae808a101 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Sun, 30 Aug 2026 20:27:05 +0000 Subject: [PATCH] Implement prodl-core shared package Implement shared models, downloader, thumbnail, subtitles, and sponsorblock modules in packages/prodl-core. Co-authored-by: ClausValcaTD <193546948+ClausValcaTD@users.noreply.github.com> --- packages/prodl-core/prodl_core/__init__.py | 26 ++ packages/prodl-core/prodl_core/downloader.py | 235 ++++++++++++++++++ packages/prodl-core/prodl_core/models.py | 69 +++++ .../prodl-core/prodl_core/sponsorblock.py | 28 +++ packages/prodl-core/prodl_core/subtitles.py | 51 ++++ packages/prodl-core/prodl_core/thumbnail.py | 13 + packages/prodl-core/tests/test_downloader.py | 97 ++++++++ packages/prodl-core/tests/test_models.py | 59 +++++ .../prodl-core/tests/test_sponsorblock.py | 28 +++ packages/prodl-core/tests/test_subtitles.py | 26 ++ packages/prodl-core/tests/test_thumbnail.py | 16 ++ 11 files changed, 648 insertions(+) create mode 100644 packages/prodl-core/tests/test_downloader.py create mode 100644 packages/prodl-core/tests/test_models.py create mode 100644 packages/prodl-core/tests/test_sponsorblock.py create mode 100644 packages/prodl-core/tests/test_subtitles.py create mode 100644 packages/prodl-core/tests/test_thumbnail.py diff --git a/packages/prodl-core/prodl_core/__init__.py b/packages/prodl-core/prodl_core/__init__.py index e69de29..e075f8f 100644 --- a/packages/prodl-core/prodl_core/__init__.py +++ b/packages/prodl-core/prodl_core/__init__.py @@ -0,0 +1,26 @@ +from .models import ( + VideoFormat, + VideoInfo, + SubtitleInfo, + DownloadOptions, + DownloadResult, + StreamInfo, +) +from .downloader import ProDLDownloader +from .thumbnail import fetch_thumbnail +from .subtitles import list_subtitles +from .sponsorblock import get_segments + +__version__ = "0.1.0" +__all__ = [ + "ProDLDownloader", + "VideoFormat", + "VideoInfo", + "SubtitleInfo", + "DownloadOptions", + "DownloadResult", + "StreamInfo", + "fetch_thumbnail", + "list_subtitles", + "get_segments", +] diff --git a/packages/prodl-core/prodl_core/downloader.py b/packages/prodl-core/prodl_core/downloader.py index e69de29..49759db 100644 --- a/packages/prodl-core/prodl_core/downloader.py +++ b/packages/prodl-core/prodl_core/downloader.py @@ -0,0 +1,235 @@ +import asyncio +import glob +import os +from typing import Any +import yt_dlp + +from .models import ( + DownloadOptions, + DownloadResult, + StreamInfo, + SubtitleInfo, + VideoFormat, + VideoInfo, +) + + +class ProDLDownloader: + + @staticmethod + async def get_info(url: str) -> VideoInfo: + """ + Use yt-dlp to extract video metadata, formats, and subtitles. + """ + opts = { + 'quiet': True, + 'no_warnings': True, + } + + def _extract() -> VideoInfo: + with yt_dlp.YoutubeDL(opts) as ydl: + info = ydl.extract_info(url, download=False) + if info is None: + raise ValueError(f"Could not extract info for URL: {url}") + + # Format extraction & filtering + raw_formats = info.get("formats") or [] + video_formats: list[VideoFormat] = [] + + for fmt in raw_formats: + vcodec = fmt.get("vcodec") + acodec = fmt.get("acodec") + + has_video = vcodec is not None and vcodec != "none" + has_audio = acodec is not None and acodec != "none" + + if not (has_video or has_audio): + continue + + res = fmt.get("resolution") + if not res: + w = fmt.get("width") + h = fmt.get("height") + if w and h: + res = f"{w}x{h}" + elif has_video: + res = "video only" + else: + res = "audio only" + + filesize = fmt.get("filesize") or fmt.get("filesize_approx") + note = fmt.get("format_note") or fmt.get("note") or "" + + video_formats.append( + VideoFormat( + format_id=str(fmt.get("format_id", "")), + ext=str(fmt.get("ext", "")), + resolution=str(res), + filesize=filesize, + note=str(note), + has_video=has_video, + has_audio=has_audio, + ) + ) + + # Subtitles extraction (manual + auto) + subtitles_list: list[SubtitleInfo] = [] + + raw_subs = info.get("subtitles") or {} + for lang, lang_formats in raw_subs.items(): + lang_name = lang + if isinstance(lang_formats, list) and len(lang_formats) > 0: + lang_name = lang_formats[0].get("name") or lang + subtitles_list.append( + SubtitleInfo(lang=str(lang), lang_name=str(lang_name), is_auto=False) + ) + + raw_auto = info.get("automatic_captions") or {} + for lang, lang_formats in raw_auto.items(): + lang_name = lang + if isinstance(lang_formats, list) and len(lang_formats) > 0: + lang_name = lang_formats[0].get("name") or lang + subtitles_list.append( + SubtitleInfo(lang=str(lang), lang_name=str(lang_name), is_auto=True) + ) + + return VideoInfo( + title=str(info.get("title") or ""), + duration=info.get("duration"), + thumbnail=info.get("thumbnail"), + uploader=info.get("uploader"), + view_count=info.get("view_count"), + webpage_url=str(info.get("webpage_url") or url), + extractor=str(info.get("extractor") or ""), + formats=video_formats, + subtitles=subtitles_list, + ) + + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, _extract) + + @staticmethod + async def get_stream_info(opts: DownloadOptions) -> StreamInfo: + """ + Extract direct CDN URL for video stream without downloading. + """ + ydl_opts = { + 'quiet': True, + 'format': opts.format_id, + 'skip_download': True, + } + + def _extract() -> StreamInfo: + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + info = ydl.extract_info(opts.url, download=False) + if info is None: + raise ValueError(f"Could not extract stream info for URL: {opts.url}") + + direct_url = info.get("url") + if not direct_url: + req_dl = info.get("requested_downloads") + if req_dl and isinstance(req_dl, list) and len(req_dl) > 0: + direct_url = req_dl[0].get("url") + + if not direct_url: + raise ValueError(f"Direct stream URL not found for: {opts.url}") + + filesize = info.get("filesize") or info.get("filesize_approx") + + return StreamInfo( + url=str(direct_url), + title=str(info.get("title") or ""), + ext=str(info.get("ext") or ""), + filesize=filesize, + ) + + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, _extract) + + @staticmethod + async def download(opts: DownloadOptions) -> DownloadResult: + """ + Download video to local disk based on options. + """ + os.makedirs(opts.output_dir, exist_ok=True) + + ydl_opts: dict[str, Any] = { + 'quiet': True, + 'no_warnings': True, + 'outtmpl': f'{opts.output_dir}/%(id)s.%(ext)s', + 'merge_output_format': 'mp4', + } + + if opts.audio_only: + ydl_opts['format'] = 'bestaudio/best' + ydl_opts['postprocessors'] = [ + { + 'key': 'FFmpegExtractAudio', + 'preferredcodec': 'mp3', + 'preferredquality': '192', + } + ] + else: + ydl_opts['format'] = opts.format_id + + if opts.subtitles: + ydl_opts['writesubtitles'] = True + ydl_opts['writeautomaticsub'] = opts.auto_subs + ydl_opts['subtitleslangs'] = opts.sub_langs + ydl_opts['embedsubtitles'] = opts.sub_embed + + if opts.thumbnail: + ydl_opts['writethumbnail'] = True + ydl_opts['embedthumbnail'] = opts.thumb_embed + + if opts.sponsorblock: + if opts.sb_action == "cut": + ydl_opts['sponsorblock_remove'] = opts.sb_categories + elif opts.sb_action == "skip": + ydl_opts['sponsorblock_mark'] = opts.sb_categories + elif opts.sb_action == "mark": + ydl_opts['sponsorblock_chapter_title'] = opts.sb_categories + + def _download() -> DownloadResult: + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + info = ydl.extract_info(opts.url, download=True) + if info is None: + raise ValueError(f"Download failed for URL: {opts.url}") + + video_id = info.get("id") + title = str(info.get("title") or "") + + # Find output file by glob pattern + matching_files = glob.glob(f"{opts.output_dir}/{video_id}.*") + + # Exclude temporary download files or info json if any + matching_files = [ + f for f in matching_files + if not f.endswith(".part") and not f.endswith(".info.json") and not f.endswith(".ytdl") + ] + + if matching_files: + filepath = matching_files[0] + else: + # Fallback to requested download filename or expected filepath + filepath = f"{opts.output_dir}/{video_id}.mp4" + + filename = os.path.basename(filepath) + ext = os.path.splitext(filepath)[1].lstrip(".") + + filesize = None + if os.path.exists(filepath): + filesize = os.path.getsize(filepath) + else: + filesize = info.get("filesize") or info.get("filesize_approx") + + return DownloadResult( + filepath=filepath, + filename=filename, + title=title, + ext=ext, + filesize=filesize, + ) + + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, _download) diff --git a/packages/prodl-core/prodl_core/models.py b/packages/prodl-core/prodl_core/models.py index e69de29..9d5b2fd 100644 --- a/packages/prodl-core/prodl_core/models.py +++ b/packages/prodl-core/prodl_core/models.py @@ -0,0 +1,69 @@ +from typing import Literal +from pydantic import BaseModel + + +class VideoFormat(BaseModel): + format_id: str + ext: str + resolution: str + filesize: int | None + note: str + has_video: bool + has_audio: bool + + +class SubtitleInfo(BaseModel): + lang: str + lang_name: str + is_auto: bool # YouTube auto-generated + + +class VideoInfo(BaseModel): + title: str + duration: int | None + thumbnail: str | None + uploader: str | None + view_count: int | None + webpage_url: str + extractor: str + formats: list[VideoFormat] + subtitles: list[SubtitleInfo] # available subs + + +class DownloadOptions(BaseModel): + url: str + format_id: str = "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best" + audio_only: bool = False + + # Subtitles + subtitles: bool = False + sub_langs: list[str] = ["en", "ar"] + sub_embed: bool = False + auto_subs: bool = True + + # Thumbnail + thumbnail: bool = False + thumb_embed: bool = True # embed in MP3 as cover art + + # SponsorBlock + sponsorblock: bool = False + sb_action: Literal["skip", "cut", "mark"] = "cut" + sb_categories: list[str] = ["sponsor", "intro", "outro"] + + # Output + output_dir: str = "/tmp/prodl_downloads" + + +class DownloadResult(BaseModel): + filepath: str + filename: str + title: str + ext: str + filesize: int | None + + +class StreamInfo(BaseModel): + url: str # direct stream URL from yt-dlp + title: str + ext: str + filesize: int | None diff --git a/packages/prodl-core/prodl_core/sponsorblock.py b/packages/prodl-core/prodl_core/sponsorblock.py index e69de29..785139e 100644 --- a/packages/prodl-core/prodl_core/sponsorblock.py +++ b/packages/prodl-core/prodl_core/sponsorblock.py @@ -0,0 +1,28 @@ +import json +import httpx + + +async def get_segments(video_id: str, categories: list[str]) -> list[dict]: + """ + Call SponsorBlock API directly (optional utility). + GET https://sponsor.ajay.app/api/skipSegments + ?videoID={video_id} + &categories={json_encoded_categories} + + Use httpx async client. + Return list of segment dicts as-is from the API. + If 404 (no segments found) → return empty list. + """ + encoded_categories = json.dumps(categories) + params = { + "videoID": video_id, + "categories": encoded_categories, + } + url = "https://sponsor.ajay.app/api/skipSegments" + + async with httpx.AsyncClient() as client: + response = await client.get(url, params=params) + if response.status_code == 404: + return [] + response.raise_for_status() + return response.json() diff --git a/packages/prodl-core/prodl_core/subtitles.py b/packages/prodl-core/prodl_core/subtitles.py index e69de29..e52a20f 100644 --- a/packages/prodl-core/prodl_core/subtitles.py +++ b/packages/prodl-core/prodl_core/subtitles.py @@ -0,0 +1,51 @@ +import asyncio +import json +import yt_dlp +from .models import SubtitleInfo + + +async def list_subtitles(url: str) -> list[SubtitleInfo]: + """ + Use yt-dlp to list available subtitles. + Options: {'listsubtitles': True, 'quiet': True} + Parse info['subtitles'] → is_auto: False + Parse info['automatic_captions'] → is_auto: True + Return list[SubtitleInfo] + """ + opts = { + 'listsubtitles': True, + 'quiet': True, + } + + def _extract() -> list[SubtitleInfo]: + with yt_dlp.YoutubeDL(opts) as ydl: + info = ydl.extract_info(url, download=False) + if info is None: + return [] + + subtitles_list: list[SubtitleInfo] = [] + + # Manual subtitles + raw_subs = info.get("subtitles") or {} + for lang, lang_formats in raw_subs.items(): + lang_name = lang + if isinstance(lang_formats, list) and len(lang_formats) > 0: + lang_name = lang_formats[0].get("name") or lang + subtitles_list.append( + SubtitleInfo(lang=lang, lang_name=str(lang_name), is_auto=False) + ) + + # Auto-generated subtitles + raw_auto = info.get("automatic_captions") or {} + for lang, lang_formats in raw_auto.items(): + lang_name = lang + if isinstance(lang_formats, list) and len(lang_formats) > 0: + lang_name = lang_formats[0].get("name") or lang + subtitles_list.append( + SubtitleInfo(lang=lang, lang_name=str(lang_name), is_auto=True) + ) + + return subtitles_list + + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, _extract) diff --git a/packages/prodl-core/prodl_core/thumbnail.py b/packages/prodl-core/prodl_core/thumbnail.py index e69de29..183d055 100644 --- a/packages/prodl-core/prodl_core/thumbnail.py +++ b/packages/prodl-core/prodl_core/thumbnail.py @@ -0,0 +1,13 @@ +import httpx + + +async def fetch_thumbnail(url: str) -> bytes: + """ + Download the thumbnail image as raw bytes. + Use httpx async client. + Just GET the URL and return response.content. + """ + async with httpx.AsyncClient() as client: + response = await client.get(url) + response.raise_for_status() + return response.content diff --git a/packages/prodl-core/tests/test_downloader.py b/packages/prodl-core/tests/test_downloader.py new file mode 100644 index 0000000..202b91c --- /dev/null +++ b/packages/prodl-core/tests/test_downloader.py @@ -0,0 +1,97 @@ +import pytest +from unittest.mock import MagicMock, patch +from prodl_core.downloader import ProDLDownloader +from prodl_core.models import DownloadOptions, DownloadResult, StreamInfo, VideoInfo + + +@pytest.mark.asyncio +async def test_get_info_mock(): + mock_info = { + "title": "Mock Title", + "duration": 60, + "thumbnail": "https://example.com/thumb.jpg", + "uploader": "Uploader", + "view_count": 100, + "webpage_url": "https://example.com/video", + "extractor": "mock", + "formats": [ + { + "format_id": "18", + "ext": "mp4", + "resolution": "640x360", + "filesize": 500, + "vcodec": "avc1", + "acodec": "mp4a", + "format_note": "medium", + } + ], + "subtitles": { + "en": [{"name": "English"}] + }, + "automatic_captions": { + "es": [{"name": "Spanish"}] + }, + } + + with patch("yt_dlp.YoutubeDL") as MockYDL: + instance = MockYDL.return_value.__enter__.return_value + instance.extract_info.return_value = mock_info + + info = await ProDLDownloader.get_info("https://example.com/video") + assert info.title == "Mock Title" + assert len(info.formats) == 1 + assert info.formats[0].format_id == "18" + assert len(info.subtitles) == 2 + + +@pytest.mark.asyncio +async def test_get_stream_info_mock(): + mock_info = { + "url": "https://cdn.example.com/direct.mp4", + "title": "Stream Title", + "ext": "mp4", + "filesize": 12345, + } + + with patch("yt_dlp.YoutubeDL") as MockYDL: + instance = MockYDL.return_value.__enter__.return_value + instance.extract_info.return_value = mock_info + + opts = DownloadOptions(url="https://example.com/video") + stream_info = await ProDLDownloader.get_stream_info(opts) + assert stream_info.url == "https://cdn.example.com/direct.mp4" + assert stream_info.title == "Stream Title" + + +@pytest.mark.asyncio +async def test_download_mock(tmp_path): + output_dir = str(tmp_path) + mock_id = "test_vid_123" + filepath = f"{output_dir}/{mock_id}.mp4" + with open(filepath, "w") as f: + f.write("dummy content") + + mock_info = { + "id": mock_id, + "title": "Downloaded Title", + "ext": "mp4", + "filesize": 13, + } + + with patch("yt_dlp.YoutubeDL") as MockYDL: + instance = MockYDL.return_value.__enter__.return_value + instance.extract_info.return_value = mock_info + + opts = DownloadOptions( + url="https://example.com/video", + output_dir=output_dir, + subtitles=True, + thumbnail=True, + sponsorblock=True, + sb_action="cut", + ) + res = await ProDLDownloader.download(opts) + assert res.title == "Downloaded Title" + assert res.filename == f"{mock_id}.mp4" + assert res.filepath == filepath + assert res.filesize == 13 diff --git a/packages/prodl-core/tests/test_models.py b/packages/prodl-core/tests/test_models.py new file mode 100644 index 0000000..8ab5053 --- /dev/null +++ b/packages/prodl-core/tests/test_models.py @@ -0,0 +1,59 @@ +import pytest +from prodl_core.models import ( + VideoFormat, + SubtitleInfo, + VideoInfo, + DownloadOptions, + DownloadResult, + StreamInfo, +) + + +def test_models_instantiation(): + fmt = VideoFormat( + format_id="137", + ext="mp4", + resolution="1080p", + filesize=1000, + note="1080p", + has_video=True, + has_audio=False, + ) + assert fmt.format_id == "137" + + sub = SubtitleInfo(lang="en", lang_name="English", is_auto=False) + assert sub.is_auto is False + + info = VideoInfo( + title="Test Video", + duration=120, + thumbnail="https://example.com/thumb.jpg", + uploader="Test User", + view_count=500, + webpage_url="https://example.com/watch?v=123", + extractor="youtube", + formats=[fmt], + subtitles=[sub], + ) + assert info.title == "Test Video" + + opts = DownloadOptions(url="https://example.com/watch?v=123") + assert opts.sub_langs == ["en", "ar"] + assert opts.sb_action == "cut" + + res = DownloadResult( + filepath="/tmp/prodl_downloads/123.mp4", + filename="123.mp4", + title="Test Video", + ext="mp4", + filesize=1000, + ) + assert res.ext == "mp4" + + stream = StreamInfo( + url="https://cdn.example.com/stream.mp4", + title="Test Stream", + ext="mp4", + filesize=1000, + ) + assert stream.url == "https://cdn.example.com/stream.mp4" diff --git a/packages/prodl-core/tests/test_sponsorblock.py b/packages/prodl-core/tests/test_sponsorblock.py new file mode 100644 index 0000000..9d22754 --- /dev/null +++ b/packages/prodl-core/tests/test_sponsorblock.py @@ -0,0 +1,28 @@ +import pytest +from unittest.mock import AsyncMock, MagicMock, patch +from prodl_core.sponsorblock import get_segments + + +@pytest.mark.asyncio +async def test_get_segments_success(): + mock_data = [{"category": "sponsor", "segment": [10, 20]}] + with patch("httpx.AsyncClient.get") as mock_get: + mock_response = AsyncMock() + mock_response.status_code = 200 + mock_response.json = MagicMock(return_value=mock_data) + mock_response.raise_for_status = MagicMock() + mock_get.return_value = mock_response + + res = await get_segments("vid123", ["sponsor"]) + assert res == mock_data + + +@pytest.mark.asyncio +async def test_get_segments_404(): + with patch("httpx.AsyncClient.get") as mock_get: + mock_response = AsyncMock() + mock_response.status_code = 404 + mock_get.return_value = mock_response + + res = await get_segments("vid123", ["sponsor"]) + assert res == [] diff --git a/packages/prodl-core/tests/test_subtitles.py b/packages/prodl-core/tests/test_subtitles.py new file mode 100644 index 0000000..83a1de4 --- /dev/null +++ b/packages/prodl-core/tests/test_subtitles.py @@ -0,0 +1,26 @@ +import pytest +from unittest.mock import patch +from prodl_core.subtitles import list_subtitles + + +@pytest.mark.asyncio +async def test_list_subtitles_mock(): + mock_info = { + "subtitles": { + "en": [{"name": "English"}] + }, + "automatic_captions": { + "fr": [{"name": "French"}] + }, + } + + with patch("yt_dlp.YoutubeDL") as MockYDL: + instance = MockYDL.return_value.__enter__.return_value + instance.extract_info.return_value = mock_info + + subs = await list_subtitles("https://example.com/video") + assert len(subs) == 2 + assert subs[0].lang == "en" + assert subs[0].is_auto is False + assert subs[1].lang == "fr" + assert subs[1].is_auto is True diff --git a/packages/prodl-core/tests/test_thumbnail.py b/packages/prodl-core/tests/test_thumbnail.py new file mode 100644 index 0000000..2911c4d --- /dev/null +++ b/packages/prodl-core/tests/test_thumbnail.py @@ -0,0 +1,16 @@ +import pytest +from unittest.mock import AsyncMock, MagicMock, patch +from prodl_core.thumbnail import fetch_thumbnail + + +@pytest.mark.asyncio +async def test_fetch_thumbnail(): + mock_bytes = b"fake image bytes" + with patch("httpx.AsyncClient.get") as mock_get: + mock_response = AsyncMock() + mock_response.content = mock_bytes + mock_response.raise_for_status = MagicMock() + mock_get.return_value = mock_response + + res = await fetch_thumbnail("https://example.com/thumb.jpg") + assert res == mock_bytes