From 62b5c94cadaa5f596dc1a7083db9db12efe357be Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sat, 22 Jul 2023 09:08:12 +0530 Subject: [PATCH] [cleanup] Misc fixes Closes #7528 --- Changelog.md | 4 ++-- README.md | 4 ++-- devscripts/changelog_override.json | 7 ++++++- devscripts/make_changelog.py | 4 ++-- test/test_YoutubeDL.py | 3 +-- test/test_YoutubeDLCookieJar.py | 12 ++++++------ yt_dlp/YoutubeDL.py | 13 +++++++------ yt_dlp/compat/_legacy.py | 4 ++-- yt_dlp/cookies.py | 10 +++++----- yt_dlp/downloader/external.py | 2 +- yt_dlp/networking/_urllib.py | 4 ++-- yt_dlp/networking/common.py | 2 +- 12 files changed, 37 insertions(+), 32 deletions(-) diff --git a/Changelog.md b/Changelog.md index 622ae68b9b..32cdaca2ab 100644 --- a/Changelog.md +++ b/Changelog.md @@ -10,7 +10,7 @@ #### Important changes - Security: [[CVE-2023-35934](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-35934)] Fix [Cookie leak](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj) - `--add-header Cookie:` is deprecated and auto-scoped to input URL domains - Cookies are scoped when passed to external downloaders - - Add `cookie` field to info.json and deprecate `http_headers.Cookie` + - Add `cookies` field to info.json and deprecate `http_headers.Cookie` #### Core changes - [Allow extractors to mark formats as potentially DRM](https://github.com/yt-dlp/yt-dlp/commit/bc344cd456380999c1ee74554dfd432a38f32ec7) ([#7396](https://github.com/yt-dlp/yt-dlp/issues/7396)) by [pukkandan](https://github.com/pukkandan) @@ -51,7 +51,7 @@ #### Downloader changes - **http**: [Avoid infinite loop when no data is received](https://github.com/yt-dlp/yt-dlp/commit/662ef1e910b72e57957f06589925b2332ba52821) by [pukkandan](https://github.com/pukkandan) #### Misc. changes -- [Add CodeQL workflow](https://github.com/yt-dlp/yt-dlp/commit/6355b5f1e1e8e7f4ef866d71d51e03baf0e82f17) ([#7497](https://github.com/yt-dlp/yt-dlp/issues/7497)) by [pukkandan](https://github.com/pukkandan) +- [Add CodeQL workflow](https://github.com/yt-dlp/yt-dlp/commit/6355b5f1e1e8e7f4ef866d71d51e03baf0e82f17) ([#7497](https://github.com/yt-dlp/yt-dlp/issues/7497)) by [jorgectf](https://github.com/jorgectf) - **cleanup**: Miscellaneous: [337734d](https://github.com/yt-dlp/yt-dlp/commit/337734d4a8a6500bc65434843db346b5cbd05e81) by [pukkandan](https://github.com/pukkandan) - **docs**: [Minor fixes](https://github.com/yt-dlp/yt-dlp/commit/b532a3481046e1eabb6232ee8196fb696c356ff6) by [pukkandan](https://github.com/pukkandan) - **make_changelog**: [Skip reverted commits](https://github.com/yt-dlp/yt-dlp/commit/fa44802809d189fca0f4782263d48d6533384503) by [pukkandan](https://github.com/pukkandan) diff --git a/README.md b/README.md index 655cd41f52..ff88f817cf 100644 --- a/README.md +++ b/README.md @@ -1569,7 +1569,7 @@ ## Sorting Formats - `aext`: Audio Extension (`m4a` > `aac` > `mp3` > `ogg` > `opus` > `webm` > other). If `--prefer-free-formats` is used, the order changes to `ogg` > `opus` > `webm` > `mp3` > `m4a` > `aac` - `ext`: Equivalent to `vext,aext` - `filesize`: Exact filesize, if known in advance - - `fs_approx`: Approximate filesize calculated from the manifests + - `fs_approx`: Approximate filesize - `size`: Exact filesize if available, otherwise approximate filesize - `height`: Height of video - `width`: Width of video @@ -1580,7 +1580,7 @@ ## Sorting Formats - `tbr`: Total average bitrate in KBit/s - `vbr`: Average video bitrate in KBit/s - `abr`: Average audio bitrate in KBit/s - - `br`: Equivalent to using `tbr,vbr,abr` + - `br`: Average bitrate in KBit/s, `tbr`/`vbr`/`abr` - `asr`: Audio sample rate in Hz **Deprecation warning**: Many of these fields have (currently undocumented) aliases, that may be removed in a future version. It is recommended to use only the documented field names. diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json index f573a74630..d03db3f232 100644 --- a/devscripts/changelog_override.json +++ b/devscripts/changelog_override.json @@ -63,6 +63,11 @@ { "action": "add", "when": "1ceb657bdd254ad961489e5060f2ccc7d556b729", - "short": "[priority] Security: [[CVE-2023-35934](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-35934)] Fix [Cookie leak](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj)\n - `--add-header Cookie:` is deprecated and auto-scoped to input URL domains\n - Cookies are scoped when passed to external downloaders\n - Add `cookie` field to info.json and deprecate `http_headers.Cookie`" + "short": "[priority] Security: [[CVE-2023-35934](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-35934)] Fix [Cookie leak](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj)\n - `--add-header Cookie:` is deprecated and auto-scoped to input URL domains\n - Cookies are scoped when passed to external downloaders\n - Add `cookies` field to info.json and deprecate `http_headers.Cookie`" + }, + { + "action": "change", + "when": "b03fa7834579a01cc5fba48c0e73488a16683d48", + "short": "[ie/twitter] Revert 92315c03774cfabb3a921884326beb4b981f786b" } ] diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py index 157c661267..84f72d52f3 100644 --- a/devscripts/make_changelog.py +++ b/devscripts/make_changelog.py @@ -53,10 +53,10 @@ def commit_lookup(cls): 'cookies', 'core', 'dependencies', + 'formats', 'jsinterp', 'networking', 'outtmpl', - 'formats', 'plugins', 'update', 'upstream', @@ -254,7 +254,7 @@ class CommitRange: (?:\ \((?P\#\d+(?:,\ \#\d+)*)\))? ''', re.VERBOSE | re.DOTALL) EXTRACTOR_INDICATOR_RE = re.compile(r'(?:Fix|Add)\s+Extractors?', re.IGNORECASE) - REVERT_RE = re.compile(r'(?i:Revert)\s+([\da-f]{40})') + REVERT_RE = re.compile(r'(?:\[[^\]]+\]\s+)?(?i:Revert)\s+([\da-f]{40})') FIXES_RE = re.compile(r'(?i:Fix(?:es)?(?:\s+bugs?)?(?:\s+in|\s+for)?|Revert)\s+([\da-f]{40})') UPSTREAM_MERGE_RE = re.compile(r'Update to ytdl-commit-([\da-f]+)') diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index c54c3ea5ce..ab1250848b 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -26,7 +26,6 @@ ) from yt_dlp.utils.traversal import traverse_obj - TEST_URL = 'http://localhost/sample.mp4' @@ -687,7 +686,7 @@ def test(tmpl, expected, *, info=None, **params): test('%(duration_string)s', ('27:46:40', '27-46-40')) test('%(resolution)s', '1080p') test('%(playlist_index|)s', '001') - test('%(playlist_index&{}!)s', '001!') + test('%(playlist_index&{}!)s', '1!') test('%(playlist_autonumber)s', '02') test('%(autonumber)s', '00001') test('%(autonumber+2)03d', '005', autonumber_start=3) diff --git a/test/test_YoutubeDLCookieJar.py b/test/test_YoutubeDLCookieJar.py index 0b7a0acdb5..fdb9baee59 100644 --- a/test/test_YoutubeDLCookieJar.py +++ b/test/test_YoutubeDLCookieJar.py @@ -17,10 +17,10 @@ class TestYoutubeDLCookieJar(unittest.TestCase): def test_keep_session_cookies(self): cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/session_cookies.txt') - cookiejar.load(ignore_discard=True, ignore_expires=True) + cookiejar.load() tf = tempfile.NamedTemporaryFile(delete=False) try: - cookiejar.save(filename=tf.name, ignore_discard=True, ignore_expires=True) + cookiejar.save(filename=tf.name) temp = tf.read().decode() self.assertTrue(re.search( r'www\.foobar\.foobar\s+FALSE\s+/\s+TRUE\s+0\s+YoutubeDLExpiresEmpty\s+YoutubeDLExpiresEmptyValue', temp)) @@ -32,7 +32,7 @@ def test_keep_session_cookies(self): def test_strip_httponly_prefix(self): cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/httponly_cookies.txt') - cookiejar.load(ignore_discard=True, ignore_expires=True) + cookiejar.load() def assert_cookie_has_value(key): self.assertEqual(cookiejar._cookies['www.foobar.foobar']['/'][key].value, key + '_VALUE') @@ -42,20 +42,20 @@ def assert_cookie_has_value(key): def test_malformed_cookies(self): cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/malformed_cookies.txt') - cookiejar.load(ignore_discard=True, ignore_expires=True) + cookiejar.load() # Cookies should be empty since all malformed cookie file entries # will be ignored self.assertFalse(cookiejar._cookies) def test_get_cookie_header(self): cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/httponly_cookies.txt') - cookiejar.load(ignore_discard=True, ignore_expires=True) + cookiejar.load() header = cookiejar.get_cookie_header('https://www.foobar.foobar') self.assertIn('HTTPONLY_COOKIE', header) def test_get_cookies_for_url(self): cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/session_cookies.txt') - cookiejar.load(ignore_discard=True, ignore_expires=True) + cookiejar.load() cookies = cookiejar.get_cookies_for_url('https://www.foobar.foobar/') self.assertEqual(len(cookies), 2) cookies = cookiejar.get_cookies_for_url('https://foobar.foobar/') diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index dae29d9f95..c9cf07e530 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -572,7 +572,7 @@ class YoutubeDL: 'width', 'height', 'aspect_ratio', 'resolution', 'dynamic_range', 'tbr', 'abr', 'acodec', 'asr', 'audio_channels', 'vbr', 'fps', 'vcodec', 'container', 'filesize', 'filesize_approx', 'rows', 'columns', 'player_url', 'protocol', 'fragment_base_url', 'fragments', 'is_from_start', - 'preference', 'language', 'language_preference', 'quality', 'source_preference', + 'preference', 'language', 'language_preference', 'quality', 'source_preference', 'cookies', 'http_headers', 'stretched_ratio', 'no_resume', 'has_drm', 'extra_param_to_segment_url', 'hls_aes', 'downloader_options', 'page_url', 'app', 'play_path', 'tc_url', 'flash_version', 'rtmp_live', 'rtmp_conn', 'rtmp_protocol', 'rtmp_real_time' } @@ -621,7 +621,8 @@ def __init__(self, params=None, auto_init=True): if self.params.get('no_color'): if self.params.get('color') is not None: - self.report_warning('Overwriting params from "color" with "no_color"') + self.params.setdefault('_warnings', []).append( + 'Overwriting params from "color" with "no_color"') self.params['color'] = 'no_color' term_allow_color = os.environ.get('TERM', '').lower() != 'dumb' @@ -949,7 +950,7 @@ def __enter__(self): def save_cookies(self): if self.params.get('cookiefile') is not None: - self.cookiejar.save(ignore_discard=True, ignore_expires=True) + self.cookiejar.save() def __exit__(self, *args): self.restore_console_title() @@ -3290,7 +3291,7 @@ def existing_video_file(*filepaths): fd, success = None, True if info_dict.get('protocol') or info_dict.get('url'): fd = get_suitable_downloader(info_dict, self.params, to_stdout=temp_filename == '-') - if fd is not FFmpegFD and 'no-direct-merge' not in self.params['compat_opts'] and ( + if fd != FFmpegFD and 'no-direct-merge' not in self.params['compat_opts'] and ( info_dict.get('section_start') or info_dict.get('section_end')): msg = ('This format cannot be partially downloaded' if FFmpegFD.available() else 'You have requested downloading the video partially, but ffmpeg is not installed') @@ -3451,7 +3452,7 @@ def ffmpeg_fixup(cndn, msg, cls): postprocessed_by_ffmpeg = info_dict.get('requested_formats') or any(( isinstance(pp, FFmpegVideoConvertorPP) and resolve_recode_mapping(ext, pp.mapping)[0] not in (ext, None) - ) for pp in self._pps['post_process']) + ) for pp in self._pps['post_process']) or fd == FFmpegFD if not postprocessed_by_ffmpeg: ffmpeg_fixup(ext == 'm4a' and info_dict.get('container') == 'm4a_dash', @@ -4031,7 +4032,7 @@ def _opener(self): """ Get a urllib OpenerDirector from the Urllib handler (deprecated). """ - self.deprecation_warning('YoutubeDL._opener() is deprecated, use YoutubeDL.urlopen()') + self.deprecation_warning('YoutubeDL._opener is deprecated, use YoutubeDL.urlopen()') handler = self._request_director.handlers['Urllib'] return handler._get_instance(cookiejar=self.cookiejar, proxies=self.proxies) diff --git a/yt_dlp/compat/_legacy.py b/yt_dlp/compat/_legacy.py index 912907a021..90ccf0f14a 100644 --- a/yt_dlp/compat/_legacy.py +++ b/yt_dlp/compat/_legacy.py @@ -16,12 +16,12 @@ import shutil import socket import struct +import subprocess import tokenize import urllib.error import urllib.parse import urllib.request import xml.etree.ElementTree as etree -from subprocess import DEVNULL # isort: split import asyncio # noqa: F401 @@ -85,7 +85,7 @@ def compat_setenv(key, value, env=os.environ): compat_Struct = struct.Struct compat_struct_pack = struct.pack compat_struct_unpack = struct.unpack -compat_subprocess_get_DEVNULL = lambda: DEVNULL +compat_subprocess_get_DEVNULL = lambda: subprocess.DEVNULL compat_tokenize_tokenize = tokenize.tokenize compat_urllib_error = urllib.error compat_urllib_HTTPError = urllib.error.HTTPError diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py index 16f1918e6a..80428c747b 100644 --- a/yt_dlp/cookies.py +++ b/yt_dlp/cookies.py @@ -97,7 +97,7 @@ def load_cookies(cookie_file, browser_specification, ydl): jar = YoutubeDLCookieJar(cookie_file) if not is_filename or os.access(cookie_file, os.R_OK): - jar.load(ignore_discard=True, ignore_expires=True) + jar.load() cookie_jars.append(jar) return _merge_cookie_jars(cookie_jars) @@ -1213,7 +1213,7 @@ def open(self, file, *, write=False): file.truncate(0) yield file - def _really_save(self, f, ignore_discard=False, ignore_expires=False): + def _really_save(self, f, ignore_discard, ignore_expires): now = time.time() for cookie in self: if (not ignore_discard and cookie.discard @@ -1234,7 +1234,7 @@ def _really_save(self, f, ignore_discard=False, ignore_expires=False): name, value ))) - def save(self, filename=None, *args, **kwargs): + def save(self, filename=None, ignore_discard=True, ignore_expires=True): """ Save cookies to a file. Code is taken from CPython 3.6 @@ -1253,9 +1253,9 @@ def save(self, filename=None, *args, **kwargs): with self.open(filename, write=True) as f: f.write(self._HEADER) - self._really_save(f, *args, **kwargs) + self._really_save(f, ignore_discard, ignore_expires) - def load(self, filename=None, ignore_discard=False, ignore_expires=False): + def load(self, filename=None, ignore_discard=True, ignore_expires=True): """Load cookies from a file.""" if filename is None: if self.filename is not None: diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index d3c3eba888..4ce8a3bf7d 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -137,7 +137,7 @@ def _write_cookies(self): self._cookies_tempfile = tmp_cookies.name self.to_screen(f'[download] Writing temporary cookies file to "{self._cookies_tempfile}"') # real_download resets _cookies_tempfile; if it's None then save() will write to cookiejar.filename - self.ydl.cookiejar.save(self._cookies_tempfile, ignore_discard=True, ignore_expires=True) + self.ydl.cookiejar.save(self._cookies_tempfile) return self.ydl.cookiejar.filename or self._cookies_tempfile def _call_downloader(self, tmpfilename, info_dict): diff --git a/yt_dlp/networking/_urllib.py b/yt_dlp/networking/_urllib.py index 8a76676d94..ff3a22c8c1 100644 --- a/yt_dlp/networking/_urllib.py +++ b/yt_dlp/networking/_urllib.py @@ -28,7 +28,7 @@ make_socks_proxy_opts, select_proxy, ) -from .common import Features, RequestHandler, Response, register +from .common import Features, RequestHandler, Response, register_rh from .exceptions import ( CertificateVerifyError, HTTPError, @@ -372,7 +372,7 @@ def handle_response_read_exceptions(e): raise TransportError(cause=e) from e -@register +@register_rh class UrllibRH(RequestHandler, InstanceStoreMixin): _SUPPORTED_URL_SCHEMES = ('http', 'https', 'data', 'ftp') _SUPPORTED_PROXY_SCHEMES = ('http', 'socks4', 'socks4a', 'socks5', 'socks5h') diff --git a/yt_dlp/networking/common.py b/yt_dlp/networking/common.py index 61196406dc..7f74579780 100644 --- a/yt_dlp/networking/common.py +++ b/yt_dlp/networking/common.py @@ -105,7 +105,7 @@ def send(self, request: Request) -> Response: _REQUEST_HANDLERS = {} -def register(handler): +def register_rh(handler): """Register a RequestHandler class""" assert issubclass(handler, RequestHandler), f'{handler} must be a subclass of RequestHandler' assert handler.RH_KEY not in _REQUEST_HANDLERS, f'RequestHandler {handler.RH_KEY} already registered'