From 2d5cae9636714ff922d28c548c349d5f2b48f317 Mon Sep 17 00:00:00 2001
From: D0LLYNH0 <67797325+D0LLYNH0@users.noreply.github.com>
Date: Thu, 9 Mar 2023 04:18:14 -0300
Subject: [PATCH 01/13] [extractor/iq] Set more language codes (#6476)

Authored by: D0LLYNH0
---
 yt_dlp/extractor/iqiyi.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/iqiyi.py b/yt_dlp/extractor/iqiyi.py
index 4443b1991..ebf49e835 100644
--- a/yt_dlp/extractor/iqiyi.py
+++ b/yt_dlp/extractor/iqiyi.py
@@ -440,12 +440,14 @@ class IqIE(InfoExtractor):
         '1': 'zh_CN',
         '2': 'zh_TW',
         '3': 'en',
-        '4': 'kor',
+        '4': 'ko',
+        '5': 'ja',
         '18': 'th',
         '21': 'my',
         '23': 'vi',
         '24': 'id',
         '26': 'es',
+        '27': 'pt',
         '28': 'ar',
     }
 

From 3588be59cee429a0ab5c4ceb2f162298bb44147d Mon Sep 17 00:00:00 2001
From: Daniel Vogt <daniel-vogt@mail.de>
Date: Thu, 9 Mar 2023 17:21:39 +0100
Subject: [PATCH 02/13] [extractor/opencast] Add ltitools to `_VALID_URL`
 (#6371)

Authored by: C0D3D3V
---
 yt_dlp/extractor/opencast.py | 37 +++++++++++++++++++++++-------------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/yt_dlp/extractor/opencast.py b/yt_dlp/extractor/opencast.py
index fa46757f7..235ca341c 100644
--- a/yt_dlp/extractor/opencast.py
+++ b/yt_dlp/extractor/opencast.py
@@ -105,10 +105,9 @@ def _parse_mediapackage(self, video):
 
 
 class OpencastIE(OpencastBaseIE):
-    _VALID_URL = r'''(?x)
-                    https?://(?P<host>%s)/paella/ui/watch.html\?.*?
-                    id=(?P<id>%s)
-                    ''' % (OpencastBaseIE._INSTANCES_RE, OpencastBaseIE._UUID_RE)
+    _VALID_URL = rf'''(?x)
+        https?://(?P<host>{OpencastBaseIE._INSTANCES_RE})/paella/ui/watch\.html\?
+        (?:[^#]+&)?id=(?P<id>{OpencastBaseIE._UUID_RE})'''
 
     _API_BASE = 'https://%s/search/episode.json?id=%s'
 
@@ -123,6 +122,9 @@ class OpencastIE(OpencastBaseIE):
                 'thumbnail': r're:^https?://.*\.jpg$',
                 'timestamp': 1606208400,
                 'upload_date': '20201124',
+                'season_id': 'cf68a4a1-36b1-4a53-a6ba-61af5705a0d0',
+                'series': 'Kryptographie - WiSe 15/16',
+                'creator': 'Alexander May',
             },
         }
     ]
@@ -134,10 +136,11 @@ def _real_extract(self, url):
 
 
 class OpencastPlaylistIE(OpencastBaseIE):
-    _VALID_URL = r'''(?x)
-                            https?://(?P<host>%s)/engage/ui/index.html\?.*?
-                            epFrom=(?P<id>%s)
-                    ''' % (OpencastBaseIE._INSTANCES_RE, OpencastBaseIE._UUID_RE)
+    _VALID_URL = rf'''(?x)
+        https?://(?P<host>{OpencastBaseIE._INSTANCES_RE})(?:
+            /engage/ui/index\.html\?(?:[^#]+&)?epFrom=|
+            /ltitools/index\.html\?(?:[^#]+&)?series=
+        )(?P<id>{OpencastBaseIE._UUID_RE})'''
 
     _API_BASE = 'https://%s/search/episode.json?sid=%s'
 
@@ -148,15 +151,23 @@ class OpencastPlaylistIE(OpencastBaseIE):
                 'id': 'cf68a4a1-36b1-4a53-a6ba-61af5705a0d0',
                 'title': 'Kryptographie - WiSe 15/16',
             },
-            'playlist_mincount': 28,
+            'playlist_mincount': 29,
         },
         {
-            'url': 'https://oc-video.ruhr-uni-bochum.de/engage/ui/index.html?e=1&p=1&epFrom=b1a54262-3684-403f-9731-8e77c3766f9a',
+            'url': 'https://oc-video1.ruhr-uni-bochum.de/ltitools/index.html?subtool=series&series=cf68a4a1-36b1-4a53-a6ba-61af5705a0d0&lng=de',
             'info_dict': {
-                'id': 'b1a54262-3684-403f-9731-8e77c3766f9a',
-                'title': 'inSTUDIES-Social movements and prefigurative politics in a global perspective',
+                'id': 'cf68a4a1-36b1-4a53-a6ba-61af5705a0d0',
+                'title': 'Kryptographie - WiSe 15/16',
             },
-            'playlist_mincount': 6,
+            'playlist_mincount': 29,
+        },
+        {
+            'url': 'https://electures.uni-muenster.de/engage/ui/index.html?e=1&p=1&epFrom=39391d10-a711-4d23-b21d-afd2ed7d758c',
+            'info_dict': {
+                'id': '39391d10-a711-4d23-b21d-afd2ed7d758c',
+                'title': '021670 Theologische Themen bei Hans Blumenberg WiSe 2017/18',
+            },
+            'playlist_mincount': 13,
         },
     ]
 

From 66aeaac9aa30b5959069ba84e53a5508232deb38 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 9 Mar 2023 21:57:44 +0530
Subject: [PATCH 03/13] [downloader/curl] Fix progress reporting

Bug in 8c53322cda75394a8d551dde20b2529ee5ad6e89
Closes #6490
---
 yt_dlp/downloader/external.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py
index 5f54017a8..ee130c827 100644
--- a/yt_dlp/downloader/external.py
+++ b/yt_dlp/downloader/external.py
@@ -176,7 +176,7 @@ def _call_downloader(self, tmpfilename, info_dict):
         return 0
 
     def _call_process(self, cmd, info_dict):
-        return Popen.run(cmd, text=True, stderr=subprocess.PIPE)
+        return Popen.run(cmd, text=True, stderr=subprocess.PIPE if self._CAPTURE_STDERR else None)
 
 
 class CurlFD(ExternalFD):

From c9abebb851e6188cb34b9eb744c1863dd46af919 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 9 Mar 2023 22:09:23 +0530
Subject: [PATCH 04/13] [extractor/youtube] Bypass throttling for `-f17`

and related cleanup

Thanks @AudricV for the finding
---
 yt_dlp/extractor/youtube.py | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index acd4077f4..6e6abd65b 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3745,13 +3745,11 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
             if mime_mobj:
                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
                 dct.update(parse_codecs(mime_mobj.group(2)))
-            no_audio = dct.get('acodec') == 'none'
-            no_video = dct.get('vcodec') == 'none'
-            if no_audio:
-                dct['vbr'] = tbr
-            if no_video:
-                dct['abr'] = tbr
-            if no_audio or no_video:
+
+            single_stream = 'none' in (dct.get('acodec'), dct.get('vcodec'))
+            if single_stream and dct.get('ext'):
+                dct['container'] = dct['ext'] + '_dash'
+            if single_stream or itag == '17':
                 CHUNK_SIZE = 10 << 20
                 dct.update({
                     'protocol': 'http_dash_segments',
@@ -3760,13 +3758,10 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
                             'range': f'{range_start}-{min(range_start + CHUNK_SIZE - 1, dct["filesize"])}'
                         })
                     } for range_start in range(0, dct['filesize'], CHUNK_SIZE)]
-                } if dct['filesize'] else {
-                    'downloader_options': {'http_chunk_size': CHUNK_SIZE}  # No longer useful?
+                } if itag != '17' and dct['filesize'] else {
+                    'downloader_options': {'http_chunk_size': CHUNK_SIZE}
                 })
 
-                if dct.get('ext'):
-                    dct['container'] = dct['ext'] + '_dash'
-
             if itag:
                 itags[itag].add(('https', dct.get('language')))
                 stream_ids.append(stream_id)

From 0551511b45f7847f40e4314aa9e624e80d086539 Mon Sep 17 00:00:00 2001
From: Elyse <26639800+elyse0@users.noreply.github.com>
Date: Fri, 10 Mar 2023 01:12:38 -0600
Subject: [PATCH 05/13] [extractor/twitch] Fix `is_live` (#6500)

Closes #6494
Authored by: elyse0
---
 yt_dlp/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/twitch.py b/yt_dlp/extractor/twitch.py
index efc7db2c9..6321297bb 100644
--- a/yt_dlp/extractor/twitch.py
+++ b/yt_dlp/extractor/twitch.py
@@ -456,7 +456,7 @@ def _extract_info_gql(self, info, item_id):
         thumbnail = url_or_none(info.get('previewThumbnailURL'))
         is_live = None
         if thumbnail:
-            if thumbnail.endswith('/404_processing_{width}x{height}.png'):
+            if re.findall(r'/404_processing_[^.?#]+\.png', thumbnail):
                 is_live, thumbnail = True, None
             else:
                 is_live = False

From 871c907454693940cb56906ed9ea49fcb7154829 Mon Sep 17 00:00:00 2001
From: makeworld <25111343+makeworld-the-better-one@users.noreply.github.com>
Date: Fri, 10 Mar 2023 02:53:19 -0500
Subject: [PATCH 06/13] [extractor/cbc:gem] Update `_VALID_URL` (#6499)

Authored by: makeworld-the-better-one
Closes #6395
---
 yt_dlp/extractor/cbc.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/cbc.py b/yt_dlp/extractor/cbc.py
index 210f5f8ee..eadb3f8c0 100644
--- a/yt_dlp/extractor/cbc.py
+++ b/yt_dlp/extractor/cbc.py
@@ -202,7 +202,7 @@ def _real_extract(self, url):
 
 class CBCGemIE(InfoExtractor):
     IE_NAME = 'gem.cbc.ca'
-    _VALID_URL = r'https?://gem\.cbc\.ca/media/(?P<id>[0-9a-z-]+/s[0-9]+[a-z][0-9]+)'
+    _VALID_URL = r'https?://gem\.cbc\.ca/(?:media/)?(?P<id>[0-9a-z-]+/s[0-9]+[a-z][0-9]+)'
     _TESTS = [{
         # This is a normal, public, TV show video
         'url': 'https://gem.cbc.ca/media/schitts-creek/s06e01',
@@ -245,6 +245,9 @@ class CBCGemIE(InfoExtractor):
         },
         'params': {'format': 'bv'},
         'skip': 'Geo-restricted to Canada',
+    }, {
+        'url': 'https://gem.cbc.ca/nadiyas-family-favourites/s01e01',
+        'only_matching': True,
     }]
 
     _GEO_COUNTRIES = ['CA']

From ab1de9cb1e39cf421c2b7dc6756c6ff1955bb313 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 10 Mar 2023 14:12:08 +0530
Subject: [PATCH 07/13] Support loading info.json with a list at it's root

---
 yt_dlp/YoutubeDL.py | 23 ++++++++++++-----------
 yt_dlp/__init__.py  |  2 ++
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index f701738c9..a7dced8e8 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -3376,18 +3376,19 @@ def download_with_info_file(self, info_filename):
                 [info_filename], mode='r',
                 openhook=fileinput.hook_encoded('utf-8'))) as f:
             # FileInput doesn't have a read method, we can't call json.load
-            info = self.sanitize_info(json.loads('\n'.join(f)), self.params.get('clean_infojson', True))
-        try:
-            self.__download_wrapper(self.process_ie_result)(info, download=True)
-        except (DownloadError, EntryNotInPlaylist, ReExtractInfo) as e:
-            if not isinstance(e, EntryNotInPlaylist):
-                self.to_stderr('\r')
-            webpage_url = info.get('webpage_url')
-            if webpage_url is not None:
+            infos = [self.sanitize_info(info, self.params.get('clean_infojson', True))
+                     for info in variadic(json.loads('\n'.join(f)))]
+        for info in infos:
+            try:
+                self.__download_wrapper(self.process_ie_result)(info, download=True)
+            except (DownloadError, EntryNotInPlaylist, ReExtractInfo) as e:
+                if not isinstance(e, EntryNotInPlaylist):
+                    self.to_stderr('\r')
+                webpage_url = info.get('webpage_url')
+                if webpage_url is None:
+                    raise
                 self.report_warning(f'The info failed to download: {e}; trying with URL {webpage_url}')
-                return self.download([webpage_url])
-            else:
-                raise
+                self.download([webpage_url])
         return self._download_retcode
 
     @staticmethod
diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py
index 9ef31601c..bdac1212c 100644
--- a/yt_dlp/__init__.py
+++ b/yt_dlp/__init__.py
@@ -952,6 +952,8 @@ def _real_main(argv=None):
         parser.destroy()
         try:
             if opts.load_info_filename is not None:
+                if all_urls:
+                    ydl.report_warning('URLs are ignored due to --load-info-json')
                 return ydl.download_with_info_file(expand_path(opts.load_info_filename))
             else:
                 return ydl.download(all_urls)

From e6ab678e36c40ded0aae305bbb866cdab554d417 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 10 Mar 2023 17:27:43 +0530
Subject: [PATCH 08/13] [extractor/hidive] Fix login

Fixes https://github.com/yt-dlp/yt-dlp/issues/6493#issuecomment-1462906556
---
 yt_dlp/extractor/hidive.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/hidive.py b/yt_dlp/extractor/hidive.py
index 8a8749859..df6868df6 100644
--- a/yt_dlp/extractor/hidive.py
+++ b/yt_dlp/extractor/hidive.py
@@ -47,15 +47,16 @@ def _perform_login(self, username, password):
         login_webpage = self._download_webpage(
             self._LOGIN_URL, None, 'Logging in', data=urlencode_postdata(data))
         # If the user has multiple profiles on their account, select one. For now pick the first profile.
-        profile_id = self._search_regex(r'<button [^>]+?data-profile-id="(\w+)"', login_webpage, 'profile_id')
+        profile_id = self._search_regex(
+            r'<button [^>]+?data-profile-id="(\w+)"', login_webpage, 'profile id', default=None)
         if profile_id is None:
             return  # If only one profile, Hidive auto-selects it
-        profile_id_hash = self._search_regex(r'\<button [^>]+?data-hash="(\w+)"', login_webpage, 'profile_id_hash')
         self._request_webpage(
             'https://www.hidive.com/ajax/chooseprofile', None,
             data=urlencode_postdata({
                 'profileId': profile_id,
-                'hash': profile_id_hash,
+                'hash': self._search_regex(
+                    r'\<button [^>]+?data-hash="(\w+)"', login_webpage, 'profile id hash'),
                 'returnUrl': '/dashboard'
             }))
 

From 89dbf0848370deaa55af88c3593a2a264124caf5 Mon Sep 17 00:00:00 2001
From: Daniel Vogt <c0d3d3v@mag-keinen-spam.de>
Date: Sat, 11 Mar 2023 16:10:32 +0100
Subject: [PATCH 09/13] [extractor/opencast] Fix format bug (#6512)

Authored by: C0D3D3V
---
 yt_dlp/extractor/opencast.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/opencast.py b/yt_dlp/extractor/opencast.py
index 235ca341c..1fafd9afb 100644
--- a/yt_dlp/extractor/opencast.py
+++ b/yt_dlp/extractor/opencast.py
@@ -55,9 +55,9 @@ def _parse_mediapackage(self, video):
             transport = track.get('transport')
 
             if transport == 'DASH' or ext == 'mpd':
-                formats.extend(self._extract_mpd_formats_and_subtitles(href, video_id, mpd_id='dash', fatal=False))
+                formats.extend(self._extract_mpd_formats(href, video_id, mpd_id='dash', fatal=False))
             elif transport == 'HLS' or ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats_and_subtitles(
+                formats.extend(self._extract_m3u8_formats(
                     href, video_id, m3u8_id='hls', entry_protocol='m3u8_native', fatal=False))
             elif transport == 'HDS' or ext == 'f4m':
                 formats.extend(self._extract_f4m_formats(href, video_id, f4m_id='hds', fatal=False))

From 7a6c8a0807941dd24fbf0d6172e811884f98e027 Mon Sep 17 00:00:00 2001
From: vampirefrog <vampirefrog@users.noreply.github.com>
Date: Sat, 11 Mar 2023 18:52:36 +0200
Subject: [PATCH 10/13] [extractor/rokfin] Re-construct manifest url (#6507)

Authored by: vampirefrog
---
 yt_dlp/extractor/rokfin.py | 50 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 49 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/rokfin.py b/yt_dlp/extractor/rokfin.py
index ade3cd0a4..0e40eb32a 100644
--- a/yt_dlp/extractor/rokfin.py
+++ b/yt_dlp/extractor/rokfin.py
@@ -45,6 +45,7 @@ class RokfinIE(InfoExtractor):
             'live_status': 'not_live',
             'dislike_count': int,
             'like_count': int,
+            'duration': 213,
         }
     }, {
         'url': 'https://rokfin.com/post/223/Julian-Assange-Arrested-Streaming-In-Real-Time',
@@ -72,7 +73,7 @@ class RokfinIE(InfoExtractor):
             'title': '"It\'s A Crazy Mess" Regional Director Blows Whistle On Pfizer\'s Vaccine Trial Data',
             'thumbnail': r're:https://img\.production\.rokfin\.com/.+',
             'description': 'md5:324ce2d3e3b62e659506409e458b9d8e',
-            'channel': 'Ryan Cristián',
+            'channel': 'TLAVagabond',
             'channel_id': 53856,
             'channel_url': 'https://rokfin.com/TLAVagabond',
             'availability': 'public',
@@ -86,6 +87,47 @@ class RokfinIE(InfoExtractor):
             'dislike_count': int,
             'like_count': int,
             'tags': ['FreeThinkingMedia^'],
+            'duration': None,
+        }
+    }, {
+        'url': 'https://rokfin.com/post/126703/Brave-New-World--Aldous-Huxley-DEEPDIVE--Chpts-13--Quite-Frankly--Jay-Dyer',
+        'info_dict': {
+            'id': 'post/126703',
+            'ext': 'mp4',
+            'title': 'Brave New World - Aldous Huxley DEEPDIVE!  (Chpts 1-3) - Quite Frankly & Jay Dyer',
+            'thumbnail': r're:https://img\.production\.rokfin\.com/.+',
+            'channel': 'Jay Dyer',
+            'channel_id': 186881,
+            'channel_url': 'https://rokfin.com/jaydyer',
+            'availability': 'premium_only',
+            'live_status': 'not_live',
+            'dislike_count': int,
+            'like_count': int,
+            'timestamp': 1678213357,
+            'upload_date': '20230307',
+            'tags': ['FreeThinkingMedia^', 'OpenMind^'],
+            'description': 'md5:cb04e32e68326c9b2b251b297bacff35',
+            'duration': 3100,
+        }
+    }, {
+        'url': 'https://rokfin.com/stream/31332/The-Grayzone-live-on-Nordstream-blame-game',
+        'info_dict': {
+            'id': 'stream/31332',
+            'ext': 'mp4',
+            'title': 'The Grayzone live on Nordstream blame game',
+            'thumbnail': r're:https://image\.v\.rokfin\.com/.+',
+            'channel': 'Max Blumenthal',
+            'channel_id': 248902,
+            'channel_url': 'https://rokfin.com/MaxBlumenthal',
+            'availability': 'premium_only',
+            'live_status': 'was_live',
+            'dislike_count': int,
+            'like_count': int,
+            'timestamp': 1678475166,
+            'release_timestamp': 1678475166.0,
+            'release_date': '20230310',
+            'upload_date': '20230310',
+            'tags': ['FreeThinkingMedia^'],
         }
     }]
 
@@ -100,6 +142,12 @@ def _real_extract(self, url):
                        else 'not_live')
 
         video_url = traverse_obj(metadata, 'url', ('content', 'contentUrl'), expected_type=url_or_none)
+        if video_url in (None, 'fake.m3u8'):
+            video_url = format_field(self._search_regex(
+                r'https?://[^/]+/([^/]+)/storyboard.vtt',
+                traverse_obj(metadata, 'timelineUrl', ('content', 'timelineUrl'), expected_type=url_or_none),
+                video_id, default=None), None, 'https://stream.v.rokfin.com/%s.m3u8')
+
         formats, subtitles = [{'url': video_url}] if video_url else [], {}
         if determine_ext(video_url) == 'm3u8':
             formats, subtitles = self._extract_m3u8_formats_and_subtitles(

From c795c39f27244cbce846067891827e4847036441 Mon Sep 17 00:00:00 2001
From: Lesmiscore <nao20010128@gmail.com>
Date: Sun, 12 Mar 2023 02:03:23 +0900
Subject: [PATCH 11/13] [extractor/youtube] Add client name to `format_note`
 when `-v` (#6254)

Authored by: Lesmiscore, pukkandan
---
 yt_dlp/extractor/youtube.py | 29 +++++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 6e6abd65b..79174b882 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -66,6 +66,8 @@
     variadic,
 )
 
+
+STREAMING_DATA_CLIENT_NAME = '__yt_dlp_client'
 # any clients starting with _ cannot be explicitly requested by the user
 INNERTUBE_CLIENTS = {
     'web': {
@@ -248,6 +250,11 @@ def _split_innertube_client(client_name):
     return client_name, base, variant[0] if variant else None
 
 
+def short_client_name(client_name):
+    main, *parts = _split_innertube_client(client_name)[0].replace('embedscreen', 'e_s').split('_')
+    return join_nonempty(main[:4], ''.join(x[0] for x in parts)).upper()
+
+
 def build_innertube_clients():
     THIRD_PARTY = {
         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
@@ -3594,6 +3601,12 @@ def append_client(*client_names):
                     self.report_warning(
                         f'Skipping player response from {client} client (got player response for video "{pr_video_id}" instead of "{video_id}")' + bug_reports_message())
                 else:
+                    # Save client name for introspection later
+                    name = short_client_name(client)
+                    sd = traverse_obj(pr, ('streamingData', {dict})) or {}
+                    sd[STREAMING_DATA_CLIENT_NAME] = name
+                    for f in traverse_obj(sd, (('formats', 'adaptiveFormats'), ..., {dict})):
+                        f[STREAMING_DATA_CLIENT_NAME] = name
                     prs.append(pr)
 
             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
@@ -3712,6 +3725,8 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
             if is_damaged:
                 self.report_warning(
                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
+
+            client_name = fmt.get(STREAMING_DATA_CLIENT_NAME)
             dct = {
                 'asr': int_or_none(fmt.get('audioSampleRate')),
                 'filesize': int_or_none(fmt.get('contentLength')),
@@ -3723,7 +3738,9 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
                     fmt.get('isDrc') and 'DRC',
                     try_get(fmt, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
                     try_get(fmt, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
-                    throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
+                    throttled and 'THROTTLED', is_damaged and 'DAMAGED',
+                    self.get_param('verbose') and client_name,
+                    delim=', '),
                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
                 'fps': int_or_none(fmt.get('fps')) or None,
@@ -3784,7 +3801,7 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
         elif skip_bad_formats and live_status == 'is_live' and needs_live_processing != 'is_live':
             skip_manifests.add('dash')
 
-        def process_manifest_format(f, proto, itag):
+        def process_manifest_format(f, proto, client_name, itag):
             key = (proto, f.get('language'))
             if key in itags[itag]:
                 return False
@@ -3798,17 +3815,21 @@ def process_manifest_format(f, proto, itag):
             f['quality'] = q(itag_qualities.get(try_get(f, lambda f: f['format_id'].split('-')[0]), -1))
             if f['quality'] == -1 and f.get('height'):
                 f['quality'] = q(res_qualities[min(res_qualities, key=lambda x: abs(x - f['height']))])
+            if self.get_param('verbose'):
+                f['format_note'] = join_nonempty(f.get('format_note'), client_name, delim=', ')
             return True
 
         subtitles = {}
         for sd in streaming_data:
+            client_name = sd.get(STREAMING_DATA_CLIENT_NAME)
+
             hls_manifest_url = 'hls' not in skip_manifests and sd.get('hlsManifestUrl')
             if hls_manifest_url:
                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
                     hls_manifest_url, video_id, 'mp4', fatal=False, live=live_status == 'is_live')
                 subtitles = self._merge_subtitles(subs, subtitles)
                 for f in fmts:
-                    if process_manifest_format(f, 'hls', self._search_regex(
+                    if process_manifest_format(f, 'hls', client_name, self._search_regex(
                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
                         yield f
 
@@ -3817,7 +3838,7 @@ def process_manifest_format(f, proto, itag):
                 formats, subs = self._extract_mpd_formats_and_subtitles(dash_manifest_url, video_id, fatal=False)
                 subtitles = self._merge_subtitles(subs, subtitles)  # Prioritize HLS subs over DASH
                 for f in formats:
-                    if process_manifest_format(f, 'dash', f['format_id']):
+                    if process_manifest_format(f, 'dash', client_name, f['format_id']):
                         f['filesize'] = int_or_none(self._search_regex(
                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
                         if needs_live_processing:

From 86cb922118b236306310a72657f70426c20e28bb Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 9 Mar 2023 23:13:02 +0530
Subject: [PATCH 12/13] [extractor/youtube] Add extractor-arg
 `include_duplicate_formats`

---
 README.md                   |  1 +
 yt_dlp/extractor/youtube.py | 38 +++++++++++++++++++++----------------
 2 files changed, 23 insertions(+), 16 deletions(-)

diff --git a/README.md b/README.md
index f28bf8e12..de83e421f 100644
--- a/README.md
+++ b/README.md
@@ -1787,6 +1787,7 @@ #### youtube
 * `comment_sort`: `top` or `new` (default) - choose comment sorting mode (on YouTube's side)
 * `max_comments`: Limit the amount of comments to gather. Comma-separated list of integers representing `max-comments,max-parents,max-replies,max-replies-per-thread`. Default is `all,all,all,all`
     * E.g. `all,all,1000,10` will get a maximum of 1000 replies total, with up to 10 replies per thread. `1000,all,100` will get a maximum of 1000 comments, with a maximum of 100 replies total
+* `include_duplicate_formats`: Extract formats with identical content but different URLs or protocol. This is useful if some of the formats are unavailable or throttled.
 * `include_incomplete_formats`: Extract formats that cannot be downloaded completely (live dash and post-live m3u8)
 * `innertube_host`: Innertube API host to use for all API requests; e.g. `studio.youtube.com`, `youtubei.googleapis.com`. Note that cookies exported from one subdomain will not work on others
 * `innertube_key`: Innertube API key to use for all API requests
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 79174b882..48f822e44 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3640,6 +3640,7 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
         ])
         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...))
+        all_formats = self._configuration_arg('include_duplicate_formats')
 
         for fmt in streaming_formats:
             if fmt.get('targetDurationSec'):
@@ -3648,8 +3649,9 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
             itag = str_or_none(fmt.get('itag'))
             audio_track = fmt.get('audioTrack') or {}
             stream_id = (itag, audio_track.get('id'), fmt.get('isDrc'))
-            if stream_id in stream_ids:
-                continue
+            if not all_formats:
+                if stream_id in stream_ids:
+                    continue
 
             quality = fmt.get('quality')
             height = int_or_none(fmt.get('height'))
@@ -3739,7 +3741,7 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
                     try_get(fmt, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
                     try_get(fmt, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
                     throttled and 'THROTTLED', is_damaged and 'DAMAGED',
-                    self.get_param('verbose') and client_name,
+                    (self.get_param('verbose') or all_formats) and client_name,
                     delim=', '),
                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
@@ -3762,26 +3764,28 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
             if mime_mobj:
                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
                 dct.update(parse_codecs(mime_mobj.group(2)))
-
+            if itag:
+                itags[itag].add(('https', dct.get('language')))
+                stream_ids.append(stream_id)
             single_stream = 'none' in (dct.get('acodec'), dct.get('vcodec'))
             if single_stream and dct.get('ext'):
                 dct['container'] = dct['ext'] + '_dash'
-            if single_stream or itag == '17':
-                CHUNK_SIZE = 10 << 20
-                dct.update({
+
+            CHUNK_SIZE = 10 << 20
+            if dct['filesize']:
+                yield {
+                    **dct,
+                    'format_id': f'{dct["format_id"]}-dashy' if all_formats else dct['format_id'],
                     'protocol': 'http_dash_segments',
                     'fragments': [{
                         'url': update_url_query(dct['url'], {
                             'range': f'{range_start}-{min(range_start + CHUNK_SIZE - 1, dct["filesize"])}'
                         })
                     } for range_start in range(0, dct['filesize'], CHUNK_SIZE)]
-                } if itag != '17' and dct['filesize'] else {
-                    'downloader_options': {'http_chunk_size': CHUNK_SIZE}
-                })
-
-            if itag:
-                itags[itag].add(('https', dct.get('language')))
-                stream_ids.append(stream_id)
+                }
+                if not all_formats:
+                    continue
+            dct['downloader_options'] = {'http_chunk_size': CHUNK_SIZE}
             yield dct
 
         needs_live_processing = self._needs_live_processing(live_status, duration)
@@ -3803,11 +3807,13 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
 
         def process_manifest_format(f, proto, client_name, itag):
             key = (proto, f.get('language'))
-            if key in itags[itag]:
+            if not all_formats and key in itags[itag]:
                 return False
             itags[itag].add(key)
 
-            if any(p != proto for p, _ in itags[itag]):
+            if itag and all_formats:
+                f['format_id'] = f'{itag}-{proto}'
+            elif any(p != proto for p, _ in itags[itag]):
                 f['format_id'] = f'{itag}-{proto}'
             elif itag:
                 f['format_id'] = itag

From 2a23d92d9ec44a0168079e38bcf3d383e5c4c7bb Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 11 Mar 2023 22:46:47 +0530
Subject: [PATCH 13/13] [extractor/youtube] Construct fragment list lazily

Building fragment list for all formats take significant time for large videos
---
 yt_dlp/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 48f822e44..4165d795c 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3777,11 +3777,11 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
                     **dct,
                     'format_id': f'{dct["format_id"]}-dashy' if all_formats else dct['format_id'],
                     'protocol': 'http_dash_segments',
-                    'fragments': [{
+                    'fragments': LazyList({
                         'url': update_url_query(dct['url'], {
                             'range': f'{range_start}-{min(range_start + CHUNK_SIZE - 1, dct["filesize"])}'
                         })
-                    } for range_start in range(0, dct['filesize'], CHUNK_SIZE)]
+                    } for range_start in range(0, dct['filesize'], CHUNK_SIZE))
                 }
                 if not all_formats:
                     continue