Merge 87c8703f90 into fe70f20aed

Closes #11524
Authored by: seproDev

README.md

@@ -1294,6 +1294,7 @@ # OUTPUT TEMPLATE
  - `playlist_uploader_id` (string): Nickname or id of the playlist uploader
  - `playlist_channel` (string): Display name of the channel that uploaded the playlist
  - `playlist_channel_id` (string): Identifier of the channel that uploaded the playlist
+ - `playlist_webpage_url` (string): URL of the playlist webpage
  - `webpage_url` (string): A URL to the video webpage which, if given to yt-dlp, should yield the same result again
  - `webpage_url_basename` (string): The basename of the webpage URL
  - `webpage_url_domain` (string): The domain of the webpage URL

pyproject.toml

@@ -52,7 +52,7 @@ default = [
     "pycryptodomex",
     "requests>=2.32.2,<3",
     "urllib3>=1.26.17,<3",
-    "websockets>=13.0,<14",
+    "websockets>=13.0",
 ]
 curl-cffi = [
     "curl-cffi==0.5.10; os_name=='nt' and implementation_name=='cpython'",

test/test_socks.py

@@ -216,7 +216,9 @@ def handle(self):
         protocol = websockets.ServerProtocol()
         connection = websockets.sync.server.ServerConnection(socket=self.request, protocol=protocol, close_timeout=0)
         connection.handshake()
-        connection.send(json.dumps(self.socks_info))
+        for message in connection:
+            if message == 'socks_info':
+                connection.send(json.dumps(self.socks_info))
         connection.close()
 
 

yt_dlp/YoutubeDL.py

@@ -1947,6 +1947,7 @@ def _playlist_infodict(ie_result, strict=False, **kwargs):
             'playlist_uploader_id': ie_result.get('uploader_id'),
             'playlist_channel': ie_result.get('channel'),
             'playlist_channel_id': ie_result.get('channel_id'),
+            'playlist_webpage_url': ie_result.get('webpage_url'),
             **kwargs,
         }
         if strict:

yt_dlp/extractor/_extractors.py

@@ -1659,10 +1659,8 @@
     RaiIE,
     RaiNewsIE,
     RaiPlayIE,
-    RaiPlayLiveIE,
     RaiPlayPlaylistIE,
     RaiPlaySoundIE,
-    RaiPlaySoundLiveIE,
     RaiPlaySoundPlaylistIE,
     RaiSudtirolIE,
 )

yt_dlp/extractor/facebook.py

@@ -50,7 +50,7 @@ class FacebookIE(InfoExtractor):
                             [^/]+/videos/(?:[^/]+/)?|
                             [^/]+/posts/|
                             events/(?:[^/]+/)?|
-                            groups/[^/]+/(?:permalink|posts)/|
+                            groups/[^/]+/(?:permalink|posts)/(?:[\da-f]+/)?|
                             watchparty/
                         )|
                     facebook:
@@ -410,6 +410,9 @@ class FacebookIE(InfoExtractor):
             'uploader': 'Comitato Liberi Pensatori',
             'uploader_id': '100065709540881',
         },
+    }, {
+        'url': 'https://www.facebook.com/groups/1513990329015294/posts/d41d8cd9/2013209885760000/?app=fbl',
+        'only_matching': True,
     }]
     _SUPPORTED_PAGLETS_REGEX = r'(?:pagelet_group_mall|permalink_video_pagelet|hyperfeed_story_id_[0-9a-f]+)'
     _api_config = {

yt_dlp/extractor/la7.py

@@ -7,10 +7,11 @@
 
 class LA7IE(InfoExtractor):
     IE_NAME = 'la7.it'
-    _VALID_URL = r'''(?x)https?://(?:
-        (?:www\.)?la7\.it/([^/]+)/(?:rivedila7|video|news)/|
-        tg\.la7\.it/repliche-tgla7\?id=
-    )(?P<id>.+)'''
+    _VALID_URL = [
+        r'https?://(?:www\.)?la7\.it/[^/]+/(?:rivedila7|video|news)/.+-(?P<id>\d{5,})',
+        r'https?://tg\.la7\.it/repliche-tgla7\?id=(?P<id>\d{5,})',
+        r'https?://tg\.la7\.it(?:/[^/]+)+-(?P<id>\d{5,})'
+    ]
 
     _TESTS = [{
         # single quality video
@@ -39,7 +40,7 @@ class LA7IE(InfoExtractor):
             'formats': 'count:8',
         },
     }, {
-        'url': 'http://www.la7.it/omnibus/rivedila7/omnibus-news-02-07-2016-189077',
+        'url': 'https://tg.la7.it/repliche-tgla7?id=464601',
         'only_matching': True,
     }]
     _HOST = 'https://awsvodpkg.iltrovatore.it'

yt_dlp/extractor/rai.py

@@ -10,6 +10,7 @@
     filter_dict,
     int_or_none,
     join_nonempty,
+    make_archive_id,
     parse_duration,
     remove_start,
     strip_or_none,
@@ -250,7 +251,10 @@ def _extract_subtitles(url, video_data):
 
 
 class RaiPlayIE(RaiBaseIE):
-    _VALID_URL = rf'(?P<base>https?://(?:www\.)?raiplay\.it/.+?-(?P<id>{RaiBaseIE._UUID_RE}))\.(?:html|json)'
+    _VALID_URL = [
+        rf'(?P<base>https?://(?:www\.)?raiplay\.it/.+?-(?P<id>{RaiBaseIE._UUID_RE}))\.(?:html|json)',
+        r'(?P<base>https?://(?:www\.)?raiplay\.it/dirette/(?P<id>[^/?#&]+))',
+    ]
     _TESTS = [{
         'url': 'https://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
         'md5': '8970abf8caf8aef4696e7b1f2adfc696',
@@ -321,6 +325,25 @@ class RaiPlayIE(RaiBaseIE):
             'timestamp': 1348495020,
             'upload_date': '20120924',
         },
+    }, {
+        # live stream
+        'url': 'https://www.raiplay.it/dirette/rainews24',
+        'info_dict': {
+            'id': 'd784ad40-e0ae-4a69-aa76-37519d238a9c',
+            'display_id': 'rainews24',
+            'ext': 'mp4',
+            'title': r're:^Diretta di Rai News 24 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'md5:4d00bcf6dc98b27c6ec480de329d1497',
+            'uploader': 'Rai News 24',
+            'creator': 'Rai News 24',
+            'is_live': True,
+            'live_status': 'is_live',
+            'upload_date': '20090502',
+            'timestamp': 1241276220,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.raiplay.it/video/2016/11/gazebotraindesi-efebe701-969c-4593-92f3-285f0d1ce750.html?',
         'only_matching': True,
@@ -354,6 +377,7 @@ def _real_extract(self, url):
         return {
             'id': remove_start(media.get('id'), 'ContentItem-') or video_id,
             'display_id': video_id,
+            '_old_archive_ids': [make_archive_id('RaiPlayLive', video_id)] if not re.match(RaiBaseIE._UUID_RE, video_id) else None,
             'title': media.get('name'),
             'alt_title': strip_or_none(alt_title or None),
             'description': media.get('description'),
@@ -377,28 +401,6 @@ def _real_extract(self, url):
         }
 
 
-class RaiPlayLiveIE(RaiPlayIE):  # XXX: Do not subclass from concrete IE
-    _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplay\.it/dirette/(?P<id>[^/?#&]+))'
-    _TESTS = [{
-        'url': 'http://www.raiplay.it/dirette/rainews24',
-        'info_dict': {
-            'id': 'd784ad40-e0ae-4a69-aa76-37519d238a9c',
-            'display_id': 'rainews24',
-            'ext': 'mp4',
-            'title': 're:^Diretta di Rai News 24 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
-            'description': 'md5:4d00bcf6dc98b27c6ec480de329d1497',
-            'uploader': 'Rai News 24',
-            'creator': 'Rai News 24',
-            'is_live': True,
-            'live_status': 'is_live',
-            'upload_date': '20090502',
-            'timestamp': 1241276220,
-            'formats': 'count:3',
-        },
-        'params': {'skip_download': True},
-    }]
-
-
 class RaiPlayPlaylistIE(InfoExtractor):
     _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplay\.it/programmi/(?P<id>[^/?#&]+))(?:/(?P<extra_id>[^?#&]+))?'
     _TESTS = [{
@@ -463,7 +465,10 @@ def _real_extract(self, url):
 
 
 class RaiPlaySoundIE(RaiBaseIE):
-    _VALID_URL = rf'(?P<base>https?://(?:www\.)?raiplaysound\.it/.+?-(?P<id>{RaiBaseIE._UUID_RE}))\.(?:html|json)'
+    _VALID_URL = [
+        rf'(?P<base>https?://(?:www\.)?raiplaysound\.it/.+?-(?P<id>{RaiBaseIE._UUID_RE}))\.(?:html|json)',
+        r'(?P<base>https?://(?:www\.)?raiplaysound\.it/(?P<id>[^/?#&]+)$)',
+    ]
     _TESTS = [{
         'url': 'https://www.raiplaysound.it/audio/2021/12/IL-RUGGITO-DEL-CONIGLIO-1ebae2a7-7cdb-42bb-842e-fe0d193e9707.html',
         'md5': '8970abf8caf8aef4696e7b1f2adfc696',
@@ -482,7 +487,26 @@ class RaiPlaySoundIE(RaiBaseIE):
             'timestamp': 1638346620,
             'upload_date': '20211201',
         },
-        'params': {'skip_download': True},
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://www.raiplaysound.it/radio2',
+        'info_dict': {
+            'id': 'b00a50e6-f404-4af6-8f8c-ff3b9af73a44',
+            'display_id': 'radio2',
+            'ext': 'mp4',
+            'title': r're:Rai Radio 2 \d+-\d+-\d+ \d+:\d+',
+            'thumbnail': r're:https://www\.raiplaysound\.it/dl/img/.+png',
+            'uploader': 'rai radio 2',
+            'series': 'Rai Radio 2',
+            'creator': 'raiplaysound',
+            'is_live': True,
+            'live_status': 'is_live',
+        },
+        'params': {
+            'skip_download': 'live',
+        },
     }]
 
     def _real_extract(self, url):
@@ -506,6 +530,7 @@ def _real_extract(self, url):
             **info,
             'id': uid or audio_id,
             'display_id': audio_id,
+            '_old_archive_ids': [make_archive_id('RaiPlaySoundLive', audio_id)] if not re.match(RaiBaseIE._UUID_RE, audio_id) else None,
             'title': traverse_obj(media, 'title', 'episode_title'),
             'alt_title': traverse_obj(media, ('track_info', 'media_name'), expected_type=strip_or_none),
             'description': media.get('description'),
@@ -521,26 +546,6 @@ def _real_extract(self, url):
         }
 
 
-class RaiPlaySoundLiveIE(RaiPlaySoundIE):  # XXX: Do not subclass from concrete IE
-    _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplaysound\.it/(?P<id>[^/?#&]+)$)'
-    _TESTS = [{
-        'url': 'https://www.raiplaysound.it/radio2',
-        'info_dict': {
-            'id': 'b00a50e6-f404-4af6-8f8c-ff3b9af73a44',
-            'display_id': 'radio2',
-            'ext': 'mp4',
-            'title': r're:Rai Radio 2 \d+-\d+-\d+ \d+:\d+',
-            'thumbnail': r're:^https://www\.raiplaysound\.it/dl/img/.+\.png',
-            'uploader': 'rai radio 2',
-            'series': 'Rai Radio 2',
-            'creator': 'raiplaysound',
-            'is_live': True,
-            'live_status': 'is_live',
-        },
-        'params': {'skip_download': True},
-    }]
-
-
 class RaiPlaySoundPlaylistIE(InfoExtractor):
     _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplaysound\.it/(?:programmi|playlist|audiolibri)/(?P<id>[^/?#&]+))(?:/(?P<extra_id>[^?#&]+))?'
     _TESTS = [{

yt_dlp/extractor/stripchat.py

@@ -28,24 +28,21 @@ class StripchatIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id, headers=self.geo_verification_headers())
+        data = self._search_json(
+            r'<script\b[^>]*>\s*window\.__PRELOADED_STATE__\s*=',
+            webpage, 'data', video_id, transform_source=lowercase_escape)
 
-        data = self._parse_json(
-            self._search_regex(
-                r'<script\b[^>]*>\s*window\.__PRELOADED_STATE__\s*=(?P<value>.*?)<\/script>',
-                webpage, 'data', default='{}', group='value'),
-            video_id, transform_source=lowercase_escape, fatal=False)
-        if not data:
-            raise ExtractorError('Unable to find configuration for stream.')
-
-        if traverse_obj(data, ('viewCam', 'show'), expected_type=dict):
-            raise ExtractorError('Model is in private show', expected=True)
-        elif not traverse_obj(data, ('viewCam', 'model', 'isLive'), expected_type=bool):
+        if traverse_obj(data, ('viewCam', 'show', {dict})):
+            raise ExtractorError('Model is in a private show', expected=True)
+        if not traverse_obj(data, ('viewCam', 'model', 'isLive', {bool})):
             raise UserNotLive(video_id=video_id)
 
-        model_id = traverse_obj(data, ('viewCam', 'model', 'id'), expected_type=int)
+        model_id = data['viewCam']['model']['id']
 
         formats = []
-        for host in traverse_obj(data, ('config', 'data', (
+        # HLS hosts are currently found in .configV3.static.features.hlsFallback.fallbackDomains[]
+        # The rest of the path is for backwards compatibility and to guard against A/B testing
+        for host in traverse_obj(data, ((('config', 'data'), ('configV3', 'static')), (
                 (('features', 'featuresV2'), 'hlsFallback', 'fallbackDomains', ...), 'hlsStreamHost'))):
             formats = self._extract_m3u8_formats(
                 f'https://edge-hls.{host}/hls/{model_id}/master/{model_id}_auto.m3u8',
@@ -53,7 +50,7 @@ def _real_extract(self, url):
             if formats:
                 break
         if not formats:
-            self.raise_no_formats('No active streams found', expected=True)
+            self.raise_no_formats('Unable to extract stream host', video_id=video_id)
 
         return {
             'id': video_id,

yt_dlp/extractor/youtube.py

@@ -4986,6 +4986,10 @@ def _grid_entries(self, grid_renderer):
         for item in grid_renderer['items']:
             if not isinstance(item, dict):
                 continue
+            if lockup_view_model := traverse_obj(item, ('lockupViewModel', {dict})):
+                if entry := self._extract_lockup_view_model(lockup_view_model):
+                    yield entry
+                continue
             renderer = self._extract_basic_item_renderer(item)
             if not isinstance(renderer, dict):
                 continue
@@ -5084,10 +5088,30 @@ def _playlist_entries(self, video_list_renderer):
                 continue
             yield self._extract_video(renderer)
 
+    def _extract_lockup_view_model(self, view_model):
+        content_id = view_model.get('contentId')
+        if not content_id:
+            return
+        content_type = view_model.get('contentType')
+        if content_type not in ('LOCKUP_CONTENT_TYPE_PLAYLIST', 'LOCKUP_CONTENT_TYPE_PODCAST'):
+            self.report_warning(
+                f'Unsupported lockup view model content type "{content_type}"{bug_reports_message()}', only_once=True)
+            return
+        return self.url_result(
+            f'https://www.youtube.com/playlist?list={content_id}', ie=YoutubeTabIE, video_id=content_id,
+            title=traverse_obj(view_model, (
+                'metadata', 'lockupMetadataViewModel', 'title', 'content', {str})),
+            thumbnails=self._extract_thumbnails(view_model, (
+                'contentImage', 'collectionThumbnailViewModel', 'primaryThumbnail', 'thumbnailViewModel', 'image'), final_key='sources'))
+
     def _rich_entries(self, rich_grid_renderer):
+        if lockup_view_model := traverse_obj(rich_grid_renderer, ('content', 'lockupViewModel', {dict})):
+            if entry := self._extract_lockup_view_model(lockup_view_model):
+                yield entry
+            return
         renderer = traverse_obj(
             rich_grid_renderer,
-            ('content', ('videoRenderer', 'reelItemRenderer', 'playlistRenderer', 'shortsLockupViewModel', 'lockupViewModel'), any)) or {}
+            ('content', ('videoRenderer', 'reelItemRenderer', 'playlistRenderer', 'shortsLockupViewModel'), any)) or {}
         video_id = renderer.get('videoId')
         if video_id:
             yield self._extract_video(renderer)
@@ -5114,18 +5138,6 @@ def _rich_entries(self, rich_grid_renderer):
                 })),
                 thumbnails=self._extract_thumbnails(renderer, 'thumbnail', final_key='sources'))
             return
-        # lockupViewModel extraction
-        content_id = renderer.get('contentId')
-        if content_id and renderer.get('contentType') == 'LOCKUP_CONTENT_TYPE_PODCAST':
-            yield self.url_result(
-                f'https://www.youtube.com/playlist?list={content_id}',
-                ie=YoutubeTabIE, video_id=content_id,
-                **traverse_obj(renderer, {
-                    'title': ('metadata', 'lockupMetadataViewModel', 'title', 'content', {str}),
-                }),
-                thumbnails=self._extract_thumbnails(renderer, (
-                    'contentImage', 'collectionThumbnailViewModel', 'primaryThumbnail', 'thumbnailViewModel', 'image'), final_key='sources'))
-            return
 
     def _video_entry(self, video_renderer):
         video_id = video_renderer.get('videoId')
@@ -5794,7 +5806,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
         'info_dict': {
             'id': 'UCYO_jab_esuFRV4b17AJtAw',
             'title': '3Blue1Brown - Playlists',
-            'description': 'md5:4d1da95432004b7ba840ebc895b6b4c9',
+            'description': 'md5:602e3789e6a0cb7d9d352186b720e395',
             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
             'channel': '3Blue1Brown',
             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',