From 7d0f6f0c4527aa1c1f99984c5b34d21ebc87228d Mon Sep 17 00:00:00 2001
From: Burve <aleksandrs.ivancenko@gmail.com>
Date: Tue, 26 Jul 2022 06:41:52 +0300
Subject: [PATCH] [extractor/Crunchyroll] Handle missing metadata correctly
 (#4405)

Closes #4399

Authored by pukkandan, Burve
---
 yt_dlp/extractor/crunchyroll.py | 58 ++++++++++++++-------------------
 1 file changed, 24 insertions(+), 34 deletions(-)

diff --git a/yt_dlp/extractor/crunchyroll.py b/yt_dlp/extractor/crunchyroll.py
index d5aa45ff8e..9dda53c68f 100644
--- a/yt_dlp/extractor/crunchyroll.py
+++ b/yt_dlp/extractor/crunchyroll.py
@@ -813,56 +813,36 @@ def _real_extract(self, url):
 
         episode_response = self._download_json(
             f'{api_domain}/cms/v2{bucket}/episodes/{internal_id}', display_id,
-            note='Retrieving episode metadata',
-            query=params)
+            note='Retrieving episode metadata', query=params)
         if episode_response.get('is_premium_only') and not episode_response.get('playback'):
             raise ExtractorError('This video is for premium members only.', expected=True)
-        stream_response = self._download_json(
-            episode_response['playback'], display_id,
-            note='Retrieving stream info')
 
-        thumbnails = []
-        for thumbnails_data in traverse_obj(episode_response, ('images', 'thumbnail')):
-            for thumbnail_data in thumbnails_data:
-                thumbnails.append({
-                    'url': thumbnail_data.get('source'),
-                    'width': thumbnail_data.get('width'),
-                    'height': thumbnail_data.get('height'),
-                })
-        subtitles = {}
-        for lang, subtitle_data in stream_response.get('subtitles').items():
-            subtitles[lang] = [{
-                'url': subtitle_data.get('url'),
-                'ext': subtitle_data.get('format')
-            }]
+        stream_response = self._download_json(episode_response['playback'], display_id, note='Retrieving stream info')
+        get_streams = lambda name: (traverse_obj(stream_response, name) or {}).items()
 
         requested_hardsubs = [('' if val == 'none' else val) for val in (self._configuration_arg('hardsub') or ['none'])]
         hardsub_preference = qualities(requested_hardsubs[::-1])
         requested_formats = self._configuration_arg('format') or ['adaptive_hls']
 
         formats = []
-        for stream_type, streams in stream_response.get('streams', {}).items():
+        for stream_type, streams in get_streams('streams'):
             if stream_type not in requested_formats:
                 continue
             for stream in streams.values():
                 hardsub_lang = stream.get('hardsub_locale') or ''
                 if hardsub_lang.lower() not in requested_hardsubs:
                     continue
-                format_id = join_nonempty(
-                    stream_type,
-                    format_field(stream, 'hardsub_locale', 'hardsub-%s'))
+                format_id = join_nonempty(stream_type, format_field(stream, 'hardsub_locale', 'hardsub-%s'))
                 if not stream.get('url'):
                     continue
-                if stream_type.split('_')[-1] == 'hls':
+                if stream_type.endswith('hls'):
                     adaptive_formats = self._extract_m3u8_formats(
                         stream['url'], display_id, 'mp4', m3u8_id=format_id,
-                        note='Downloading %s information' % format_id,
-                        fatal=False)
-                elif stream_type.split('_')[-1] == 'dash':
+                        fatal=False, note=f'Downloading {format_id} HLS manifest')
+                elif stream_type.endswith('dash'):
                     adaptive_formats = self._extract_mpd_formats(
                         stream['url'], display_id, mpd_id=format_id,
-                        note='Downloading %s information' % format_id,
-                        fatal=False)
+                        fatal=False, note=f'Downloading {format_id} MPD manifest')
                 for f in adaptive_formats:
                     if f.get('acodec') != 'none':
                         f['language'] = stream_response.get('audio_locale')
@@ -872,10 +852,10 @@ def _real_extract(self, url):
 
         return {
             'id': internal_id,
-            'title': '%s Episode %s – %s' % (episode_response.get('season_title'), episode_response.get('episode'), episode_response.get('title')),
-            'description': episode_response.get('description').replace(r'\r\n', '\n'),
+            'title': '%s Episode %s – %s' % (
+                episode_response.get('season_title'), episode_response.get('episode'), episode_response.get('title')),
+            'description': try_get(episode_response, lambda x: x['description'].replace(r'\r\n', '\n')),
             'duration': float_or_none(episode_response.get('duration_ms'), 1000),
-            'thumbnails': thumbnails,
             'series': episode_response.get('series_title'),
             'series_id': episode_response.get('series_id'),
             'season': episode_response.get('season_title'),
@@ -883,8 +863,18 @@ def _real_extract(self, url):
             'season_number': episode_response.get('season_number'),
             'episode': episode_response.get('title'),
             'episode_number': episode_response.get('sequence_number'),
-            'subtitles': subtitles,
-            'formats': formats
+            'formats': formats,
+            'thumbnails': [{
+                'url': thumb.get('source'),
+                'width': thumb.get('width'),
+                'height': thumb.get('height'),
+            } for thumb in traverse_obj(episode_response, ('images', 'thumbnail', ..., ...)) or []],
+            'subtitles': {
+                lang: [{
+                    'url': subtitle_data.get('url'),
+                    'ext': subtitle_data.get('format')
+                }] for lang, subtitle_data in get_streams('subtitles')
+            },
         }