From 97f4aecfc1c5fd446e1d3edd37e49aafe246fe0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 16 Jul 2015 01:14:08 +0600 Subject: [PATCH 1/6] [extractor/common] Handle malformed f4m manifests --- youtube_dl/extractor/common.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 3a396c0b0a..f8a5eccedd 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -28,6 +28,7 @@ clean_html, compiled_regex_type, ExtractorError, + fix_xml_ampersands, float_or_none, int_or_none, RegexNotFoundError, @@ -837,7 +838,10 @@ def _sleep(self, timeout, video_id, msg_template=None): def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None): manifest = self._download_xml( manifest_url, video_id, 'Downloading f4m manifest', - 'Unable to download f4m manifest') + 'Unable to download f4m manifest', + # Some manifests may be malformed, e.g. prosiebensat1 generated manifests + # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244) + transform_source=lambda s: fix_xml_ampersands(s).strip()) formats = [] manifest_version = '1.0' From cc357c4db8112ff6736a227b47fb9527d327797f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 16 Jul 2015 01:14:52 +0600 Subject: [PATCH 2/6] [extractor/common] Properly handle full URLs --- youtube_dl/extractor/common.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index f8a5eccedd..78e5cf8d09 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -851,8 +851,10 @@ def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=N media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media') for i, media_el in enumerate(media_nodes): if manifest_version == '2.0': - manifest_url = ('/'.join(manifest_url.split('/')[:-1]) + '/' + - (media_el.attrib.get('href') or media_el.attrib.get('url'))) + media_url = media_el.attrib.get('href') or media_el.attrib['url'] + manifest_url = ( + media_url if media_url.startswith('http://') or media_url.startswith('https://') + else ('/'.join(manifest_url.split('/')[:-1]) + '/' + media_url)) tbr = int_or_none(media_el.attrib.get('bitrate')) formats.append({ 'format_id': '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)])), From 70f0f5a8ca53d4426fc079b3ab46e9d4a8e81ea4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 16 Jul 2015 01:15:15 +0600 Subject: [PATCH 3/6] [extractor/common] Recursively extract child f4m manifests --- youtube_dl/extractor/common.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 78e5cf8d09..e3c610aa42 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -27,6 +27,7 @@ bug_reports_message, clean_html, compiled_regex_type, + determine_ext, ExtractorError, fix_xml_ampersands, float_or_none, @@ -855,6 +856,13 @@ def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=N manifest_url = ( media_url if media_url.startswith('http://') or media_url.startswith('https://') else ('/'.join(manifest_url.split('/')[:-1]) + '/' + media_url)) + # If media_url is itself a f4m manifest do the recursive extraction + # since bitrates in parent manifest (this one) and media_url manifest + # may differ leading to inability to resolve the format by requested + # bitrate in f4m downloader + if determine_ext(manifest_url) == 'f4m': + formats.extend(self._extract_f4m_formats(manifest_url, video_id, preference, f4m_id)) + continue tbr = int_or_none(media_el.attrib.get('bitrate')) formats.append({ 'format_id': '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)])), From f01f731107010e0c10fc94782daa7a3ba543e92a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 16 Jul 2015 01:15:47 +0600 Subject: [PATCH 4/6] [prosiebensat1] Use generic f4m manifest extraction --- youtube_dl/extractor/prosiebensat1.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py index 22efa903fa..2f9d958009 100644 --- a/youtube_dl/extractor/prosiebensat1.py +++ b/youtube_dl/extractor/prosiebensat1.py @@ -9,7 +9,7 @@ compat_urllib_parse, ) from ..utils import ( - fix_xml_ampersands, + determine_ext, int_or_none, unified_strdate, ) @@ -295,15 +295,8 @@ def fix_bitrate(bitrate): 'ext': 'mp4', 'format_id': '%s_%s' % (source['cdn'], source['bitrate']), }) - elif 'f4mgenerator' in source_url: - manifest = self._download_xml( - source_url, clip_id, 'Downloading generated f4m manifest', - transform_source=lambda s: fix_xml_ampersands(s).strip()) - for media in manifest.findall('./{http://ns.adobe.com/f4m/2.0}media'): - manifest_url = media.get('href') - if manifest_url: - formats.extend(self._extract_f4m_formats( - manifest_url, clip_id, f4m_id='hds')) + elif 'f4mgenerator' in source_url or determine_ext(source_url) == 'f4m': + formats.extend(self._extract_f4m_formats(source_url, clip_id)) else: formats.append({ 'url': source_url, From 31c746e5dc46491f997eca757c5e35842f04cb59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 16 Jul 2015 01:25:33 +0600 Subject: [PATCH 5/6] [extractor/common] Keep going in some media_url is missing --- youtube_dl/extractor/common.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index e3c610aa42..271bf85968 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -852,7 +852,9 @@ def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=N media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media') for i, media_el in enumerate(media_nodes): if manifest_version == '2.0': - media_url = media_el.attrib.get('href') or media_el.attrib['url'] + media_url = media_el.attrib.get('href') or media_el.attrib.get('url') + if not media_url: + continue manifest_url = ( media_url if media_url.startswith('http://') or media_url.startswith('https://') else ('/'.join(manifest_url.split('/')[:-1]) + '/' + media_url)) From ab9b890b524a49a9ffa4c8ac7243cd8afc15d270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 16 Jul 2015 02:23:07 +0600 Subject: [PATCH 6/6] [prosiebensat1] Clarify test purpose --- youtube_dl/extractor/prosiebensat1.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py index 2f9d958009..fec008ce76 100644 --- a/youtube_dl/extractor/prosiebensat1.py +++ b/youtube_dl/extractor/prosiebensat1.py @@ -22,6 +22,11 @@ class ProSiebenSat1IE(InfoExtractor): _TESTS = [ { + # Tests changes introduced in https://github.com/rg3/youtube-dl/pull/6242 + # in response to fixing https://github.com/rg3/youtube-dl/issues/6215: + # - malformed f4m manifest support + # - proper handling of URLs starting with `https?://` in 2.0 manifests + # - recursive child f4m manifests extraction 'url': 'http://www.prosieben.de/tv/circus-halligalli/videos/218-staffel-2-episode-18-jahresrueckblick-ganze-folge', 'info_dict': { 'id': '2104602',