From e37c932fca29d93af77b7a47cccc9bb8578e3163 Mon Sep 17 00:00:00 2001 From: fnord Date: Wed, 15 Jul 2015 15:13:56 -0500 Subject: [PATCH 01/10] compat_urllib_parse_unquote: crash fix: only decode valid hex on python 2 the following has a { "crash_rate": "100%" } of the time as it tries to parse '" ' as hex. --- youtube_dl/compat.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index c3783337a5..1f4ccf4432 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -94,6 +94,8 @@ def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'): try: if not item: raise ValueError + if not re.match('[0-9a-fA-F][0-9a-fA-F]',item[:2]): + raise ValueError pct_sequence += item[:2].decode('hex') rest = item[2:] if not rest: From 45eedbe58c8ab6344f11f1e1376d01648c1967ee Mon Sep 17 00:00:00 2001 From: fnord Date: Wed, 15 Jul 2015 15:30:47 -0500 Subject: [PATCH 02/10] Generic: use compat_urllib_parse_unquote to prevent utf8 mangling of the entire page in python 2. -requires- fixed compat_urllib_parse_unquote example - the following will save with a mangled playlist title, instead of the kanji for 'tsunami'. This affects all utf8encoded urls as well youtube-dl -f18 -o '%(playlist_title)s-%(title)s.%(ext)s' \ https://gist.githubusercontent.com/atomicdryad/fcb97465e6060fc519e1/raw/61c14c1e3a4985471dcf56c281d24d7e781a4e0e/tsunami.html --- youtube_dl/extractor/generic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 392ad36486..fc1bf2b6e2 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -1115,7 +1115,7 @@ def _real_extract(self, url): # Sometimes embedded video player is hidden behind percent encoding # (e.g. https://github.com/rg3/youtube-dl/issues/2448) # Unescaping the whole page allows to handle those cases in a generic way - webpage = compat_urllib_parse.unquote(webpage) + webpage = compat_urllib_parse_unquote(webpage) # it's tempting to parse this further, but you would # have to take into account all the variations like From c9c854cea7fa5992356dee5eab0d3615b4d40dc6 Mon Sep 17 00:00:00 2001 From: fnord Date: Fri, 17 Jul 2015 01:31:29 -0500 Subject: [PATCH 03/10] replace old compat_urllib_parse_unquote with backport from python3's function * required unquote_to_bytes function ported as well (uses .decode('hex') instead of dynamically populated _hextobyte global) * required implicit conversion to bytes and/or unicode in places due to differing type assumptions in p3 --- youtube_dl/compat.py | 75 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index 1f4ccf4432..2fd2278aa8 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -74,10 +74,81 @@ except ImportError: import BaseHTTPServer as compat_http_server +from pprint import (pprint, pformat) + + +def dprint(fmt): + sys.stderr.write(pformat(fmt) + "\n") + try: from urllib.parse import unquote as compat_urllib_parse_unquote except ImportError: - def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'): + def compat_urllib_parse_unquote_to_bytes(string): + """unquote_to_bytes('abc%20def') -> b'abc def'.""" + # Note: strings are encoded as UTF-8. This is only an issue if it contains + # unescaped non-ASCII characters, which URIs should not. + if not string: + # Is it a string-like object? + string.split + return b'' + if isinstance(string, str): + string = string.encode('utf-8') + # string = encode('utf-8') + + # python3 -> 2: must implicitly convert to bits + bits = bytes(string).split(b'%') + + if len(bits) == 1: + return string + res = [bits[0]] + append = res.append + + for item in bits[1:]: + try: + append(item[:2].decode('hex')) + append(item[2:]) + except: + append(b'%') + append(item) + return b''.join(res) + + compat_urllib_parse_asciire = re.compile('([\x00-\x7f]+)') + + def new_compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'): + """Replace %xx escapes by their single-character equivalent. The optional + encoding and errors parameters specify how to decode percent-encoded + sequences into Unicode characters, as accepted by the bytes.decode() + method. + By default, percent-encoded sequences are decoded with UTF-8, and invalid + sequences are replaced by a placeholder character. + + unquote('abc%20def') -> 'abc def'. + """ + + if '%' not in string: + string.split + return string + if encoding is None: + encoding = 'utf-8' + if errors is None: + errors = 'replace' + + bits = compat_urllib_parse_asciire.split(string) + res = [bits[0]] + append = res.append + for i in range(1, len(bits), 2): + foo = compat_urllib_parse_unquote_to_bytes(bits[i]) + foo = foo.decode(encoding, errors) + append(foo) + + if bits[i + 1]: + bar = bits[i + 1] + if not isinstance(bar, unicode): + bar = bar.decode('utf-8') + append(bar) + return ''.join(res) + + def old_compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'): if string == '': return string res = string.split('%') @@ -114,6 +185,8 @@ def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'): string += pct_sequence.decode(encoding, errors) return string + compat_urllib_parse_unquote = new_compat_urllib_parse_unquote + try: compat_str = unicode # Python 2 except NameError: From 851229a01f34129286a57d46f8a27b9bb5fd9a6b Mon Sep 17 00:00:00 2001 From: fnord Date: Fri, 17 Jul 2015 01:49:55 -0500 Subject: [PATCH 04/10] remove debugprint --- youtube_dl/compat.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index 2fd2278aa8..554e3d5db8 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -74,12 +74,6 @@ except ImportError: import BaseHTTPServer as compat_http_server -from pprint import (pprint, pformat) - - -def dprint(fmt): - sys.stderr.write(pformat(fmt) + "\n") - try: from urllib.parse import unquote as compat_urllib_parse_unquote except ImportError: From a0f28f90fa277d9c00f0305624dea36a20b8066e Mon Sep 17 00:00:00 2001 From: fnord Date: Fri, 17 Jul 2015 01:50:43 -0500 Subject: [PATCH 05/10] remove kebab --- youtube_dl/compat.py | 41 +---------------------------------------- 1 file changed, 1 insertion(+), 40 deletions(-) diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index 554e3d5db8..de9ba2c14e 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -108,7 +108,7 @@ def compat_urllib_parse_unquote_to_bytes(string): compat_urllib_parse_asciire = re.compile('([\x00-\x7f]+)') - def new_compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'): + def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'): """Replace %xx escapes by their single-character equivalent. The optional encoding and errors parameters specify how to decode percent-encoded sequences into Unicode characters, as accepted by the bytes.decode() @@ -142,45 +142,6 @@ def new_compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'): append(bar) return ''.join(res) - def old_compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'): - if string == '': - return string - res = string.split('%') - if len(res) == 1: - return string - if encoding is None: - encoding = 'utf-8' - if errors is None: - errors = 'replace' - # pct_sequence: contiguous sequence of percent-encoded bytes, decoded - pct_sequence = b'' - string = res[0] - for item in res[1:]: - try: - if not item: - raise ValueError - if not re.match('[0-9a-fA-F][0-9a-fA-F]',item[:2]): - raise ValueError - pct_sequence += item[:2].decode('hex') - rest = item[2:] - if not rest: - # This segment was just a single percent-encoded character. - # May be part of a sequence of code units, so delay decoding. - # (Stored in pct_sequence). - continue - except ValueError: - rest = '%' + item - # Encountered non-percent-encoded characters. Flush the current - # pct_sequence. - string += pct_sequence.decode(encoding, errors) + rest - pct_sequence = b'' - if pct_sequence: - # Flush the final pct_sequence - string += pct_sequence.decode(encoding, errors) - return string - - compat_urllib_parse_unquote = new_compat_urllib_parse_unquote - try: compat_str = unicode # Python 2 except NameError: From 9fefc88656eceac13604fd86dfb25dc736ed239a Mon Sep 17 00:00:00 2001 From: fnord Date: Fri, 17 Jul 2015 07:24:07 -0500 Subject: [PATCH 06/10] fix TestCompat test_all_present --- youtube_dl/compat.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index de9ba2c14e..8b4d0287c4 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -451,7 +451,9 @@ def compat_itertools_count(start=0, step=1): 'compat_subprocess_get_DEVNULL', 'compat_urllib_error', 'compat_urllib_parse', + 'compat_urllib_parse_asciire', 'compat_urllib_parse_unquote', + 'compat_urllib_parse_unquote_to_bytes', 'compat_urllib_parse_urlparse', 'compat_urllib_request', 'compat_urlparse', From 593b77064c51c411071e310578b542017b9b2ec8 Mon Sep 17 00:00:00 2001 From: fnord Date: Fri, 17 Jul 2015 09:45:49 -0500 Subject: [PATCH 07/10] Don't forget trailing '%' --- youtube_dl/compat.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index 8b4d0287c4..9e506352fe 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -98,6 +98,9 @@ def compat_urllib_parse_unquote_to_bytes(string): append = res.append for item in bits[1:]: + if item == '': + append(b'%') + continue try: append(item[:2].decode('hex')) append(item[2:]) From 4a632911443f0dbc2384fb82ade85382aeecc8dc Mon Sep 17 00:00:00 2001 From: fnord Date: Fri, 17 Jul 2015 09:46:08 -0500 Subject: [PATCH 08/10] Add tests for compat_urllib_parse_unquote --- test/test_compat.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/test_compat.py b/test/test_compat.py index 1eb454e068..431e6bdf18 100644 --- a/test/test_compat.py +++ b/test/test_compat.py @@ -14,6 +14,7 @@ from youtube_dl.compat import ( compat_getenv, compat_expanduser, + compat_urllib_parse_unquote, ) @@ -42,5 +43,24 @@ def test_all_present(self): dir(youtube_dl.compat))) - set(['unicode_literals']) self.assertEqual(all_names, sorted(present_names)) + def test_compat_urllib_parse_unquote(self): + test_strings = [ + ['''''', ''''''], + ['''津波''', '''%E6%B4%A5%E6%B3%A2'''], + ['''津波''', str('%E6%B4%A5%E6%B3%A2')], + [''' +%%a''', + ''' +%%a'''], + ['''(^◣_◢^)っ︻デ═一 ⇀ ⇀ ⇀ ⇀ ⇀ ↶%I%Break%Things%''', + '''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%86%B6%I%Break%25Things%'''] + ] + for test in test_strings: + strutf = test[0] + strurlenc = test[1] + strurldec = compat_urllib_parse_unquote(strurlenc) + self.assertEqual(strutf, strurldec) + self.assertEqual(strutf, compat_urllib_parse_unquote(strurlenc)) + if __name__ == '__main__': unittest.main() From 55139679261f8c2409ca150906a2693731452a13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 17 Jul 2015 22:58:13 +0600 Subject: [PATCH 09/10] [compat] Simplify and use latest cpython 3 code --- youtube_dl/compat.py | 41 +++++++++++++---------------------------- 1 file changed, 13 insertions(+), 28 deletions(-) diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index 9e506352fe..54ccf1d287 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -75,8 +75,13 @@ import BaseHTTPServer as compat_http_server try: + from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes from urllib.parse import unquote as compat_urllib_parse_unquote -except ImportError: +except ImportError: # Python 2 + # HACK: The following are the correct unquote_to_bytes and unquote + # implementations from cpython 3.4.3's stdlib. Python 2's version + # is apparently broken (see https://github.com/rg3/youtube-dl/pull/6244) + def compat_urllib_parse_unquote_to_bytes(string): """unquote_to_bytes('abc%20def') -> b'abc def'.""" # Note: strings are encoded as UTF-8. This is only an issue if it contains @@ -85,32 +90,22 @@ def compat_urllib_parse_unquote_to_bytes(string): # Is it a string-like object? string.split return b'' - if isinstance(string, str): + if isinstance(string, unicode): string = string.encode('utf-8') - # string = encode('utf-8') - - # python3 -> 2: must implicitly convert to bits - bits = bytes(string).split(b'%') - + bits = string.split(b'%') if len(bits) == 1: return string res = [bits[0]] append = res.append - for item in bits[1:]: - if item == '': - append(b'%') - continue try: - append(item[:2].decode('hex')) + append(compat_urllib_parse._hextochr[item[:2]]) append(item[2:]) - except: + except KeyError: append(b'%') append(item) return b''.join(res) - compat_urllib_parse_asciire = re.compile('([\x00-\x7f]+)') - def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'): """Replace %xx escapes by their single-character equivalent. The optional encoding and errors parameters specify how to decode percent-encoded @@ -121,7 +116,6 @@ def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'): unquote('abc%20def') -> 'abc def'. """ - if '%' not in string: string.split return string @@ -129,20 +123,12 @@ def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'): encoding = 'utf-8' if errors is None: errors = 'replace' - - bits = compat_urllib_parse_asciire.split(string) + bits = compat_urllib_parse._asciire.split(string) res = [bits[0]] append = res.append for i in range(1, len(bits), 2): - foo = compat_urllib_parse_unquote_to_bytes(bits[i]) - foo = foo.decode(encoding, errors) - append(foo) - - if bits[i + 1]: - bar = bits[i + 1] - if not isinstance(bar, unicode): - bar = bar.decode('utf-8') - append(bar) + append(compat_urllib_parse_unquote_to_bytes(bits[i]).decode(encoding, errors)) + append(bits[i + 1]) return ''.join(res) try: @@ -454,7 +440,6 @@ def compat_itertools_count(start=0, step=1): 'compat_subprocess_get_DEVNULL', 'compat_urllib_error', 'compat_urllib_parse', - 'compat_urllib_parse_asciire', 'compat_urllib_parse_unquote', 'compat_urllib_parse_unquote_to_bytes', 'compat_urllib_parse_urlparse', From 14309e1ddc476a7e2fc444a0443b2fc23186a385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 17 Jul 2015 22:58:39 +0600 Subject: [PATCH 10/10] [test_compat] Make tests more idiomatic --- test/test_compat.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/test/test_compat.py b/test/test_compat.py index 431e6bdf18..2ffbc2c488 100644 --- a/test/test_compat.py +++ b/test/test_compat.py @@ -44,23 +44,22 @@ def test_all_present(self): self.assertEqual(all_names, sorted(present_names)) def test_compat_urllib_parse_unquote(self): - test_strings = [ - ['''''', ''''''], - ['''津波''', '''%E6%B4%A5%E6%B3%A2'''], - ['''津波''', str('%E6%B4%A5%E6%B3%A2')], - [''' -%%a''', - ''' -%%a'''], - ['''(^◣_◢^)っ︻デ═一 ⇀ ⇀ ⇀ ⇀ ⇀ ↶%I%Break%Things%''', - '''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%86%B6%I%Break%25Things%'''] - ] - for test in test_strings: - strutf = test[0] - strurlenc = test[1] - strurldec = compat_urllib_parse_unquote(strurlenc) - self.assertEqual(strutf, strurldec) - self.assertEqual(strutf, compat_urllib_parse_unquote(strurlenc)) + self.assertEqual(compat_urllib_parse_unquote(''), '') + self.assertEqual(compat_urllib_parse_unquote('%'), '%') + self.assertEqual(compat_urllib_parse_unquote('%%'), '%%') + self.assertEqual(compat_urllib_parse_unquote('%%%'), '%%%') + self.assertEqual(compat_urllib_parse_unquote('%2F'), '/') + self.assertEqual(compat_urllib_parse_unquote('%2f'), '/') + self.assertEqual(compat_urllib_parse_unquote('%E6%B4%A5%E6%B3%A2'), '津波') + self.assertEqual(compat_urllib_parse_unquote(str('%E6%B4%A5%E6%B3%A2')), '津波') + self.assertEqual( + compat_urllib_parse_unquote(''' +%%a'''), + ''' +%%a''') + self.assertEqual( + compat_urllib_parse_unquote('''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%86%B6%I%Break%25Things%'''), + '''(^◣_◢^)っ︻デ═一 ⇀ ⇀ ⇀ ⇀ ⇀ ↶%I%Break%Things%''') if __name__ == '__main__': unittest.main()