From a047eeb6d2cbbee944e5088f7cba111746f4bb0b Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 5 Aug 2021 03:01:23 +0530 Subject: [PATCH] Add regex to `--match-filter` This does not fully deprecate `--match-title`/`--reject-title` since `--match-filter` is only checked after the extraction is complete, while `--match-title` can often be checked from the flat playlist. Fixes: https://github.com/ytdl-org/youtube-dl/issues/9092, https://github.com/ytdl-org/youtube-dl/issues/23035 --- README.md | 27 ++++++++++++----------- test/test_utils.py | 54 +++++++++++++++++++++++++++++++++++----------- yt_dlp/options.py | 15 +++++++------ yt_dlp/utils.py | 29 ++++++++++++++----------- 4 files changed, 82 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 1967d216cf..a308aa196f 100644 --- a/README.md +++ b/README.md @@ -340,19 +340,22 @@ ## Video Selection: COUNT views --match-filter FILTER Generic video filter. Any field (see "OUTPUT TEMPLATE") can be compared with a - number or a quoted string using the - operators defined in "Filtering formats". - You can also simply specify a field to - match if the field is present and "!field" - to check if the field is not present. - Multiple filters can be checked using "&". - For example, to only match videos that are - not live, has a like count more than 100, a - dislike count less than 50 (or the dislike + number or a string using the operators + defined in "Filtering formats". You can + also simply specify a field to match if the + field is present and "!field" to check if + the field is not present. In addition, + Python style regular expression matching + can be done using "~=", and multiple + filters can be checked with "&". Use a "\" + to escape "&" or quotes if needed. Eg: + --match-filter "!is_live & like_count>?100 + & description~=\'(?i)\bcats \& dogs\b\'" + matches only videos that are not live, has + a like count more than 100 (or the like field is not available), and also has a - description that contains "python", use - --match-filter "!is_live & like_count>100 & - dislike_count0', {'x': 0})) self.assertFalse(match_str('x>0', {})) self.assertTrue(match_str('x>?0', {})) @@ -1219,6 +1234,8 @@ def test_match_str(self): self.assertFalse(match_str('x>2K', {'x': 1200})) self.assertTrue(match_str('x>=1200 & x < 1300', {'x': 1200})) self.assertFalse(match_str('x>=1100 & x < 1200', {'x': 1200})) + + # String self.assertFalse(match_str('y=a212', {'y': 'foobar42'})) self.assertTrue(match_str('y=foobar42', {'y': 'foobar42'})) self.assertFalse(match_str('y!=foobar42', {'y': 'foobar42'})) @@ -1234,6 +1251,8 @@ def test_match_str(self): self.assertTrue(match_str('y!*=baz', {'y': 'foobar42'})) self.assertTrue(match_str('y$=42', {'y': 'foobar42'})) self.assertFalse(match_str('y$=43', {'y': 'foobar42'})) + + # And self.assertFalse(match_str( 'like_count > 100 & dislike_count 100 & dislike_count ?100 & description~=\'(?i)\bcats \& dogs\b\'', + {'description': 'Raining Cats & Dogs'}) def test_parse_dfxp_time_expr(self): self.assertEqual(parse_dfxp_time_expr(None), None) diff --git a/yt_dlp/options.py b/yt_dlp/options.py index fba2313829..b5ddbeaff7 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -378,13 +378,14 @@ def _dict_from_options_callback( 'Generic video filter. Any field (see "OUTPUT TEMPLATE") can be compared with a ' 'number or a string using the operators defined in "Filtering formats". ' 'You can also simply specify a field to match if the field is present ' - 'and "!field" to check if the field is not present. ' - 'Multiple filters can be checked using "&". ' - 'For example, to only match videos that are not live, ' - 'has a like count more than 100, a dislike count less than 50 ' - '(or the dislike field is not available), and also has a description ' - 'that contains "python", use --match-filter "!is_live & ' - 'like_count>100 & dislike_count?100 & description~=\'(?i)\bcats \& dogs\b\'" ' + 'matches only videos that are not live, has a like count more than 100 ' + '(or the like field is not available), and also has a description ' + 'that contains the phrase "cats & dogs" (ignoring case)')) selection.add_option( '--no-match-filter', metavar='FILTER', dest='match_filter', action='store_const', const=None, diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index d06b18e005..b04fbd22cf 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -4664,23 +4664,28 @@ def filter_using_list(row, filterArray): def _match_one(filter_part, dct): # TODO: Generalize code with YoutubeDL._build_format_filter - COMPARISON_OPERATORS = { - '<': operator.lt, - '<=': operator.le, - '>': operator.gt, - '>=': operator.ge, - '=': operator.eq, + STRING_OPERATORS = { '*=': operator.contains, '^=': lambda attr, value: attr.startswith(value), '$=': lambda attr, value: attr.endswith(value), + '~=': lambda attr, value: re.search(value, attr), } + COMPARISON_OPERATORS = { + **STRING_OPERATORS, + '<=': operator.le, # "<=" must be defined above "<" + '<': operator.lt, + '>=': operator.ge, + '>': operator.gt, + '=': operator.eq, + } + operator_rex = re.compile(r'''(?x)\s* (?P[a-z_]+) \s*(?P!\s*)?(?P%s)(?P\s*\?)?\s* (?: (?P[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)| - (?P["\'])(?P(?:\\.|(?!(?P=quote)|\\).)+?)(?P=quote)| - (?P(?![0-9.])[a-z0-9A-Z]*) + (?P["\'])(?P.+?)(?P=quote)| + (?P.+?) ) \s*$ ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys()))) @@ -4705,9 +4710,8 @@ def _match_one(filter_part, dct): if quote is not None: comparison_value = comparison_value.replace(r'\%s' % quote, quote) else: - if m.group('op') in ('*=', '^=', '$='): - raise ValueError( - 'Operator %s only supports string values!' % m.group('op')) + if m.group('op') in STRING_OPERATORS: + raise ValueError('Operator %s only supports string values!' % m.group('op')) try: comparison_value = int(m.group('intval')) except ValueError: @@ -4743,7 +4747,8 @@ def match_str(filter_str, dct): """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """ return all( - _match_one(filter_part, dct) for filter_part in filter_str.split('&')) + _match_one(filter_part.replace(r'\&', '&'), dct) + for filter_part in re.split(r'(?