From 47fb5ba64791d57a3ba1fafee6bdba7394f0306f Mon Sep 17 00:00:00 2001 From: Synarp Date: Wed, 12 Jun 2024 01:46:57 +0200 Subject: [PATCH 1/3] workaround for the 1000 item API-limit --- yt_dlp/extractor/lbry.py | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/yt_dlp/extractor/lbry.py b/yt_dlp/extractor/lbry.py index dcb44d07f..e0ffbf479 100644 --- a/yt_dlp/extractor/lbry.py +++ b/yt_dlp/extractor/lbry.py @@ -96,6 +96,30 @@ def _fetch_page(self, display_id, url, params, page): 'url': self._permanent_url(url, item['name'], item['claim_id']), } + def _metapage_entries(self, display_id, url, params): + if 'release_time' in params: + raise ExtractorError('release_time isn\'t allowed because _metapage_entires needs to specify it.') + if not ('order_by' in params and params['order_by'] == ['release_time']): + raise ExtractorError('videos must be sorted by release_time for _metapage_entries to work.') + + last_metapage = [] + metapage = OnDemandPagedList( + functools.partial(self._fetch_page, display_id, url, params), + self._PAGE_SIZE).getslice() + + while len(metapage) > 0: + yield from metapage + + next_metapage_params = { + **params, + "release_time": "<=%s" % metapage[-1]["release_timestamp"] + } + last_metapage = metapage + metapage = OnDemandPagedList( + functools.partial(self._fetch_page, display_id, url, next_metapage_params), + self._PAGE_SIZE).getslice() + metapage = [x for x in metapage if x not in last_metapage] + def _playlist_entries(self, url, display_id, claim_param, metadata): qs = parse_qs(url) content = qs.get('content', [None])[0] @@ -123,9 +147,13 @@ def _playlist_entries(self, url, display_id, claim_param, metadata): languages.append('none') params['any_languages'] = languages - entries = OnDemandPagedList( - functools.partial(self._fetch_page, display_id, url, params), - self._PAGE_SIZE) + if qs.get('order', ['new'])[0] == 'new': + entries = self._metapage_entries(display_id, url, params) + else: + self.report_warning("Extraction is limited to 1000 Videos when not sorting by newest.") + entries = OnDemandPagedList( + functools.partial(self._fetch_page, display_id, url, params), + self._PAGE_SIZE) return self.playlist_result( entries, display_id, **traverse_obj(metadata, ('value', { From 5ea1d902d3eaae12a58630956a0ec02adac1d22b Mon Sep 17 00:00:00 2001 From: Synarp Date: Wed, 12 Jun 2024 02:34:23 +0200 Subject: [PATCH 2/3] fix style issues --- yt_dlp/extractor/lbry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yt_dlp/extractor/lbry.py b/yt_dlp/extractor/lbry.py index e0ffbf479..49a849e68 100644 --- a/yt_dlp/extractor/lbry.py +++ b/yt_dlp/extractor/lbry.py @@ -112,7 +112,7 @@ def _metapage_entries(self, display_id, url, params): next_metapage_params = { **params, - "release_time": "<=%s" % metapage[-1]["release_timestamp"] + 'release_time': '<={}'.format(metapage[-1]['release_timestamp']), } last_metapage = metapage metapage = OnDemandPagedList( @@ -150,7 +150,7 @@ def _playlist_entries(self, url, display_id, claim_param, metadata): if qs.get('order', ['new'])[0] == 'new': entries = self._metapage_entries(display_id, url, params) else: - self.report_warning("Extraction is limited to 1000 Videos when not sorting by newest.") + self.report_warning('Extraction is limited to 1000 Videos when not sorting by newest.') entries = OnDemandPagedList( functools.partial(self._fetch_page, display_id, url, params), self._PAGE_SIZE) From 440238ebc9aac9e958244a7375b5cf30c6d5d57d Mon Sep 17 00:00:00 2001 From: Synarp Date: Wed, 12 Jun 2024 02:45:36 +0200 Subject: [PATCH 3/3] fix displayed page number --- yt_dlp/extractor/lbry.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/yt_dlp/extractor/lbry.py b/yt_dlp/extractor/lbry.py index 49a849e68..753bf600d 100644 --- a/yt_dlp/extractor/lbry.py +++ b/yt_dlp/extractor/lbry.py @@ -78,7 +78,7 @@ def _parse_stream(self, stream, url): return info - def _fetch_page(self, display_id, url, params, page): + def _fetch_page(self, display_id, url, params, metapage_nr, page): page += 1 page_params = { 'no_totals': True, @@ -86,8 +86,12 @@ def _fetch_page(self, display_id, url, params, page): 'page_size': self._PAGE_SIZE, **params, } + if metapage_nr == 0: + resource = f'page {page}' + else: + resource = f'page {metapage_nr+1}_{page}' result = self._call_api_proxy( - 'claim_search', display_id, page_params, f'page {page}') + 'claim_search', display_id, page_params, resource) for item in traverse_obj(result, ('items', lambda _, v: v['name'] and v['claim_id'])): yield { **self._parse_stream(item, url), @@ -104,9 +108,10 @@ def _metapage_entries(self, display_id, url, params): last_metapage = [] metapage = OnDemandPagedList( - functools.partial(self._fetch_page, display_id, url, params), + functools.partial(self._fetch_page, display_id, url, params, 0), self._PAGE_SIZE).getslice() + metapage_nr = 1 while len(metapage) > 0: yield from metapage @@ -116,9 +121,10 @@ def _metapage_entries(self, display_id, url, params): } last_metapage = metapage metapage = OnDemandPagedList( - functools.partial(self._fetch_page, display_id, url, next_metapage_params), + functools.partial(self._fetch_page, display_id, url, next_metapage_params, metapage_nr), self._PAGE_SIZE).getslice() metapage = [x for x in metapage if x not in last_metapage] + metapage_nr += 1 def _playlist_entries(self, url, display_id, claim_param, metadata): qs = parse_qs(url) @@ -152,7 +158,7 @@ def _playlist_entries(self, url, display_id, claim_param, metadata): else: self.report_warning('Extraction is limited to 1000 Videos when not sorting by newest.') entries = OnDemandPagedList( - functools.partial(self._fetch_page, display_id, url, params), + functools.partial(self._fetch_page, display_id, url, params, 0), self._PAGE_SIZE) return self.playlist_result(