xZero707/yt-dlp
1
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2024-11-27 14:14:54 +01:00
Code Issues Packages Projects Releases Wiki Activity

[youtube_live_chat] use clickTrackingParams (#449)

Browse Source 
Authored by: siikamiika
This commit is contained in:
siikamiika 2021-06-27 02:22:32 +03:00 committed by GitHub
parent 1e79316e20
commit c2603313b1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 29 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
yt_dlp/downloader/youtube_live_chat.py
View File

@ -44,7 +44,7 @@ def dl_fragment(url, data=None, headers=None):
return self._download_fragment(ctx, url, info_dict, http_headers, data) return self._download_fragment(ctx, url, info_dict, http_headers, data)
def parse_actions_replay(live_chat_continuation): def parse_actions_replay(live_chat_continuation):
offset = continuation_id = None offset = continuation_id = click_tracking_params = None
processed_fragment = bytearray() processed_fragment = bytearray()
for action in live_chat_continuation.get('actions', []): for action in live_chat_continuation.get('actions', []):
if 'replayChatItemAction' in action: if 'replayChatItemAction' in action:
@ -53,28 +53,34 @@ def parse_actions_replay(live_chat_continuation):
processed_fragment.extend( processed_fragment.extend(
json.dumps(action, ensure_ascii=False).encode('utf-8') + b'\n') json.dumps(action, ensure_ascii=False).encode('utf-8') + b'\n')
if offset is not None: if offset is not None:
continuation_id = try_get( continuation = try_get(
live_chat_continuation, live_chat_continuation,
lambda x: x['continuations'][0]['liveChatReplayContinuationData']['continuation']) lambda x: x['continuations'][0]['liveChatReplayContinuationData'], dict)
if continuation:
continuation_id = continuation.get('continuation')
click_tracking_params = continuation.get('clickTrackingParams')
self._append_fragment(ctx, processed_fragment) self._append_fragment(ctx, processed_fragment)
return continuation_id, offset return continuation_id, offset, click_tracking_params
def try_refresh_replay_beginning(live_chat_continuation): def try_refresh_replay_beginning(live_chat_continuation):
# choose the second option that contains the unfiltered live chat replay # choose the second option that contains the unfiltered live chat replay
refresh_continuation_id = try_get( refresh_continuation = try_get(
live_chat_continuation, live_chat_continuation,
lambda x: x['header']['liveChatHeaderRenderer']['viewSelector']['sortFilterSubMenuRenderer']['subMenuItems'][1]['continuation']['reloadContinuationData']['continuation'], str) lambda x: x['header']['liveChatHeaderRenderer']['viewSelector']['sortFilterSubMenuRenderer']['subMenuItems'][1]['continuation']['reloadContinuationData'], dict)
if refresh_continuation_id: if refresh_continuation:
# no data yet but required to call _append_fragment # no data yet but required to call _append_fragment
self._append_fragment(ctx, b'') self._append_fragment(ctx, b'')
return refresh_continuation_id, 0 refresh_continuation_id = refresh_continuation.get('continuation')
offset = 0
click_tracking_params = refresh_continuation.get('trackingParams')
return refresh_continuation_id, offset, click_tracking_params
return parse_actions_replay(live_chat_continuation) return parse_actions_replay(live_chat_continuation)
live_offset = 0 live_offset = 0
def parse_actions_live(live_chat_continuation): def parse_actions_live(live_chat_continuation):
nonlocal live_offset nonlocal live_offset
continuation_id = None continuation_id = click_tracking_params = None
processed_fragment = bytearray() processed_fragment = bytearray()
for action in live_chat_continuation.get('actions', []): for action in live_chat_continuation.get('actions', []):
timestamp = self.parse_live_timestamp(action) timestamp = self.parse_live_timestamp(action)
@ -95,11 +101,12 @@ def parse_actions_live(live_chat_continuation):
continuation_data = try_get(live_chat_continuation, continuation_data_getters, dict) continuation_data = try_get(live_chat_continuation, continuation_data_getters, dict)
if continuation_data: if continuation_data:
continuation_id = continuation_data.get('continuation') continuation_id = continuation_data.get('continuation')
click_tracking_params = continuation_data.get('clickTrackingParams')
timeout_ms = int_or_none(continuation_data.get('timeoutMs')) timeout_ms = int_or_none(continuation_data.get('timeoutMs'))
if timeout_ms is not None: if timeout_ms is not None:
time.sleep(timeout_ms / 1000) time.sleep(timeout_ms / 1000)
self._append_fragment(ctx, processed_fragment) self._append_fragment(ctx, processed_fragment)
return continuation_id, live_offset return continuation_id, live_offset, click_tracking_params
def download_and_parse_fragment(url, frag_index, request_data=None, headers=None): def download_and_parse_fragment(url, frag_index, request_data=None, headers=None):
count = 0 count = 0
@ -107,7 +114,7 @@ def download_and_parse_fragment(url, frag_index, request_data=None, headers=None
try: try:
success, raw_fragment = dl_fragment(url, request_data, headers) success, raw_fragment = dl_fragment(url, request_data, headers)
if not success: if not success:
return False, None, None return False, None, None, None
try: try:
data = ie._extract_yt_initial_data(video_id, raw_fragment.decode('utf-8', 'replace')) data = ie._extract_yt_initial_data(video_id, raw_fragment.decode('utf-8', 'replace'))
except RegexNotFoundError: except RegexNotFoundError:
@ -119,19 +126,19 @@ def download_and_parse_fragment(url, frag_index, request_data=None, headers=None
lambda x: x['continuationContents']['liveChatContinuation'], dict) or {} lambda x: x['continuationContents']['liveChatContinuation'], dict) or {}
if info_dict['protocol'] == 'youtube_live_chat_replay': if info_dict['protocol'] == 'youtube_live_chat_replay':
if frag_index == 1: if frag_index == 1:
continuation_id, offset = try_refresh_replay_beginning(live_chat_continuation) continuation_id, offset, click_tracking_params = try_refresh_replay_beginning(live_chat_continuation)
else: else:
continuation_id, offset = parse_actions_replay(live_chat_continuation) continuation_id, offset, click_tracking_params = parse_actions_replay(live_chat_continuation)
elif info_dict['protocol'] == 'youtube_live_chat': elif info_dict['protocol'] == 'youtube_live_chat':
continuation_id, offset = parse_actions_live(live_chat_continuation) continuation_id, offset, click_tracking_params = parse_actions_live(live_chat_continuation)
return True, continuation_id, offset return True, continuation_id, offset, click_tracking_params
except compat_urllib_error.HTTPError as err: except compat_urllib_error.HTTPError as err:
count += 1 count += 1
if count <= fragment_retries: if count <= fragment_retries:
self.report_retry_fragment(err, frag_index, count, fragment_retries) self.report_retry_fragment(err, frag_index, count, fragment_retries)
if count > fragment_retries: if count > fragment_retries:
self.report_error('giving up after %s fragment retries' % fragment_retries) self.report_error('giving up after %s fragment retries' % fragment_retries)
return False, None, None return False, None, None, None
self._prepare_and_start_frag_download(ctx) self._prepare_and_start_frag_download(ctx)
@ -165,6 +172,7 @@ def download_and_parse_fragment(url, frag_index, request_data=None, headers=None
chat_page_url = 'https://www.youtube.com/live_chat?continuation=' + continuation_id chat_page_url = 'https://www.youtube.com/live_chat?continuation=' + continuation_id
frag_index = offset = 0 frag_index = offset = 0
click_tracking_params = None
while continuation_id is not None: while continuation_id is not None:
frag_index += 1 frag_index += 1
request_data = { request_data = {
@ -173,13 +181,16 @@ def download_and_parse_fragment(url, frag_index, request_data=None, headers=None
} }
if frag_index > 1: if frag_index > 1:
request_data['currentPlayerState'] = {'playerOffsetMs': str(max(offset - 5000, 0))} request_data['currentPlayerState'] = {'playerOffsetMs': str(max(offset - 5000, 0))}
if click_tracking_params:
request_data['context']['clickTracking'] = {'clickTrackingParams': click_tracking_params}
headers = ie._generate_api_headers(ytcfg, visitor_data=visitor_data) headers = ie._generate_api_headers(ytcfg, visitor_data=visitor_data)
headers.update({'content-type': 'application/json'}) headers.update({'content-type': 'application/json'})
fragment_request_data = json.dumps(request_data, ensure_ascii=False).encode('utf-8') + b'\n' fragment_request_data = json.dumps(request_data, ensure_ascii=False).encode('utf-8') + b'\n'
success, continuation_id, offset = download_and_parse_fragment( success, continuation_id, offset, click_tracking_params = download_and_parse_fragment(
url, frag_index, fragment_request_data, headers) url, frag_index, fragment_request_data, headers)
else: else:
success, continuation_id, offset = download_and_parse_fragment(chat_page_url, frag_index) success, continuation_id, offset, click_tracking_params = download_and_parse_fragment(
chat_page_url, frag_index)
if not success: if not success:
return False return False
if test: if test: