From 836e06d246512f286f30c1371b2c54b72c9ecd93 Mon Sep 17 00:00:00 2001 From: Simon Sawicki Date: Sun, 17 Sep 2023 12:56:50 +0200 Subject: [PATCH] [core] Fix support for upcoming Python 3.12 (#8130) This also adds the following test runners: - `3.12-dev` on `ubuntu-latest` - `3.12-dev` on `windows-latest` - `pypy-3.10` on `ubuntu-latest` Authored by: Grub4K --- .github/workflows/core.yml | 5 ++++- devscripts/update-version.py | 4 ++-- yt_dlp/YoutubeDL.py | 2 +- yt_dlp/extractor/aws.py | 2 +- yt_dlp/extractor/goplay.py | 4 ++-- yt_dlp/extractor/motherless.py | 2 +- yt_dlp/extractor/panopto.py | 4 ++-- yt_dlp/networking/_urllib.py | 2 +- yt_dlp/networking/exceptions.py | 2 +- yt_dlp/utils/_utils.py | 12 ++++++++---- 10 files changed, 23 insertions(+), 16 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index dead444c0b..689408c500 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -13,13 +13,16 @@ jobs: matrix: os: [ubuntu-latest] # CPython 3.11 is in quick-test - python-version: ['3.8', '3.9', '3.10', pypy-3.7, pypy-3.8] + python-version: ['3.8', '3.9', '3.10', '3.12-dev', pypy-3.7, pypy-3.8, pypy-3.10] run-tests-ext: [sh] include: # atleast one of each CPython/PyPy tests must be in windows - os: windows-latest python-version: '3.7' run-tests-ext: bat + - os: windows-latest + python-version: '3.12-dev' + run-tests-ext: bat - os: windows-latest python-version: pypy-3.9 run-tests-ext: bat diff --git a/devscripts/update-version.py b/devscripts/update-version.py index c873d10a5d..0144bd284a 100644 --- a/devscripts/update-version.py +++ b/devscripts/update-version.py @@ -10,14 +10,14 @@ import argparse import contextlib import sys -from datetime import datetime +from datetime import datetime, timezone from devscripts.utils import read_version, run_process, write_file def get_new_version(version, revision): if not version: - version = datetime.utcnow().strftime('%Y.%m.%d') + version = datetime.now(timezone.utc).strftime('%Y.%m.%d') if revision: assert revision.isdigit(), 'Revision must be a number' diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 666d89b461..1feed30524 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -2591,7 +2591,7 @@ def _fill_common_fields(self, info_dict, final=True): # Working around out-of-range timestamp values (e.g. negative ones on Windows, # see http://bugs.python.org/issue1646728) with contextlib.suppress(ValueError, OverflowError, OSError): - upload_date = datetime.datetime.utcfromtimestamp(info_dict[ts_key]) + upload_date = datetime.datetime.fromtimestamp(info_dict[ts_key], datetime.timezone.utc) info_dict[date_key] = upload_date.strftime('%Y%m%d') live_keys = ('is_live', 'was_live') diff --git a/yt_dlp/extractor/aws.py b/yt_dlp/extractor/aws.py index eb831a1530..c4741a6a11 100644 --- a/yt_dlp/extractor/aws.py +++ b/yt_dlp/extractor/aws.py @@ -12,7 +12,7 @@ class AWSIE(InfoExtractor): # XXX: Conventionally, base classes should end with def _aws_execute_api(self, aws_dict, video_id, query=None): query = query or {} - amz_date = datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%SZ') + amz_date = datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%dT%H%M%SZ') date = amz_date[:8] headers = { 'Accept': 'application/json', diff --git a/yt_dlp/extractor/goplay.py b/yt_dlp/extractor/goplay.py index 960d7d7bc0..0a3c8340f1 100644 --- a/yt_dlp/extractor/goplay.py +++ b/yt_dlp/extractor/goplay.py @@ -383,9 +383,9 @@ def __get_current_timestamp(): months = [None, 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] days = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] - time_now = datetime.datetime.utcnow() + time_now = datetime.datetime.now(datetime.timezone.utc) format_string = "{} {} {} %H:%M:%S UTC %Y".format(days[time_now.weekday()], months[time_now.month], time_now.day) - time_string = datetime.datetime.utcnow().strftime(format_string) + time_string = time_now.strftime(format_string) return time_string def __str__(self): diff --git a/yt_dlp/extractor/motherless.py b/yt_dlp/extractor/motherless.py index 769b52ce6d..e359c44e93 100644 --- a/yt_dlp/extractor/motherless.py +++ b/yt_dlp/extractor/motherless.py @@ -151,7 +151,7 @@ def _real_extract(self, url): 'd': 'days', } kwargs = {_AGO_UNITS.get(uploaded_ago[-1]): delta} - upload_date = (datetime.datetime.utcnow() - datetime.timedelta(**kwargs)).strftime('%Y%m%d') + upload_date = (datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(**kwargs)).strftime('%Y%m%d') comment_count = len(re.findall(r'''class\s*=\s*['"]media-comment-contents\b''', webpage)) uploader_id = self._html_search_regex( diff --git a/yt_dlp/extractor/panopto.py b/yt_dlp/extractor/panopto.py index 6e3c9f442d..5ab2b2bcec 100644 --- a/yt_dlp/extractor/panopto.py +++ b/yt_dlp/extractor/panopto.py @@ -1,7 +1,7 @@ import calendar import json import functools -from datetime import datetime +from datetime import datetime, timezone from random import random from .common import InfoExtractor @@ -243,7 +243,7 @@ def _mark_watched(self, base_url, video_id, delivery_info): invocation_id = delivery_info.get('InvocationId') stream_id = traverse_obj(delivery_info, ('Delivery', 'Streams', ..., 'PublicID'), get_all=False, expected_type=str) if invocation_id and stream_id and duration: - timestamp_str = f'/Date({calendar.timegm(datetime.utcnow().timetuple())}000)/' + timestamp_str = f'/Date({calendar.timegm(datetime.now(timezone.utc).timetuple())}000)/' data = { 'streamRequests': [ { diff --git a/yt_dlp/networking/_urllib.py b/yt_dlp/networking/_urllib.py index b3e705b844..3c0647ecf9 100644 --- a/yt_dlp/networking/_urllib.py +++ b/yt_dlp/networking/_urllib.py @@ -429,7 +429,7 @@ def _send(self, request): except urllib.error.HTTPError as e: if isinstance(e.fp, (http.client.HTTPResponse, urllib.response.addinfourl)): # Prevent file object from being closed when urllib.error.HTTPError is destroyed. - e._closer.file = None + e._closer.close_called = True raise HTTPError(UrllibResponseAdapter(e.fp), redirect_loop='redirect error' in str(e)) from e raise # unexpected except urllib.error.URLError as e: diff --git a/yt_dlp/networking/exceptions.py b/yt_dlp/networking/exceptions.py index 10afc9ccbf..465b18ba94 100644 --- a/yt_dlp/networking/exceptions.py +++ b/yt_dlp/networking/exceptions.py @@ -115,7 +115,7 @@ def __init__(self, http_error: HTTPError): hdrs=http_error.response.headers, fp=http_error.response ) - self._closer.file = None # Disable auto close + self._closer.close_called = True # Disable auto close self._http_error = http_error HTTPError.__init__(self, http_error.response, redirect_loop=http_error.redirect_loop) diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py index 180bec245a..ef26de1160 100644 --- a/yt_dlp/utils/_utils.py +++ b/yt_dlp/utils/_utils.py @@ -669,6 +669,7 @@ def replace_insane(char): def sanitize_path(s, force=False): """Sanitizes and normalizes path on Windows""" + # XXX: this handles drive relative paths (c:sth) incorrectly if sys.platform == 'win32': force = False drive_or_unc, _ = os.path.splitdrive(s) @@ -687,7 +688,10 @@ def sanitize_path(s, force=False): sanitized_path.insert(0, drive_or_unc + os.path.sep) elif force and s and s[0] == os.path.sep: sanitized_path.insert(0, os.path.sep) - return os.path.join(*sanitized_path) + # TODO: Fix behavioral differences <3.12 + # The workaround using `normpath` only superficially passes tests + # Ref: https://github.com/python/cpython/pull/100351 + return os.path.normpath(os.path.join(*sanitized_path)) def sanitize_url(url, *, scheme='http'): @@ -1256,7 +1260,7 @@ def datetime_from_str(date_str, precision='auto', format='%Y%m%d'): if precision == 'auto': auto_precision = True precision = 'microsecond' - today = datetime_round(datetime.datetime.utcnow(), precision) + today = datetime_round(datetime.datetime.now(datetime.timezone.utc), precision) if date_str in ('now', 'today'): return today if date_str == 'yesterday': @@ -1319,8 +1323,8 @@ def datetime_round(dt, precision='day'): 'second': 1, } roundto = lambda x, n: ((x + n / 2) // n) * n - timestamp = calendar.timegm(dt.timetuple()) - return datetime.datetime.utcfromtimestamp(roundto(timestamp, unit_seconds[precision])) + timestamp = roundto(calendar.timegm(dt.timetuple()), unit_seconds[precision]) + return datetime.datetime.fromtimestamp(timestamp, datetime.timezone.utc) def hyphenate_date(date_str):