From 4c88ff87fc0e84659f7b6a7a88997eb6851125a0 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 24 Sep 2021 06:31:43 +0530 Subject: [PATCH] [build] Improve release process (#880) * Automate more of the release process by animelover1984, pukkandan - closes #823 * Fix sha256 by nihil-admirari - closes #385 * Bring back brew taps by nao20010128nao #865 * Provide `--onedir` zip for windows by pukkandan - Closes #1024, #661, #705 and #890 Authored by: pukkandan, animelover1984, nihil-admirari, nao20010128nao --- .github/workflows/build.yml | 186 +++++++++++++++++++++++++----------- Changelog.md | 7 +- pyinst.py | 7 +- yt_dlp/YoutubeDL.py | 10 +- yt_dlp/update.py | 34 ++++--- 5 files changed, 162 insertions(+), 82 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4c56a5180b..ccc2b2e471 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,11 +12,15 @@ jobs: outputs: ytdlp_version: ${{ steps.bump_version.outputs.ytdlp_version }} upload_url: ${{ steps.create_release.outputs.upload_url }} - sha256_unix: ${{ steps.sha256_file.outputs.sha256_unix }} - sha512_unix: ${{ steps.sha512_file.outputs.sha512_unix }} + sha256_bin: ${{ steps.sha256_bin.outputs.sha256_bin }} + sha512_bin: ${{ steps.sha512_bin.outputs.sha512_bin }} + sha256_tar: ${{ steps.sha256_tar.outputs.sha256_tar }} + sha512_tar: ${{ steps.sha512_tar.outputs.sha512_tar }} steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@v2 with: @@ -25,11 +29,76 @@ jobs: run: sudo apt-get -y install zip pandoc man - name: Bump version id: bump_version - run: python devscripts/update-version.py + run: | + python devscripts/update-version.py + make issuetemplates - name: Print version run: echo "${{ steps.bump_version.outputs.ytdlp_version }}" + - name: Update master + id: push_update + run: | + git config --global user.email "${{ github.event.pusher.email }}" + git config --global user.name "${{ github.event.pusher.name }}" + git add -u + git commit -m "[version] update" -m ":ci skip all" + git pull --rebase origin ${{ github.event.repository.master_branch }} + git push origin ${{ github.event.ref }}:${{ github.event.repository.master_branch }} + echo ::set-output name=head_sha::$(git rev-parse HEAD) + - name: Get Changelog + id: get_changelog + run: | + changelog=$(cat Changelog.md | grep -oPz '(?s)(?<=### ${{ steps.bump_version.outputs.ytdlp_version }}\n{2}).+?(?=\n{2,3}###)') + echo "changelog<> $GITHUB_ENV + echo "$changelog" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV - name: Run Make run: make all tar + - name: Get SHA2-256SUMS for yt-dlp + id: sha256_bin + run: echo "::set-output name=sha256_bin::$(sha256sum yt-dlp | awk '{print $1}')" + - name: Get SHA2-256SUMS for yt-dlp.tar.gz + id: sha256_tar + run: echo "::set-output name=sha256_tar::$(sha256sum yt-dlp.tar.gz | awk '{print $1}')" + - name: Get SHA2-512SUMS for yt-dlp + id: sha512_bin + run: echo "::set-output name=sha512_bin::$(sha512sum yt-dlp | awk '{print $1}')" + - name: Get SHA2-512SUMS for yt-dlp.tar.gz + id: sha512_tar + run: echo "::set-output name=sha512_tar::$(sha512sum yt-dlp.tar.gz | awk '{print $1}')" + - name: Install SSH private key + env: + BREW_TOKEN: ${{ secrets.BREW_TOKEN }} + if: "env.BREW_TOKEN != ''" + uses: webfactory/ssh-agent@v0.5.3 + with: + ssh-private-key: ${{ env.BREW_TOKEN }} + - name: Update Homebrew Formulae + env: + BREW_TOKEN: ${{ secrets.BREW_TOKEN }} + if: "env.BREW_TOKEN != ''" + run: | + git clone git@github.com:yt-dlp/homebrew-taps taps/ + python3 devscripts/update-formulae.py taps/Formula/yt-dlp.rb "${{ steps.bump_version.outputs.ytdlp_version }}" + git -C taps/ config user.name github-actions + git -C taps/ config user.email github-actions@example.com + git -C taps/ commit -am 'yt-dlp: ${{ steps.bump_version.outputs.ytdlp_version }}' + git -C taps/ push + - name: Install dependencies for pypi + env: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + if: "env.PYPI_TOKEN != ''" + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish on pypi + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + if: "env.TWINE_PASSWORD != ''" + run: | + rm -rf dist/* + python setup.py sdist bdist_wheel + twine upload dist/* - name: Create Release id: create_release uses: actions/create-release@v1 @@ -38,9 +107,10 @@ jobs: with: tag_name: ${{ steps.bump_version.outputs.ytdlp_version }} release_name: yt-dlp ${{ steps.bump_version.outputs.ytdlp_version }} + commitish: ${{ steps.push_update.outputs.head_sha }} body: | Changelog: - PLACEHOLDER + ${{ env.changelog }} draft: false prerelease: false - name: Upload yt-dlp Unix binary @@ -62,36 +132,16 @@ jobs: asset_path: ./yt-dlp.tar.gz asset_name: yt-dlp.tar.gz asset_content_type: application/gzip - - name: Get SHA2-256SUMS for yt-dlp - id: sha256_file - run: echo "::set-output name=sha256_unix::$(sha256sum yt-dlp | awk '{print $1}')" - - name: Get SHA2-512SUMS for yt-dlp - id: sha512_file - run: echo "::set-output name=sha512_unix::$(sha512sum yt-dlp | awk '{print $1}')" - - name: Install dependencies for pypi - env: - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - if: "env.PYPI_TOKEN != ''" - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish on pypi - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - if: "env.TWINE_PASSWORD != ''" - run: | - rm -rf dist/* - python setup.py sdist bdist_wheel - twine upload dist/* build_windows: runs-on: windows-latest needs: build_unix outputs: - sha256_windows: ${{ steps.sha256_file_win.outputs.sha256_windows }} - sha512_windows: ${{ steps.sha512_file_win.outputs.sha512_windows }} + sha256_win: ${{ steps.sha256_win.outputs.sha256_win }} + sha512_win: ${{ steps.sha512_win.outputs.sha512_win }} + sha256_win_zip: ${{ steps.sha256_win_zip.outputs.sha256_win_zip }} + sha512_win_zip: ${{ steps.sha512_win_zip.outputs.sha512_win_zip }} steps: - uses: actions/checkout@v2 @@ -104,7 +154,7 @@ jobs: run: python -m pip install --upgrade pip setuptools wheel - name: Install Requirements # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds - run: pip install "https://yt-dlp.github.io/pyinstaller-builds/x86_64/pyinstaller-4.5.1-py3-none-any.whl" mutagen pycryptodome websockets + run: pip install "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-4.5.1-py3-none-any.whl" mutagen pycryptodome websockets - name: Bump version id: bump_version run: python devscripts/update-version.py @@ -123,19 +173,41 @@ jobs: asset_name: yt-dlp.exe asset_content_type: application/vnd.microsoft.portable-executable - name: Get SHA2-256SUMS for yt-dlp.exe - id: sha256_file_win - run: echo "::set-output name=sha256_windows::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA256).Hash.ToLower())" + id: sha256_win + run: echo "::set-output name=sha256_win::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA256).Hash.ToLower())" - name: Get SHA2-512SUMS for yt-dlp.exe - id: sha512_file_win - run: echo "::set-output name=sha512_windows::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA512).Hash.ToLower())" + id: sha512_win + run: echo "::set-output name=sha512_win::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA512).Hash.ToLower())" + - name: Run PyInstaller Script with --onedir + run: python pyinst.py 64 --onedir + - uses: papeloto/action-zip@v1 + with: + files: ./dist/yt-dlp + dest: ./dist/yt-dlp.zip + - name: Upload yt-dlp.zip Windows onedir + id: upload-release-windows-zip + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.build_unix.outputs.upload_url }} + asset_path: ./dist/yt-dlp.zip + asset_name: yt-dlp.zip + asset_content_type: application/zip + - name: Get SHA2-256SUMS for yt-dlp.zip + id: sha256_win_zip + run: echo "::set-output name=sha256_win_zip::$((Get-FileHash dist\yt-dlp.zip -Algorithm SHA256).Hash.ToLower())" + - name: Get SHA2-512SUMS for yt-dlp.zip + id: sha512_win_zip + run: echo "::set-output name=sha512_win_zip::$((Get-FileHash dist\yt-dlp.zip -Algorithm SHA512).Hash.ToLower())" build_windows32: runs-on: windows-latest needs: [build_unix, build_windows] outputs: - sha256_windows32: ${{ steps.sha256_file_win32.outputs.sha256_windows32 }} - sha512_windows32: ${{ steps.sha512_file_win32.outputs.sha512_windows32 }} + sha256_win32: ${{ steps.sha256_win32.outputs.sha256_win32 }} + sha512_win32: ${{ steps.sha512_win32.outputs.sha512_win32 }} steps: - uses: actions/checkout@v2 @@ -148,7 +220,7 @@ jobs: - name: Upgrade pip and enable wheel support run: python -m pip install --upgrade pip setuptools wheel - name: Install Requirements - run: pip install "https://yt-dlp.github.io/pyinstaller-builds/i686/pyinstaller-4.5.1-py3-none-any.whl" mutagen pycryptodome websockets + run: pip install "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-4.5.1-py3-none-any.whl" mutagen pycryptodome websockets - name: Bump version id: bump_version run: python devscripts/update-version.py @@ -167,11 +239,11 @@ jobs: asset_name: yt-dlp_x86.exe asset_content_type: application/vnd.microsoft.portable-executable - name: Get SHA2-256SUMS for yt-dlp_x86.exe - id: sha256_file_win32 - run: echo "::set-output name=sha256_windows32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA256).Hash.ToLower())" + id: sha256_win32 + run: echo "::set-output name=sha256_win32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA256).Hash.ToLower())" - name: Get SHA2-512SUMS for yt-dlp_x86.exe - id: sha512_file_win32 - run: echo "::set-output name=sha512_windows32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA512).Hash.ToLower())" + id: sha512_win32 + run: echo "::set-output name=sha512_win32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA512).Hash.ToLower())" finish: runs-on: ubuntu-latest @@ -180,15 +252,17 @@ jobs: steps: - name: Make SHA2-256SUMS file env: - SHA256_WINDOWS: ${{ needs.build_windows.outputs.sha256_windows }} - SHA256_WINDOWS32: ${{ needs.build_windows32.outputs.sha256_windows32 }} - SHA256_UNIX: ${{ needs.build_unix.outputs.sha256_unix }} - YTDLP_VERSION: ${{ needs.build_unix.outputs.ytdlp_version }} + SHA256_WIN: ${{ needs.build_windows.outputs.sha256_win }} + SHA256_WIN_ZIP: ${{ needs.build_windows.outputs.sha256_win_zip }} + SHA256_WIN32: ${{ needs.build_windows32.outputs.sha256_win32 }} + SHA256_BIN: ${{ needs.build_unix.outputs.sha256_bin }} + SHA256_TAR: ${{ needs.build_unix.outputs.sha256_tar }} run: | - echo "version:${{ env.YTDLP_VERSION }}" >> SHA2-256SUMS - echo "yt-dlp.exe:${{ env.SHA256_WINDOWS }}" >> SHA2-256SUMS - echo "yt-dlp_x86.exe:${{ env.SHA256_WINDOWS32 }}" >> SHA2-256SUMS - echo "yt-dlp:${{ env.SHA256_UNIX }}" >> SHA2-256SUMS + echo "${{ env.SHA256_WIN }} yt-dlp.exe" >> SHA2-256SUMS + echo "${{ env.SHA256_WIN32 }} yt-dlp_x86.exe" >> SHA2-256SUMS + echo "${{ env.SHA256_BIN }} yt-dlp" >> SHA2-256SUMS + echo "${{ env.SHA256_TAR }} yt-dlp.tar.gz" >> SHA2-256SUMS + echo "${{ env.SHA256_WIN_ZIP }} yt-dlp.zip" >> SHA2-256SUMS - name: Upload 256SUMS file id: upload-sums uses: actions/upload-release-asset@v1 @@ -201,13 +275,17 @@ jobs: asset_content_type: text/plain - name: Make SHA2-512SUMS file env: - SHA512_WINDOWS: ${{ needs.build_windows.outputs.sha512_windows }} - SHA512_WINDOWS32: ${{ needs.build_windows32.outputs.sha512_windows32 }} - SHA512_UNIX: ${{ needs.build_unix.outputs.sha512_unix }} + SHA512_WIN: ${{ needs.build_windows.outputs.sha512_win }} + SHA512_WIN_ZIP: ${{ needs.build_windows.outputs.sha512_win_zip }} + SHA512_WIN32: ${{ needs.build_windows32.outputs.sha512_win32 }} + SHA512_BIN: ${{ needs.build_unix.outputs.sha512_bin }} + SHA512_TAR: ${{ needs.build_unix.outputs.sha512_tar }} run: | - echo "${{ env.SHA512_WINDOWS }} yt-dlp.exe" >> SHA2-512SUMS - echo "${{ env.SHA512_WINDOWS32 }} yt-dlp_x86.exe" >> SHA2-512SUMS - echo "${{ env.SHA512_UNIX }} yt-dlp" >> SHA2-512SUMS + echo "${{ env.SHA512_WIN }} yt-dlp.exe" >> SHA2-512SUMS + echo "${{ env.SHA512_WIN32 }} yt-dlp_x86.exe" >> SHA2-512SUMS + echo "${{ env.SHA512_BIN }} yt-dlp" >> SHA2-512SUMS + echo "${{ env.SHA512_TAR }} yt-dlp.tar.gz" >> SHA2-512SUMS + echo "${{ env.SHA512_WIN_ZIP }} yt-dlp.zip" >> SHA2-512SUMS - name: Upload 512SUMS file id: upload-512sums uses: actions/upload-release-asset@v1 diff --git a/Changelog.md b/Changelog.md index 6901e28f2f..b555c953f0 100644 --- a/Changelog.md +++ b/Changelog.md @@ -7,14 +7,9 @@ # Instuctions for creating release * Update Changelog.md and CONTRIBUTORS * Change "Merged with ytdl" version in Readme.md if needed * Add new/fixed extractors in "new features" section of Readme.md -* Commit to master as `Release ` +* Commit as `Release ` * Push to origin/release using `git push origin master:release` build task will now run -* Update version.py using `devscripts\update-version.py` -* Run `make issuetemplates` -* Commit to master as `[version] update :ci skip all` -* Push to origin/master -* Update changelog in /releases --> diff --git a/pyinst.py b/pyinst.py index fb8eca3e5c..d65243f880 100644 --- a/pyinst.py +++ b/pyinst.py @@ -15,9 +15,11 @@ arch = sys.argv[1] if len(sys.argv) > 1 else platform.architecture()[0][:2] assert arch in ('32', '64') -print('Building %sbit version' % arch) _x86 = '_x86' if arch == '32' else '' +opts = sys.argv[2:] or ['--onefile'] +print(f'Building {arch}bit version with options {opts}') + FILE_DESCRIPTION = 'yt-dlp%s' % (' (32 Bit)' if _x86 else '') # root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) @@ -72,11 +74,12 @@ PyInstaller.__main__.run([ '--name=yt-dlp%s' % _x86, - '--onefile', '--icon=devscripts/logo.ico', *[f'--exclude-module={module}' for module in excluded_modules], *[f'--hidden-import={module}' for module in dependancies], '--upx-exclude=vcruntime140.dll', + '--noconfirm', + *opts, 'yt_dlp/__main__.py', ]) SetVersion('dist/yt-dlp%s.exe' % _x86, VERSION_FILE) diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 8df8f16754..2258e22af4 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -27,7 +27,6 @@ import random from string import ascii_letters -from zipimport import zipimporter from .compat import ( compat_basestring, @@ -143,6 +142,7 @@ FFmpegPostProcessor, MoveFilesAfterDownloadPP, ) +from .update import detect_variant from .version import __version__ if compat_os_name == 'nt': @@ -3266,12 +3266,8 @@ def print_debug_header(self): self.get_encoding())) write_string(encoding_str, encoding=None) - source = ( - '(exe)' if hasattr(sys, 'frozen') - else '(zip)' if isinstance(globals().get('__loader__'), zipimporter) - else '(source)' if os.path.basename(sys.argv[0]) == '__main__.py' - else '') - self._write_string('[debug] yt-dlp version %s %s\n' % (__version__, source)) + source = detect_variant() + self._write_string('[debug] yt-dlp version %s%s\n' % (__version__, '' if source == 'unknown' else f' ({source})')) if _LAZY_LOADER: self._write_string('[debug] Lazy loading extractors enabled\n') if _PLUGIN_CLASSES: diff --git a/yt_dlp/update.py b/yt_dlp/update.py index d3681b8323..531eea7c91 100644 --- a/yt_dlp/update.py +++ b/yt_dlp/update.py @@ -31,6 +31,18 @@ def rsa_verify(message, signature, key): ''' +def detect_variant(): + if hasattr(sys, 'frozen') and getattr(sys, '_MEIPASS', None): + if sys._MEIPASS == os.path.dirname(sys.executable): + return 'dir' + return 'exe' + elif isinstance(globals().get('__loader__'), zipimporter): + return 'zip' + elif os.path.basename(sys.argv[0]) == '__main__.py': + return 'source' + return 'unknown' + + def update_self(to_screen, verbose, opener): ''' Exists for backward compatibility. Use run_update(ydl) instead ''' @@ -87,13 +99,14 @@ def calc_sha256sum(path): h.update(mv[:n]) return h.hexdigest() - err = None - if isinstance(globals().get('__loader__'), zipimporter): - pass - elif hasattr(sys, 'frozen'): - pass - else: - err = 'It looks like you installed yt-dlp with a package manager, pip, setup.py or a tarball. Please use that to update' + ERRORS = { + 'exe': None, + 'zip': None, + 'dir': 'Auto-update is not supported for unpackaged windows executable. Re-download the latest release', + 'source': 'You cannot update when running from source code', + 'unknown': 'It looks like you installed yt-dlp with a package manager, pip, setup.py or a tarball. Use that to update', + } + err = ERRORS.get(detect_variant(), ERRORS['unknown']) if err: return report_error(err, expected=True) @@ -138,12 +151,7 @@ def get_sha256sum(bin_or_exe, version): if not urlh: return None hash_data = ydl._opener.open(urlh).read().decode('utf-8') - if hash_data.startswith('version:'): - # Old colon-separated hash file - return dict(ln.split(':') for ln in hash_data.splitlines()).get(filename) - else: - # GNU-style hash file - return dict(ln.split()[::-1] for ln in hash_data.splitlines()).get(filename) + return dict(ln.split()[::-1] for ln in hash_data.splitlines()).get(filename) if not os.access(filename, os.W_OK): return report_error('no write permissions on %s' % filename, expected=True)