From d84713d57a40c4fa3fc978d4b824e030eaf5700d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20=C4=8Ciha=C5=99?= Date: Tue, 21 May 2024 08:27:24 +0200 Subject: [PATCH] ci: use uv to install packages - use uv instead of pip for better peformance - drop cache as uv cache is way bigger than pip - install setuptools only when invoking setup.py manually --- .github/workflows/codeql-analysis.yml | 3 +-- .github/workflows/docs.yml | 15 +++++++------- .github/workflows/licenses-update.yml | 9 +++------ .github/workflows/migrations.yml | 3 +-- .github/workflows/mypy.yml | 1 - .github/workflows/pre-commit.yml | 14 ++++++-------- .github/workflows/pylint.yml | 6 ++---- .github/workflows/schema-update.yml | 9 ++++----- .github/workflows/setup.yml | 28 +++++++++++++-------------- .github/workflows/test.yml | 3 +-- ci/pip-install | 24 +++++++++++------------ ci/run-migrate | 8 ++++---- pyproject.toml | 7 ++----- 13 files changed, 56 insertions(+), 74 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 998bf6a50e2a..7a320622d76a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -35,10 +35,9 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.11' - cache: pip - name: Used versions run: ./ci/print-versions - - name: Install pip dependencies + - name: Install Python dependencies run: ./ci/pip-install - name: Initialize CodeQL diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e68cc55ad36f..06d3e6b32322 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -26,14 +26,14 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.11' - cache: pip - cache-dependency-path: docs/requirements.txt - name: Install apt dependencies run: | sudo apt update sudo apt install -y graphviz - - name: Install pip dependencies - run: pip install -r docs/requirements.txt + - name: Install Python dependencies + run: | + pip install uv + uv pip install --system -r docs/requirements.txt - name: Sphinx build run: | echo "::add-matcher::.github/matchers/sphinx.json" @@ -54,10 +54,11 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.11' - cache: pip cache-dependency-path: docs/requirements.txt - - name: Install pip dependencies - run: pip install -r docs/requirements.txt + - name: Install Python dependencies + run: | + pip install uv + uv pip install --system -r docs/requirements.txt - name: Sphinx linkcheck run: | ./ci/run-docs linkcheck diff --git a/.github/workflows/licenses-update.yml b/.github/workflows/licenses-update.yml index d0f5d2591078..c262d4add039 100644 --- a/.github/workflows/licenses-update.yml +++ b/.github/workflows/licenses-update.yml @@ -28,9 +28,7 @@ jobs: submodules: true - uses: actions/cache@v4 with: - path: | - ~/.cache/pip - ~/.cache/pre-commit + path: ~/.cache/pre-commit key: ${{ runner.os }}-pre-commit-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('.pre-commit-config.yaml') }} - name: Set up Python uses: actions/setup-python@v5 @@ -38,9 +36,8 @@ jobs: python-version: '3.11' - name: Install dependencies run: | - python -m pip install $(sed -n 's/.*"\(pip==\([^"]*\)\)".*/\1/p' pyproject.toml) - pip install $(sed -n 's/.*"\(wheel==\([^"]*\)\)".*/\1/p' pyproject.toml) - pip install $(sed -n 's/.*"\(pre-commit==\([^"]*\)\)".*/\1/p' pyproject.toml) + python -m pip install $(sed -n 's/.*"\(uv==\([^"]*\)\)".*/\1/p' pyproject.toml) + uv pip install --system $(sed -n 's/.*"\(pre-commit==\([^"]*\)\)".*/\1/p' pyproject.toml) - run: ./scripts/generate-license-data - name: Update renovate branch if: github.ref != 'refs/heads/main' diff --git a/.github/workflows/migrations.yml b/.github/workflows/migrations.yml index 8c8142a44c40..45ae705f02a1 100644 --- a/.github/workflows/migrations.yml +++ b/.github/workflows/migrations.yml @@ -44,8 +44,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: 3.11 - cache: pip - - name: Install pip dependencies + - name: Install Python dependencies run: ./ci/pip-install migrations - name: Check missing migrations run: ./ci/run-checkmigrate diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 7eaec86755af..7acc0777d415 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -28,7 +28,6 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.11' - cache: pip - name: Install dependencies run: ./ci/pip-install mypy diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 722e6d53572f..b3061192f2c2 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -24,22 +24,20 @@ jobs: - uses: actions/checkout@v4 - uses: actions/cache@v4 with: - path: | - ~/.cache/pip - ~/.cache/pre-commit - key: ${{ runner.os }}-pre-commit-${{ hashFiles('**/requirements*.txt') }}-${{ hashFiles('.pre-commit-config.yaml') }}-${{ hashFiles('pyproject.toml') }} + path: ~/.cache/pre-commit + key: ${{ runner.os }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} - name: Setup Python uses: actions/setup-python@v5 with: python-version: '3.11' - - name: Update pip - run: python -m pip install --upgrade pip wheel + - name: Install uv + run: curl -LsSf https://astral.sh/uv/install.sh | sh - name: Install dependencies run: | if [ -f requirements-lint.txt ] ; then - pip install -r requirements-lint.txt + uv pip install --system -r requirements-lint.txt else - pip install $(sed -n 's/.*"\(pre-commit==\([^"]*\)\)".*/\1/p' pyproject.toml) + uv pip install --system $(sed -n 's/.*"\(pre-commit==\([^"]*\)\)".*/\1/p' pyproject.toml) fi - name: pre-commit run: pre-commit run --all diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index a1250a00a37f..65f0ee674407 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -26,12 +26,10 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.11' - cache: pip - name: Install dependencies run: | - python -m pip install $(sed -n 's/.*"\(pip==\([^"]*\)\)".*/\1/p' pyproject.toml) - pip install $(sed -n 's/.*"\(wheel==\([^"]*\)\)".*/\1/p' pyproject.toml) - pip install $(sed -n 's/.*"\(pylint==\([^"]*\)\)".*/\1/p' pyproject.toml) + python -m pip install $(sed -n 's/.*"\(uv==\([^"]*\)\)".*/\1/p' pyproject.toml) + uv pip install --system $(sed -n 's/.*"\(pylint==\([^"]*\)\)".*/\1/p' pyproject.toml) - name: Add pylint annotator uses: pr-annotators/pylint-pr-annotator@v0.0.1 diff --git a/.github/workflows/schema-update.yml b/.github/workflows/schema-update.yml index db1ee6610b54..2bd5d23cdf07 100644 --- a/.github/workflows/schema-update.yml +++ b/.github/workflows/schema-update.yml @@ -28,12 +28,11 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.11' - - name: Install pip dependencies + - name: Install Python dependencies run: | - python -m pip install $(sed -n 's/.*"\(pip==\([^"]*\)\)".*/\1/p' pyproject.toml) - pip install $(sed -n 's/.*"\(wheel==\([^"]*\)\)".*/\1/p' pyproject.toml) - pip install $(sed -n 's/.*"\(weblate-schemas==\([^"]*\)\)".*/\1/p' pyproject.toml) - pip install $(sed -n 's/.*"\(pre-commit==\([^"]*\)\)".*/\1/p' pyproject.toml) + python -m pip install $(sed -n 's/.*"\(uv==\([^"]*\)\)".*/\1/p' pyproject.toml) + uv pip install --system $(sed -n 's/.*"\(weblate-schemas==\([^"]*\)\)".*/\1/p' pyproject.toml) + uv pip install --system $(sed -n 's/.*"\(pre-commit==\([^"]*\)\)".*/\1/p' pyproject.toml) - run: make -C docs update-schemas - run: pre-commit run --files $(git diff --name-only) continue-on-error: true diff --git a/.github/workflows/setup.yml b/.github/workflows/setup.yml index a5c857dad6b1..d0ddb9462f7c 100644 --- a/.github/workflows/setup.yml +++ b/.github/workflows/setup.yml @@ -33,11 +33,12 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.11' - cache: pip - name: Used versions run: ./ci/print-versions - - name: Install pip dependencies + - name: Install Python dependencies run: ./ci/pip-install + - name: Install setuptools + run: uv pip install --system $(sed -n 's/.*"\(setuptools[<>=]=\([^"]*\)\)".*/\1/p' pyproject.toml) - name: Install Weblate run: coverage run ./setup.py install - name: Check difference @@ -67,13 +68,12 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.11' - cache: pip - name: Used versions run: ./ci/print-versions - - name: Install pip dependencies + - name: Install Python dependencies run: ./ci/pip-install - name: Install Weblate - run: pip install '.[all]' + run: uv pip install --system '.[all]' - name: Check difference run: diff -ruNqp weblate $(python -c 'import weblate; import os; print(os.path.dirname(weblate.__file__))') @@ -92,13 +92,11 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.11' - cache: pip - - name: Install pip dependencies + - name: Install Python dependencies run: | - python -m pip install $(sed -n 's/.*"\(pip==\([^"]*\)\)".*/\1/p' pyproject.toml) - pip install $(sed -n 's/.*"\(wheel==\([^"]*\)\)".*/\1/p' pyproject.toml) - pip install $(sed -n 's/.*"\(twine==\([^"]*\)\)".*/\1/p' pyproject.toml) - pip install $(sed -n 's/.*"\(build==\([^"]*\)\)".*/\1/p' pyproject.toml) + python -m pip install $(sed -n 's/.*"\(uv==\([^"]*\)\)".*/\1/p' pyproject.toml) + uv pip install --system $(sed -n 's/.*"\(twine==\([^"]*\)\)".*/\1/p' pyproject.toml) + uv pip install --system $(sed -n 's/.*"\(build==\([^"]*\)\)".*/\1/p' pyproject.toml) - name: build run: python -m build - name: Twine check @@ -122,14 +120,14 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.11' - cache: pip - cache-dependency-path: docs/requirements.txt - name: Install apt dependencies run: | sudo apt update sudo apt install -y graphviz pandoc - - name: Install pip dependencies - run: pip install -r docs/requirements.txt + - name: Install Python dependencies + run: | + python -m pip install $(sed -n 's/.*"\(uv==\([^"]*\)\)".*/\1/p' pyproject.toml) + uv pip install --system -r docs/requirements.txt - name: Sphinx build run: ./ci/run-docs - name: Convert release notes diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5ecc05435a53..87fc4073cd68 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,10 +68,9 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - cache: pip - name: Used versions run: ./ci/print-versions - - name: Install pip dependencies + - name: Install Python dependencies run: ./ci/pip-install ${{ matrix.requirements }} - name: Prepare database run: ./ci/prepare-database diff --git a/ci/pip-install b/ci/pip-install index b089b7269a29..17e4760a4503 100755 --- a/ci/pip-install +++ b/ci/pip-install @@ -8,16 +8,14 @@ set -e -x # Install CI requirements # shellcheck disable=SC2046 -python -m pip install $(sed -n 's/.*"\(pip==\([^"]*\)\)".*/\1/p' pyproject.toml) -# shellcheck disable=SC2046 -pip install $(sed -n 's/.*"\(wheel==\([^"]*\)\)".*/\1/p' pyproject.toml) +python -m pip install $(sed -n 's/.*"\(uv==\([^"]*\)\)".*/\1/p' pyproject.toml) # Workaround for # SystemError: ffi_prep_closure(): bad user_data (it seems that the version of the libffi library seen at runtime is different from the 'ffi.h' file seen at compile-time) -pip install --no-binary :all: cffi +uv pip install --system --no-binary :all: cffi if [ "${1:-latest}" = migrations ] ; then - pip install -e ".[all,mysql,ci]" + uv pip install --system -e ".[all,mysql,ci]" else if [ "${1:-latest}" = minimal ] ; then # Adjust deps to force minimal version @@ -25,19 +23,19 @@ else fi if [ "${1:-latest}" = mypy ] ; then - pip install -e ".[all,mysql,ci,mypy,test]" + uv pip install --system -e ".[all,mysql,ci,mypy,test]" else - pip install -e ".[all,mysql,ci,test]" + uv pip install --system -e ".[all,mysql,ci,test]" fi if [ "${1:-latest}" = edge ] ; then - pip install --no-binary=lxml --upgrade-strategy eager -U -e ".[all,mysql,test,ci]" + uv pip install --system --no-binary=lxml --upgrade -e ".[all,mysql,test,ci]" # Install from git / pre-release - pip install --no-deps --upgrade --force-reinstall https://github.com/translate/translate/archive/master.zip - pip install --no-deps --upgrade --force-reinstall https://github.com/WeblateOrg/language-data/archive/main.zip - pip install --no-deps --upgrade --force-reinstall https://github.com/WeblateOrg/translation-finder/archive/main.zip - pip install --upgrade --pre Django + uv pip install --system --no-deps --upgrade --force-reinstall https://github.com/translate/translate/archive/master.zip + uv pip install --system --no-deps --upgrade --force-reinstall https://github.com/WeblateOrg/language-data/archive/main.zip + uv pip install --system --no-deps --upgrade --force-reinstall https://github.com/WeblateOrg/translation-finder/archive/main.zip + uv pip install --system --upgrade --pre Django fi fi # Verify that deps are consistent -pip check +uv pip check diff --git a/ci/run-migrate b/ci/run-migrate index cf4f4b9ba21a..93a2792e01b4 100755 --- a/ci/run-migrate +++ b/ci/run-migrate @@ -27,14 +27,14 @@ fi check git checkout "$TAG" check -# Use clean virtualenv for each version, this avoids pip problems when trying to downgrade from current versions -virtualenv --python python3.11 ".venv-$TAG" +# Use clean virtualenv for each version, this avoids problems when trying to downgrade from current versions +uv venv --python python3.11 ".venv-$TAG" check # shellcheck source=/dev/null . ".venv-$TAG/bin/activate" # Workaround for SystemError: ffi_prep_closure(): bad user_data -pip install --no-binary :all: cffi -pip install -e ".[all,mysql]" +uv pip install --no-binary :all: cffi +uv pip install -e ".[all,mysql]" check echo "DATABASES['default']['HOST'] = '$CI_DB_HOST'" >> weblate/settings_test.py check diff --git a/pyproject.toml b/pyproject.toml index 736c82dc1092..4e7782a30bc9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,11 +113,8 @@ antispam = [ ci = [ "build==1.2.1", "coverage==7.5.1", - "pip==24.0", - "setuptools==69.5.1", - "twine==5.1.0", - "virtualenv==20.26.2", - "wheel==0.43.0" + "uv==0.1.45", + "twine==5.1.0" ] dev = [ "Weblate[all,ci,test,lint,mypy]",