From 03d7db254d159832479d4f51a87ce6aa3f283c4e Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Mon, 25 Nov 2024 21:28:19 -0800 Subject: [PATCH] Node: Add server-modules to npm release (#2754) Signed-off-by: Andrew Carbonetto Signed-off-by: avifenesh --- .github/DEVELOPER.md | 2 +- .../supported-languages-versions.json | 4 +- .github/workflows/pypi-cd.yml | 96 ++++++++++++------- node/npm/glide/package.json | 4 +- python/Pipfile | 2 +- python/README.md | 2 +- python/pyproject.toml | 10 +- .../async_commands/standalone_commands.py | 8 +- python/python/tests/conftest.py | 2 +- python/requirements.txt | 3 +- 10 files changed, 77 insertions(+), 56 deletions(-) diff --git a/.github/DEVELOPER.md b/.github/DEVELOPER.md index 2acc4ccb68..9b45af17e0 100644 --- a/.github/DEVELOPER.md +++ b/.github/DEVELOPER.md @@ -133,7 +133,7 @@ We use dynamic matrices for our CI/CD workflows, which are created using the `cr 4. It generates three matrices: - Engine matrix: Defines the types and versions of the engine to test against, for example Valkey 7.2.5. - Host matrix: Defines the host platforms to run the tests on, for example Ubuntu on ARM64. - - Language-version matrix: Defines the supported versions of languages, for example python 3.8. + - Language-version matrix: Defines the supported versions of languages, for example python 3.9. #### Outputs diff --git a/.github/json_matrices/supported-languages-versions.json b/.github/json_matrices/supported-languages-versions.json index 3cefd8b7da..6eb3f41bff 100644 --- a/.github/json_matrices/supported-languages-versions.json +++ b/.github/json_matrices/supported-languages-versions.json @@ -6,8 +6,8 @@ }, { "language": "python", - "versions": ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"], - "always-run-versions": ["3.8", "3.13"] + "versions": ["3.9", "3.10", "3.11", "3.12", "3.13"], + "always-run-versions": ["3.9", "3.13"] }, { "language": "node", diff --git a/.github/workflows/pypi-cd.yml b/.github/workflows/pypi-cd.yml index b72da2a211..6126d32195 100644 --- a/.github/workflows/pypi-cd.yml +++ b/.github/workflows/pypi-cd.yml @@ -79,8 +79,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - with: - submodules: "true" - name: Set the release version shell: bash @@ -117,6 +115,13 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.12" + + - name: Setup Python for self-hosted Ubuntu runners + if: contains(matrix.build.RUNNER, 'self-hosted') + run: | + sudo apt update -y + sudo apt upgrade -y + sudo apt install python3 python3-venv python3-pip -y - name: Update package version in config.toml uses: ./.github/workflows/update-glide-version @@ -140,44 +145,59 @@ jobs: # Log the edited .gitignore file cat .gitignore - - name: Build Python wheels (linux) - if: startsWith(matrix.build.NAMED_OS, 'linux') + - name: Set python versions to build + run: | + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + echo "PYTHON_VERSIONS=python3.13" >> $GITHUB_ENV + else + echo "PYTHON_VERSIONS=python3.9 python3.10 python3.11 python3.12 python3.13" >> $GITHUB_ENV + fi + + - name: Build Python wheels (linux x86_64) + if: matrix.build.TARGET == 'x86_64-unknown-linux-gnu' uses: PyO3/maturin-action@v1 with: - working-directory: ./python - target: ${{ matrix.build.TARGET }} - args: --release --strip --out wheels -i ${{ github.event_name != 'pull_request' && 'python3.8 python3.9 python3.10 python3.11 python3.12 python3.13' || 'python3.12' }} - manylinux: auto - container: ${{ matrix.build.CONTAINER != '' && matrix.build.CONTAINER || '2014' }} - before-script-linux: | - # Install protobuf compiler - if [[ $(`which apt`) != '' ]] - then - echo "installing unzip and curl" - apt install unzip curl -y - fi - PB_REL="https://github.com/protocolbuffers/protobuf/releases" - ARCH=`uname -p` - if [[ $ARCH == 'x86_64' ]]; then - PROTOC_ARCH="x86_64" - elif [[ $ARCH == 'aarch64' ]]; then - PROTOC_ARCH="aarch_64" - else - echo "Running on unsupported architecture: $ARCH. Expected one of: ['x86_64', 'aarch64']" - exit 1 - fi - curl -LO $PB_REL/download/v3.20.3/protoc-3.20.3-linux-${PROTOC_ARCH}.zip - unzip protoc-3.20.3-linux-${PROTOC_ARCH}.zip -d $HOME/.local - export PATH="$PATH:$HOME/.local/bin" + working-directory: ./python + target: ${{ matrix.build.TARGET }} + args: --release --strip --out wheels -i ${PYTHON_VERSIONS} + manylinux: auto + container: ${{ matrix.build.CONTAINER }} + before-script-linux: | + # Install protobuf compiler + apt update && apt install unzip curl -y + PB_REL="https://github.com/protocolbuffers/protobuf/releases" + curl -LO $PB_REL/download/v3.20.3/protoc-3.20.3-linux-x86_64.zip + unzip protoc-3.20.3-linux-x86_64.zip -d $HOME/.local + export PATH="$PATH:$HOME/.local/bin" + + - name: Build Python wheels (linux aarch64) + if: matrix.build.TARGET == 'aarch64-unknown-linux-gnu' + uses: PyO3/maturin-action@v1 + with: + working-directory: ./python + target: ${{ matrix.build.TARGET }} + args: --release --strip --out wheels -i ${PYTHON_VERSIONS} + manylinux: "2014" + container: ${{ matrix.build.CONTAINER }} + before-script-linux: | + # Install protobuf compiler + apt update && apt install unzip curl -y + PB_REL="https://github.com/protocolbuffers/protobuf/releases" + curl -LO $PB_REL/download/v3.20.3/protoc-3.20.3-linux-aarch_64.zip + unzip protoc-3.20.3-linux-aarch_64.zip -d $HOME/.local + export PATH="$PATH:$HOME/.local/bin" + env: + CFLAGS: "-D__ARM_ARCH=8" + RUSTFLAGS: "-C target-feature=+neon" - name: Build Python wheels (macos) if: startsWith(matrix.build.NAMED_OS, 'darwin') uses: PyO3/maturin-action@v1 with: - maturin-version: latest + maturin-version: 0.14.14 working-directory: ./python target: ${{ matrix.build.TARGET }} - args: --release --strip --out wheels -i ${{ github.event_name != 'pull_request' && 'python3.8 python3.9 python3.10 python3.11 python3.12 python3.13' || 'python3.12' }} + args: --release --strip --out wheels -i ${PYTHON_VERSIONS} - name: Upload Python wheels if: github.event_name != 'pull_request' @@ -217,6 +237,10 @@ jobs: matrix: build: ${{ fromJson(needs.load-platform-matrix.outputs.PLATFORM_MATRIX) }} steps: + - name: Setup self-hosted runner access + if: ${{ matrix.build.TARGET == 'aarch64-unknown-linux-gnu' }} + run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/valkey-glide + - name: checkout uses: actions/checkout@v4 @@ -228,7 +252,8 @@ jobs: - name: Install engine uses: ./.github/workflows/install-engine with: - version: "8.0" + engine-version: "8.0" + target: ${{ matrix.build.TARGET }} - name: Check if RC and set a distribution tag for the package shell: bash @@ -236,12 +261,11 @@ jobs: if [[ "${GITHUB_REF:11}" == *"rc"* ]] then echo "This is a release candidate" - export pip_pre="--pre" + echo "DISTRIBUTION_TAG=--pre" >> $GITHUB_ENV else echo "This is a stable release" - export pip_pre="" + echo "DISTRIBUTION_TAG=" >> $GITHUB_ENV fi - echo "PIP_PRE=${pip_pre}" >> $GITHUB_ENV - name: Run the tests shell: bash @@ -250,7 +274,7 @@ jobs: python -m venv venv source venv/bin/activate pip install -U pip - pip install ${PIP_PRE} valkey-glide + pip install ${DISTRIBUTION_TAG} valkey-glide python rc_test.py # Reset the repository to make sure we get the clean checkout of the action later in other actions. diff --git a/node/npm/glide/package.json b/node/npm/glide/package.json index 9514160893..78ec8d0821 100644 --- a/node/npm/glide/package.json +++ b/node/npm/glide/package.json @@ -10,8 +10,8 @@ "lint": "eslint .", "lint:fix": "eslint . --fix", "clean": "rm -rf build-ts/", - "copy-declaration-files": "cp ../../build-ts/*.d.ts build-ts/ && cp ../../build-ts/src/*.d.ts build-ts/src/", - "build": "tsc && mkdir -p build-ts/src && npm run copy-declaration-files" + "copy-declaration-files": "cp ../../build-ts/*.d.ts build-ts/ && cp ../../build-ts/src/*.d.ts build-ts/src/ && cp ../../build-ts/src/server-modules/*.d.ts build-ts/src/server-modules/", + "build": "tsc && mkdir -p build-ts/src && mkdir -p build-ts/src/server-modules && npm run copy-declaration-files" }, "files": [ "/build-ts" diff --git a/python/Pipfile b/python/Pipfile index 5d44a4887c..d582066226 100644 --- a/python/Pipfile +++ b/python/Pipfile @@ -8,4 +8,4 @@ name = "pypi" [dev-packages] [requires] -python_version = "3.8" +python_version = "3.9" diff --git a/python/README.md b/python/README.md index 0bbd9d89f8..aa8ee70a63 100644 --- a/python/README.md +++ b/python/README.md @@ -25,11 +25,11 @@ macOS: | Python Version | |----------------| -| 3.8 | | 3.9 | | 3.10 | | 3.11 | | 3.12 | +| 3.13 | ## Installation and Setup diff --git a/python/pyproject.toml b/python/pyproject.toml index 4f4e79b91d..48ac6014d9 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,14 +1,14 @@ [build-system] -requires = ["maturin>=0.13,<0.14"] +requires = ["maturin>=1.0.0"] build-backend = "maturin" [project] name = "valkey-glide" -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = [ "async-timeout>=4.0.2; python_version < '3.11'", "typing-extensions>=4.8.0; python_version < '3.11'", - "protobuf>=3.20" + "protobuf>=3.20", ] classifiers = [ "Topic :: Database", @@ -30,7 +30,7 @@ max-line-length = 127 extend-ignore = ['E203'] [tool.black] -target-version = ['py38', 'py39', 'py310', 'py311', 'py312'] +target-version = ['py39', 'py310', 'py311', 'py312', 'py313'] [tool.mypy] -exclude = [ 'submodules', 'utils/release-candidate-testing' ] +exclude = ['submodules', 'utils/release-candidate-testing'] diff --git a/python/python/glide/async_commands/standalone_commands.py b/python/python/glide/async_commands/standalone_commands.py index 6bf5e81140..1cb2230a87 100644 --- a/python/python/glide/async_commands/standalone_commands.py +++ b/python/python/glide/async_commands/standalone_commands.py @@ -2,19 +2,17 @@ from __future__ import annotations -from typing import Any, Dict, List, Mapping, Optional, Set, Union, cast +from typing import Dict, List, Mapping, Optional, Union, cast -from glide.async_commands.command_args import Limit, ObjectType, OrderBy +from glide.async_commands.command_args import ObjectType from glide.async_commands.core import ( CoreCommands, FlushMode, FunctionRestorePolicy, InfoSection, - _build_sort_args, ) from glide.async_commands.transaction import Transaction from glide.constants import ( - OK, TOK, TEncodable, TFunctionListResponse, @@ -23,7 +21,7 @@ ) from glide.protobuf.command_request_pb2 import RequestType -from ..glide import ClusterScanCursor, Script +from ..glide import Script class StandaloneCommands(CoreCommands): diff --git a/python/python/tests/conftest.py b/python/python/tests/conftest.py index 15ff15cf4e..0937ca2067 100644 --- a/python/python/tests/conftest.py +++ b/python/python/tests/conftest.py @@ -12,7 +12,7 @@ ReadFrom, ServerCredentials, ) -from glide.exceptions import ClosingError, RequestError +from glide.exceptions import ClosingError from glide.glide_client import GlideClient, GlideClusterClient, TGlideClient from glide.logger import Level as logLevel from glide.logger import Logger diff --git a/python/requirements.txt b/python/requirements.txt index cffc0870cb..b39d1d96c8 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,8 +1,7 @@ async-timeout==4.0.2;python_version<"3.11" -maturin==0.13.0 +maturin==0.14.17 # higher version break the needs structure changes, the name of the project is not the same as the package name, and the naming both glide create a circular dependency - TODO: fix this protobuf==3.20.* pytest pytest-asyncio typing_extensions==4.8.0;python_version<"3.11" pytest-html -pyrsistent