CI: Add Support for Valkey 6.2, 7.0 and 7.2

.github/json_matrices/engine-matrix.json

@@ -0,0 +1,14 @@
+[
+  {
+    "type": "valkey",
+    "version": "7.2.5"
+  },
+  {
+    "type": "valkey",
+    "version": "redis-7.0.15"
+  },
+  {
+    "type": "valkey",
+    "version": "redis-6.2.14"
+  }
+]

.github/workflows/build-node-wrapper/action.yml

@@ -29,6 +29,10 @@ inputs:
         description: "Specified target for rust toolchain, ex. x86_64-apple-darwin"
         type: string
         required: true
+    engine-version:
+        description: "Engine version to install"
+        required: true
+        type: string
     publish:
         description: "Enable building the wrapper in release mode"
         required: false
@@ -56,6 +60,7 @@ runs:
               os: ${{ inputs.os }}
               target: ${{ inputs.target }}
               github-token: ${{ inputs.github-token }}
+              engine-version: ${{ inputs.engine-version }}
 
         - name: Create package.json file
           uses: ./.github/workflows/node-create-package-file

.github/workflows/build-python-wrapper/action.yml

@@ -13,6 +13,10 @@ inputs:
         description: "Specified target for rust toolchain, ex. x86_64-apple-darwin"
         type: string
         required: true
+    engine-version:
+        description: "Engine version to install"
+        required: true
+        type: string
     publish:
         description: "Enable building the wrapper in release mode"
         required: false
@@ -34,6 +38,7 @@ runs:
               os: ${{ inputs.os }}
               target: ${{ inputs.target }}
               github-token: ${{ inputs.github-token }}
+              engine-version: ${{ inputs.engine-version }}
 
         - name: Install Python software dependencies
           shell: bash

.github/workflows/csharp.yml

@@ -9,19 +9,19 @@ on:
             - submodules/**
             - .github/workflows/csharp.yml
             - .github/workflows/install-shared-dependencies/action.yml
-            - .github/workflows/install-redis/action.yml
             - .github/workflows/test-benchmark/action.yml
             - .github/workflows/lint-rust/action.yml
+            - .github/workflows/install-valkey/action.yml
     pull_request:
         paths:
             - csharp/**
             - glide-core/src/**
             - submodules/**
             - .github/workflows/csharp.yml
             - .github/workflows/install-shared-dependencies/action.yml
-            - .github/workflows/install-redis/action.yml
             - .github/workflows/test-benchmark/action.yml
             - .github/workflows/lint-rust/action.yml
+            - .github/workflows/install-valkey/action.yml
 
 permissions:
     contents: read
@@ -31,14 +31,26 @@ concurrency:
     cancel-in-progress: true
 
 jobs:
+    load-engine-matrix:
+        runs-on: ubuntu-latest
+        outputs:
+          matrix: ${{ steps.load-engine-matrix.outputs.matrix }}
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+
+            - name: Load the engine matrix
+              id: load-engine-matrix
+              shell: bash
+              run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT
+
     run-tests:
+        needs: load-engine-matrix
         timeout-minutes: 25
         strategy:
             fail-fast: false
             matrix:
-                redis:
-                    - 6.2.14
-                    - 7.2.3
+                engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }}
                 dotnet:
                     - '6.0'
                     - '8.0'
@@ -61,13 +73,6 @@ jobs:
               with:
                   submodules: recursive
 
-            - name: Install redis
-              # TODO: make this step macos compatible: https://github.com/aws/glide-for-redis/issues/781
-              if: ${{ matrix.host.OS == 'ubuntu' }}
-              uses: ./.github/workflows/install-redis
-              with:
-                  redis-version: ${{ matrix.redis }}
-
             - name: Set up dotnet ${{ matrix.dotnet }}
               uses: actions/setup-dotnet@v4
               with:
@@ -79,6 +84,7 @@ jobs:
                   os: ${{ matrix.host.OS }}
                   target: ${{ matrix.host.TARGET }}
                   github-token: ${{ secrets.GITHUB_TOKEN }}
+                  engine-version: ${{ matrix.engine.version }}
 
             - name: Format
               working-directory: ./csharp

.github/workflows/go.yml

@@ -8,20 +8,41 @@ on:
             - submodules/**
             - go/**
             - .github/workflows/go.yml
+            - .github/workflows/install-shared-dependencies/action.yml
+            - .github/workflows/test-benchmark/action.yml
+            - .github/workflows/lint-rust/action.yml
+            - .github/workflows/install-valkey/action.yml
     pull_request:
         paths:
             - glide-core/src/**
             - submodules/**
             - go/**
             - .github/workflows/go.yml
-
+            - .github/workflows/install-shared-dependencies/action.yml
+            - .github/workflows/test-benchmark/action.yml
+            - .github/workflows/lint-rust/action.yml
+            - .github/workflows/install-valkey/action.yml
 concurrency:
     group: go-${{ github.head_ref || github.ref }}
     cancel-in-progress: true
 
 
 jobs:
+    load-engine-matrix:
+      runs-on: ubuntu-latest
+      outputs:
+        matrix: ${{ steps.load-engine-matrix.outputs.matrix }}
+      steps:
+          - name: Checkout
+            uses: actions/checkout@v4
+
+          - name: Load the engine matrix
+            id: load-engine-matrix
+            shell: bash
+            run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT
+
     build-and-test-go-client:
+        needs: load-engine-matrix
         timeout-minutes: 35
         strategy:
             # Run all jobs
@@ -30,9 +51,7 @@ jobs:
                 go:
                     - '1.18.10'
                     - '1.22.0'
-                redis:
-                    - 6.2.14
-                    - 7.2.3
+                engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }}
                 host:
                   - {
                     OS: ubuntu,
@@ -64,13 +83,7 @@ jobs:
                   os: ${{ matrix.host.OS }}
                   target: ${{ matrix.host.TARGET }}
                   github-token: ${{ secrets.GITHUB_TOKEN }}
-
-            - name: Install redis
-              # TODO: make this step macos compatible: https://github.com/aws/glide-for-redis/issues/781
-              if: ${{ matrix.os == 'ubuntu-latest' }}
-              uses: ./.github/workflows/install-redis
-              with:
-                  redis-version: ${{ matrix.redis }}
+                  engine-version: ${{ matrix.engine.version }}
 
             - name: Install tools for Go ${{ matrix.go }}
               working-directory: ./go
@@ -127,11 +140,7 @@ jobs:
                   os: "amazon-linux"
                   target: "x86_64-unknown-linux-gnu"
                   github-token: ${{ secrets.GITHUB_TOKEN }}
-
-            - name: Create a symbolic Link for redis6 binaries
-              run: |
-                  ln -s /usr/bin/redis6-server /usr/bin/redis-server
-                  ln -s /usr/bin/redis6-cli /usr/bin/redis-cli
+                  engine-version: "7.2.5"
 
             - name: Install Go
               run: |

.github/workflows/install-shared-dependencies/action.yml

@@ -21,6 +21,11 @@ inputs:
             - aarch64-apple-darwin
             - aarch64-unknown-linux-musl
             - x86_64-unknown-linux-musl
+    engine-version:
+      description: "Engine version to install"
+      required: true
+      type: string
+
     github-token:
         description: "GITHUB_TOKEN, GitHub App installation access token"
         required: true
@@ -36,7 +41,7 @@ runs:
           run: |
               brew update
               brew upgrade || true
-              brew install git gcc pkgconfig openssl redis coreutils
+              brew install git gcc pkgconfig openssl coreutils
 
         - name: Install software dependencies for Ubuntu GNU
           shell: bash
@@ -52,17 +57,23 @@ runs:
               apk update
               wget -O - https://sh.rustup.rs | sh -s -- -y
               source "$HOME/.cargo/env"
-              apk add protobuf-dev musl-dev make gcc redis envsubst
+              apk add protobuf-dev musl-dev make gcc envsubst openssl libressl-dev
 
         - name: Install software dependencies for Amazon-Linux
           shell: bash
           if: "${{ inputs.os == 'amazon-linux' }}"
           run: |
-              yum install -y gcc pkgconfig openssl openssl-devel which curl redis6 gettext --allowerasing
+              yum install -y gcc pkgconfig openssl openssl-devel which curl gettext --allowerasing
 
         - name: Install Rust toolchain and protoc
           if: "${{ !contains(inputs.target, 'musl') }}"
           uses: ./.github/workflows/install-rust-and-protoc
           with:
             target: ${{ inputs.target }}
             github-token: ${{ inputs.github-token }}
+
+        - name: Install Valkey
+          uses: ./.github/workflows/install-valkey
+          with:
+              engine-version: ${{ inputs.engine-version }}
+              target: ${{ inputs.target }}

.github/workflows/install-valkey/action.yml

@@ -0,0 +1,78 @@
+name: Install Valkey
+
+inputs:
+    engine-version:
+        description: "Engine version to install"
+        required: true
+        type: string
+    target:
+        description: "Specified target toolchain, ex. x86_64-unknown-linux-gnu"
+        type: string
+        required: true
+        options:
+            - x86_64-unknown-linux-gnu
+            - aarch64-unknown-linux-gnu
+            - x86_64-apple-darwin
+            - aarch64-apple-darwin
+            - aarch64-unknown-linux-musl
+            - x86_64-unknown-linux-musl
+
+env:
+    CARGO_TERM_COLOR: always
+    VALKEY_MIN_VERSION: "7.2.5"
+
+runs:
+    using: "composite"
+
+    steps:
+        - name: Cache Valkey
+           # TODO: remove the musl ARM64 limitation when https://github.com/actions/runner/issues/801 is resolved
+          if: ${{ inputs.target != 'aarch64-unknown-linux-musl' }}
+          uses: actions/cache@v4
+          id: cache-valkey
+          with:
+            path: |
+                ~/valkey
+            key: valkey-${{ inputs.engine-version }}-${{ inputs.target }}
+
+        - name: Build Valkey
+          if: ${{ steps.cache-valkey.outputs.cache-hit != 'true' }}
+          shell: bash
+          run: | 
+            echo "Building valkey ${{ inputs.engine-version }}"
+            cd ~
+            rm -rf valkey
+            git clone https://github.com/valkey-io/valkey.git
+            cd valkey
+            git checkout ${{ inputs.engine-version }}
+            make BUILD_TLS=yes
+
+        - name: Install Valkey
+          shell: bash
+          run: |
+            cd ~/valkey
+            if command -v sudo &> /dev/null
+            then
+                echo "sudo command exists"
+                sudo make install
+            else
+                echo "sudo command does not exist"
+                make install
+            fi
+            echo 'export PATH=/usr/local/bin:$PATH' >>~/.bash_profile
+
+        - name: Verify Valkey installation and symlinks
+          shell: bash
+          run: | 
+            # In Valkey releases, the engine is built with symlinks from valkey-server and valkey-cli
+            # to redis-server and redis-cli. This step ensures that the engine is properly installed
+            # with the expected version and that Valkey symlinks are correctly created.
+            EXPECTED_VERSION=`echo ${{ inputs.engine-version }} | sed -e "s/^redis-//"`
+            INSTALLED_VER=$(redis-server -v)
+            if [[ $INSTALLED_VER != *"${EXPECTED_VERSION}"* ]]; then
+                echo "Wrong version has been installed. Expected: $EXPECTED_VERSION, Installed: $INSTALLED_VER"
+                exit 1
+            else
+                echo "Successfully installed the server: $INSTALLED_VER"
+            fi
+