diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index ab5c15fa2c..312ff066d5 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -72,19 +72,17 @@ jobs: - name: test hybrid node modules - commonjs run: | - cd hybrid-node-tests - cd commonjs-test - npm install + npm install --package-lock-only + npm ci npm run build-and-test - working-directory: ./node + working-directory: ./node/hybrid-node-tests/commonjs-test - name: test hybrid node modules - ecma run: | - cd hybrid-node-tests - cd ecmascript-test - npm install + npm install --package-lock-only + npm ci npm run build-and-test - working-directory: ./node + working-directory: ./node/hybrid-node-tests/ecmascript-test - name: test redis modules run: npm run test-modules -- --load-module=$GITHUB_WORKSPACE/redisearch.so --load-module=$GITHUB_WORKSPACE/redisjson.so @@ -109,7 +107,7 @@ jobs: build-macos-latest: runs-on: macos-latest - timeout-minutes: 15 + timeout-minutes: 25 steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/pypi-cd.yml b/.github/workflows/pypi-cd.yml index 8e0b81c8be..0ea96db6bd 100644 --- a/.github/workflows/pypi-cd.yml +++ b/.github/workflows/pypi-cd.yml @@ -20,6 +20,7 @@ jobs: if: github.repository_owner == 'aws' name: Publish packages to PyPi runs-on: ${{ matrix.build.RUNNER }} + timeout-minutes: 25 strategy: fail-fast: false matrix: diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index f819208006..3f4f52a5eb 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -130,7 +130,7 @@ jobs: build-macos-latest: runs-on: macos-latest - timeout-minutes: 15 + timeout-minutes: 25 steps: - uses: actions/checkout@v4 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index 9101c9c73a..f9d6da1493 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,19 +13,23 @@ * Python, Node: Added ZPOPMAX command ([#996](https://github.com/aws/glide-for-redis/pull/996), [#1009](https://github.com/aws/glide-for-redis/pull/1009)) * Python: Added ZRANGE command ([#906](https://github.com/aws/glide-for-redis/pull/906)) * Python, Node: Added PTTL command ([#1036](https://github.com/aws/glide-for-redis/pull/1036), [#1082](https://github.com/aws/glide-for-redis/pull/1082)) -* Node: Added HVAL command ([#1022](https://github.com/aws/glide-for-redis/pull/1022)) -* Node: Added PERSIST command ([#1023](https://github.com/aws/glide-for-redis/pull/1023)) +* Python, Node: Added HVAL command ([#1130](https://github.com/aws/glide-for-redis/pull/1130)), ([#1022](https://github.com/aws/glide-for-redis/pull/1022)) +* Python, Node: Added PERSIST command ([#1129](https://github.com/aws/glide-for-redis/pull/1129)), ([#1023](https://github.com/aws/glide-for-redis/pull/1023)) * Node: Added ZREMRANGEBYSCORE command ([#926](https://github.com/aws/glide-for-redis/pull/926)) * Node: Added ZREMRANGEBYRANK command ([#924](https://github.com/aws/glide-for-redis/pull/924)) * Node: Added Xadd, Xtrim commands. ([#1057](https://github.com/aws/glide-for-redis/pull/1057)) * Python: Added json module and JSON.SET JSON.GET commands ([#1056](https://github.com/aws/glide-for-redis/pull/1056)) * Node: Added Time command. ([#1114](https://github.com/aws/glide-for-redis/pull/1114)) * Python, Node: Added LINDEX command ([#1058](https://github.com/aws/glide-for-redis/pull/1058), [#999](https://github.com/aws/glide-for-redis/pull/999)) +* Python: Added ZRANK command ([#1065](https://github.com/aws/glide-for-redis/pull/1065)) +* Core: Enabled Cluster Mode periodic checks by default ([#1089](https://github.com/aws/glide-for-redis/pull/1089)) #### Features * Python: Allow chaining function calls on transaction. ([#987](https://github.com/aws/glide-for-redis/pull/987)) * Node: Adding support for GLIDE's usage in projects based on either `CommonJS` or `ECMAScript` modules. ([#1132](https://github.com/aws/glide-for-redis/pull/1132)) +* Python: Added Cluster Mode configuration for periodic checks interval ([#1089](https://github.com/aws/glide-for-redis/pull/1089)) + ## 0.2.0 (2024-02-11) diff --git a/csharp/tests/Integration/GetAndSet.cs b/csharp/tests/Integration/GetAndSet.cs index 1375baa39b..69dd062857 100644 --- a/csharp/tests/Integration/GetAndSet.cs +++ b/csharp/tests/Integration/GetAndSet.cs @@ -3,6 +3,8 @@ */ +using System.Runtime.InteropServices; + using System.Runtime.InteropServices; using Glide; @@ -61,7 +63,10 @@ public async Task GetReturnsEmptyString() [Test] public async Task HandleVeryLargeInput() { + // TODO invesitage and fix if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + Assert.Ignore("Flaky on MacOS"); + using (var client = new AsyncClient("localhost", TestConfiguration.STANDALONE_PORTS[0], false)) { Assert.Ignore("Flaky on MacOS"); } diff --git a/glide-core/THIRD_PARTY_LICENSES_RUST b/glide-core/THIRD_PARTY_LICENSES_RUST index 26fe2c6a0b..366314b916 100644 --- a/glide-core/THIRD_PARTY_LICENSES_RUST +++ b/glide-core/THIRD_PARTY_LICENSES_RUST @@ -2764,7 +2764,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ---- -Package: async-trait:0.1.77 +Package: async-trait:0.1.78 The following copyrights and licenses were found in the source code of this package: @@ -27447,7 +27447,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ---- -Package: syn:2.0.52 +Package: syn:2.0.53 The following copyrights and licenses were found in the source code of this package: diff --git a/glide-core/src/client/mod.rs b/glide-core/src/client/mod.rs index 2ad44da68a..138e352045 100644 --- a/glide-core/src/client/mod.rs +++ b/glide-core/src/client/mod.rs @@ -2,7 +2,7 @@ * Copyright GLIDE-for-Redis Project Contributors - SPDX Identifier: Apache-2.0 */ use crate::connection_request::{ - ConnectionRequest, NodeAddress, ProtocolVersion, ReadFrom, TlsMode, + connection_request, ConnectionRequest, NodeAddress, ProtocolVersion, ReadFrom, TlsMode, }; use crate::scripts_container::get_script; use futures::FutureExt; @@ -26,6 +26,7 @@ pub const HEARTBEAT_SLEEP_DURATION: Duration = Duration::from_secs(1); pub const DEFAULT_RESPONSE_TIMEOUT: Duration = Duration::from_millis(250); pub const DEFAULT_CONNECTION_ATTEMPT_TIMEOUT: Duration = Duration::from_millis(250); +pub const DEFAULT_PERIODIC_CHECKS_INTERVAL: Duration = Duration::from_secs(60); pub const INTERNAL_CONNECTION_TIMEOUT: Duration = Duration::from_millis(250); pub(super) fn get_port(address: &NodeAddress) -> u16 { @@ -284,11 +285,23 @@ async fn create_cluster_client( .collect(); let read_from = request.read_from.enum_value().unwrap_or(ReadFrom::Primary); let read_from_replicas = !matches!(read_from, ReadFrom::Primary,); // TODO - implement different read from replica strategies. + let periodic_checks = match request.periodic_checks { + Some(periodic_checks) => match periodic_checks { + connection_request::Periodic_checks::PeriodicChecksManualInterval(interval) => { + Some(Duration::from_secs(interval.duration_in_sec.into())) + } + connection_request::Periodic_checks::PeriodicChecksDisabled(_) => None, + }, + None => Some(DEFAULT_PERIODIC_CHECKS_INTERVAL), + }; let mut builder = redis::cluster::ClusterClientBuilder::new(initial_nodes) .connection_timeout(INTERNAL_CONNECTION_TIMEOUT); if read_from_replicas { builder = builder.read_from_replicas(); } + if let Some(interval_duration) = periodic_checks { + builder = builder.periodic_topology_checks(interval_duration); + } builder = builder.use_protocol(convert_to_redis_protocol( request.protocol.enum_value_or_default(), )); @@ -388,7 +401,6 @@ fn sanitized_request_string(request: &ConnectionRequest) -> String { let connection_retry_strategy = request.connection_retry_strategy.0.as_ref().map(|strategy| format!("\nreconnect backoff strategy: number of increasing duration retries: {}, base: {}, factor: {}", strategy.number_of_retries, strategy.exponent_base, strategy.factor)).unwrap_or_default(); - let protocol = request .protocol .enum_value() @@ -397,9 +409,30 @@ fn sanitized_request_string(request: &ConnectionRequest) -> String { let client_name = chars_to_string_option(&request.client_name) .map(|client_name| format!("\nClient name: {client_name}")) .unwrap_or_default(); + let periodic_checks = if request.cluster_mode_enabled { + match request.periodic_checks { + Some(ref periodic_checks) => match periodic_checks { + connection_request::Periodic_checks::PeriodicChecksManualInterval(interval) => { + format!( + "\nPeriodic Checks: Enabled with manual interval of {:?}s", + interval.duration_in_sec + ) + } + connection_request::Periodic_checks::PeriodicChecksDisabled(_) => { + "\nPeriodic Checks: Disabled".to_string() + } + }, + None => format!( + "\nPeriodic Checks: Enabled with default interval of {:?}", + DEFAULT_PERIODIC_CHECKS_INTERVAL + ), + } + } else { + "".to_string() + }; format!( - "\nAddresses: {addresses}{tls_mode}{cluster_mode}{request_timeout}{rfr_strategy}{connection_retry_strategy}{database_id}{protocol}{client_name}", + "\nAddresses: {addresses}{tls_mode}{cluster_mode}{request_timeout}{rfr_strategy}{connection_retry_strategy}{database_id}{protocol}{client_name}{periodic_checks}", ) } diff --git a/glide-core/src/protobuf/connection_request.proto b/glide-core/src/protobuf/connection_request.proto index 4745365ef9..ecdeeae1b2 100644 --- a/glide-core/src/protobuf/connection_request.proto +++ b/glide-core/src/protobuf/connection_request.proto @@ -29,6 +29,13 @@ enum ProtocolVersion { RESP2 = 1; } +message PeriodicChecksManualInterval { + uint32 duration_in_sec = 1; +} + +message PeriodicChecksDisabled { +} + // IMPORTANT - if you add fields here, you probably need to add them also in client/mod.rs:`sanitized_request_string`. message ConnectionRequest { repeated NodeAddress addresses = 1; @@ -41,6 +48,10 @@ message ConnectionRequest { uint32 database_id = 8; ProtocolVersion protocol = 9; string client_name = 10; + oneof periodic_checks { + PeriodicChecksManualInterval periodic_checks_manual_interval = 11; + PeriodicChecksDisabled periodic_checks_disabled = 12; + } } message ConnectionRetryStrategy { diff --git a/glide-core/src/protobuf/redis_request.proto b/glide-core/src/protobuf/redis_request.proto index 652334fa7f..444c2bb991 100644 --- a/glide-core/src/protobuf/redis_request.proto +++ b/glide-core/src/protobuf/redis_request.proto @@ -133,6 +133,7 @@ enum RequestType { JsonGet = 89; ZRemRangeByScore = 90; Time = 91; + Zrank = 92; } message Command { diff --git a/glide-core/src/socket_listener.rs b/glide-core/src/socket_listener.rs index 5c5bd3be7b..153f5224ae 100644 --- a/glide-core/src/socket_listener.rs +++ b/glide-core/src/socket_listener.rs @@ -363,6 +363,7 @@ fn get_command(request: &Command) -> Option { RequestType::JsonGet => Some(cmd("JSON.GET")), RequestType::ZRemRangeByScore => Some(cmd("ZREMRANGEBYSCORE")), RequestType::Time => Some(cmd("TIME")), + RequestType::Zrank => Some(cmd("ZRANK")), } } diff --git a/node/THIRD_PARTY_LICENSES_NODE b/node/THIRD_PARTY_LICENSES_NODE index 7b83783e44..f5bbfa2014 100644 --- a/node/THIRD_PARTY_LICENSES_NODE +++ b/node/THIRD_PARTY_LICENSES_NODE @@ -2816,7 +2816,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ---- -Package: async-trait:0.1.77 +Package: async-trait:0.1.78 The following copyrights and licenses were found in the source code of this package: @@ -29072,7 +29072,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ---- -Package: syn:2.0.52 +Package: syn:2.0.53 The following copyrights and licenses were found in the source code of this package: @@ -40953,6 +40953,67 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ---- +Package: child_process:1.0.2 + +The following copyrights and licenses were found in the source code of this package: + +Permission to use, copy, modify, and/or distribute this software for any purpose +with or without fee is hereby granted, provided that the above copyright notice +and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF +THIS SOFTWARE. + +---- + +Package: find-free-port:2.0.0 + +The following copyrights and licenses were found in the source code of this package: + +Permission to use, copy, modify, and/or distribute this software for any purpose +with or without fee is hereby granted, provided that the above copyright notice +and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF +THIS SOFTWARE. + +---- + +Package: find-free-ports:3.1.1 + +The following copyrights and licenses were found in the source code of this package: + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +---- + Package: long:5.2.3 The following copyrights and licenses were found in the source code of this package: @@ -41602,7 +41663,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---- -Package: @types:node:20.11.26 +Package: @types:node:20.11.29 The following copyrights and licenses were found in the source code of this package: diff --git a/python/THIRD_PARTY_LICENSES_PYTHON b/python/THIRD_PARTY_LICENSES_PYTHON index 36d4be2d27..faa28a4e8f 100644 --- a/python/THIRD_PARTY_LICENSES_PYTHON +++ b/python/THIRD_PARTY_LICENSES_PYTHON @@ -2764,7 +2764,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ---- -Package: async-trait:0.1.77 +Package: async-trait:0.1.78 The following copyrights and licenses were found in the source code of this package: @@ -29512,7 +29512,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ---- -Package: syn:2.0.52 +Package: syn:2.0.53 The following copyrights and licenses were found in the source code of this package: @@ -44430,7 +44430,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---- -Package: protobuf:4.25.3 +Package: protobuf:5.26.0 The following copyrights and licenses were found in the source code of this package: diff --git a/python/python/glide/async_commands/core.py b/python/python/glide/async_commands/core.py index 97fc71a836..bd1cf208e9 100644 --- a/python/python/glide/async_commands/core.py +++ b/python/python/glide/async_commands/core.py @@ -634,6 +634,24 @@ async def hlen(self, key: str) -> int: """ return cast(int, await self._execute_command(RequestType.HLen, [key])) + async def hvals(self, key: str) -> List[str]: + """ + Returns all values in the hash stored at `key`. + + See https://redis.io/commands/hvals/ for more details. + + Args: + key (str): The key of the hash. + + Returns: + List[str]: A list of values in the hash, or an empty list when the key does not exist. + + Examples: + >>> await client.hvals("my_hash") + ["value1", "value2", "value3"] # Returns all the values stored in the hash "my_hash". + """ + return cast(List[str], await self._execute_command(RequestType.Hvals, [key])) + async def lpush(self, key: str, elements: List[str]) -> int: """ Insert all the specified values at the head of the list stored at `key`. @@ -1228,6 +1246,31 @@ async def pttl( await self._execute_command(RequestType.PTTL, [key]), ) + async def persist( + self, + key: str, + ) -> bool: + """ + Remove the existing timeout on `key`, turning the key from volatile (a key with an expire set) to + persistent (a key that will never expire as no timeout is associated). + + See https://redis.io/commands/persist/ for more details. + + Args: + key (str): TThe key to remove the existing timeout on. + + Returns: + bool: False if `key` does not exist or does not have an associated timeout, True if the timeout has been removed. + + Examples: + >>> await client.persist("my_key") + True # Indicates that the timeout associated with the key "my_key" was successfully removed. + """ + return cast( + bool, + await self._execute_command(RequestType.Persist, [key]), + ) + async def echo(self, message: str) -> str: """ Echoes the provided `message` back. @@ -1591,6 +1634,65 @@ async def zrange_withscores( Mapping[str, float], await self._execute_command(RequestType.Zrange, args) ) + async def zrank( + self, + key: str, + member: str, + ) -> Optional[int]: + """ + Returns the rank of `member` in the sorted set stored at `key`, with scores ordered from low to high. + + See https://redis.io/commands/zrank for more details. + + To get the rank of `member` with it's score, see `zrank_withscore`. + + Args: + key (str): The key of the sorted set. + member (str): The member whose rank is to be retrieved. + + Returns: + Optional[int]: The rank of `member` in the sorted set. + If `key` doesn't exist, or if `member` is not present in the set, None will be returned. + + Examples: + >>> await client.zrank("my_sorted_set", "member2") + 1 # Indicates that "member2" has the second-lowest score in the sorted set "my_sorted_set". + >>> await client.zrank("my_sorted_set", "non_existing_member") + None # Indicates that "non_existing_member" is not present in the sorted set "my_sorted_set". + """ + return cast( + Optional[int], await self._execute_command(RequestType.Zrank, [key, member]) + ) + + async def zrank_withscore( + self, + key: str, + member: str, + ) -> Optional[List[Union[int, float]]]: + """ + Returns the rank of `member` in the sorted set stored at `key` with it's score, where scores are ordered from the lowest to highest. + + See https://redis.io/commands/zrank for more details. + + Args: + key (str): The key of the sorted set. + member (str): The member whose rank is to be retrieved. + + Returns: + Optional[List[Union[int, float]]]: A list containing the rank and score of `member` in the sorted set. + If `key` doesn't exist, or if `member` is not present in the set, None will be returned. + + Examples: + >>> await client.zrank_withscore("my_sorted_set", "member2") + [1 , 6.0] # Indicates that "member2" with score 6.0 has the second-lowest score in the sorted set "my_sorted_set". + >>> await client.zrank_withscore("my_sorted_set", "non_existing_member") + None # Indicates that "non_existing_member" is not present in the sorted set "my_sorted_set". + """ + return cast( + Optional[List[Union[int, float]]], + await self._execute_command(RequestType.Zrank, [key, member, "WITHSCORE"]), + ) + async def zrem( self, key: str, diff --git a/python/python/glide/async_commands/transaction.py b/python/python/glide/async_commands/transaction.py index a8d141a79f..036af4b951 100644 --- a/python/python/glide/async_commands/transaction.py +++ b/python/python/glide/async_commands/transaction.py @@ -529,6 +529,20 @@ def hdel(self: TTransaction, key: str, fields: List[str]) -> TTransaction: """ return self.append_command(RequestType.HashDel, [key] + fields) + def hvals(self: TTransaction, key: str) -> TTransaction: + """ + Returns all values in the hash stored at `key`. + + See https://redis.io/commands/hvals/ for more details. + + Args: + key (str): The key of the hash. + + Command response: + List[str]: A list of values in the hash, or an empty list when the key does not exist. + """ + return self.append_command(RequestType.Hvals, [key]) + def lpush(self: TTransaction, key: str, elements: List[str]) -> TTransaction: """ Insert all the specified values at the head of the list stored at `key`. @@ -979,6 +993,24 @@ def pttl( """ return self.append_command(RequestType.PTTL, [key]) + def persist( + self: TTransaction, + key: str, + ) -> TTransaction: + """ + Remove the existing timeout on `key`, turning the key from volatile (a key with an expire set) to + persistent (a key that will never expire as no timeout is associated). + + See https://redis.io/commands/persist/ for more details. + + Args: + key (str): TThe key to remove the existing timeout on. + + Commands response: + bool: False if `key` does not exist or does not have an associated timeout, True if the timeout has been removed. + """ + return self.append_command(RequestType.Persist, [key]) + def echo(self: TTransaction, message: str) -> TTransaction: """ Echoes the provided `message` back. @@ -1264,6 +1296,48 @@ def zrange_withscores( return self.append_command(RequestType.Zrange, args) + def zrank( + self: TTransaction, + key: str, + member: str, + ) -> TTransaction: + """ + Returns the rank of `member` in the sorted set stored at `key`, with scores ordered from low to high. + + See https://redis.io/commands/zrank for more details. + + To get the rank of `member` with it's score, see `zrank_withscore`. + + Args: + key (str): The key of the sorted set. + member (str): The member whose rank is to be retrieved. + + Commands response: + Optional[int]: The rank of `member` in the sorted set. + If `key` doesn't exist, or if `member` is not present in the set, None will be returned. + """ + return self.append_command(RequestType.Zrank, [key, member]) + + def zrank_withscore( + self: TTransaction, + key: str, + member: str, + ) -> TTransaction: + """ + Returns the rank of `member` in the sorted set stored at `key` with it's score, where scores are ordered from the lowest to highest. + + See https://redis.io/commands/zrank for more details. + + Args: + key (str): The key of the sorted set. + member (str): The member whose rank is to be retrieved. + + Commands response: + Optional[List[Union[int, float]]]: A list containing the rank and score of `member` in the sorted set. + If `key` doesn't exist, or if `member` is not present in the set, None will be returned. + """ + return self.append_command(RequestType.Zrank, [key, member, "WITHSCORE"]) + def zrem( self: TTransaction, key: str, diff --git a/python/python/glide/config.py b/python/python/glide/config.py index 581d686608..5c6ba07969 100644 --- a/python/python/glide/config.py +++ b/python/python/glide/config.py @@ -1,7 +1,7 @@ # Copyright GLIDE-for-Redis Project Contributors - SPDX Identifier: Apache-2.0 from enum import Enum -from typing import List, Optional +from typing import List, Optional, Union from glide.protobuf.connection_request_pb2 import ConnectionRequest from glide.protobuf.connection_request_pb2 import ProtocolVersion as SentProtocolVersion @@ -92,6 +92,33 @@ def __init__( self.username = username +class PeriodicChecksManualInterval: + def __init__(self, duration_in_sec: int) -> None: + """ + Represents a manually configured interval for periodic checks. + + Args: + duration_in_sec (int): The duration in seconds for the interval between periodic checks. + """ + self.duration_in_sec = duration_in_sec + + +class PeriodicChecksStatus(Enum): + """ + Represents the cluster's periodic checks status. + To configure specific interval, see PeriodicChecksManualInterval. + """ + + ENABLED_DEFAULT_CONFIGS = 0 + """ + Enables the periodic checks with the default configurations. + """ + DISABLED = 1 + """ + Disables the periodic checks. + """ + + class BaseClientConfiguration: def __init__( self, @@ -191,7 +218,7 @@ class RedisClientConfiguration(BaseClientConfiguration): reconnect_strategy (Optional[BackoffStrategy]): Strategy used to determine how and when to reconnect, in case of connection failures. If not set, a default backoff strategy will be used. - database_id (Optional[Int]): index of the logical database to connect to. + database_id (Optional[int]): index of the logical database to connect to. client_name (Optional[str]): Client name to be used for the client. Will be used with CLIENT SETNAME command during connection establishment. protocol (ProtocolVersion): The version of the Redis RESP protocol to communicate with the server. """ @@ -224,7 +251,7 @@ def _create_a_protobuf_conn_request( self, cluster_mode: bool = False ) -> ConnectionRequest: assert cluster_mode is False - request = super()._create_a_protobuf_conn_request(False) + request = super()._create_a_protobuf_conn_request(cluster_mode) if self.reconnect_strategy: request.connection_retry_strategy.number_of_retries = ( self.reconnect_strategy.num_of_retries @@ -259,6 +286,10 @@ class ClusterClientConfiguration(BaseClientConfiguration): If the specified timeout is exceeded for a pending request, it will result in a timeout error. If not set, a default value will be used. client_name (Optional[str]): Client name to be used for the client. Will be used with CLIENT SETNAME command during connection establishment. protocol (ProtocolVersion): The version of the Redis RESP protocol to communicate with the server. + periodic_checks (Union[PeriodicChecksStatus, PeriodicChecksManualInterval]): Configure the periodic topology checks. + These checks evaluate changes in the cluster's topology, triggering a slot refresh when detected. + Periodic checks ensure a quick and efficient process by querying a limited number of nodes. + Defaults to PeriodicChecksStatus.ENABLED_DEFAULT_CONFIGS. Notes: Currently, the reconnection strategy in cluster mode is not configurable, and exponential backoff @@ -274,6 +305,9 @@ def __init__( request_timeout: Optional[int] = None, client_name: Optional[str] = None, protocol: ProtocolVersion = ProtocolVersion.RESP3, + periodic_checks: Union[ + PeriodicChecksStatus, PeriodicChecksManualInterval + ] = PeriodicChecksStatus.ENABLED_DEFAULT_CONFIGS, ): super().__init__( addresses=addresses, @@ -284,3 +318,18 @@ def __init__( client_name=client_name, protocol=protocol, ) + self.periodic_checks = periodic_checks + + def _create_a_protobuf_conn_request( + self, cluster_mode: bool = False + ) -> ConnectionRequest: + assert cluster_mode is True + request = super()._create_a_protobuf_conn_request(cluster_mode) + if type(self.periodic_checks) is PeriodicChecksManualInterval: + request.periodic_checks_manual_interval.duration_in_sec = ( + self.periodic_checks.duration_in_sec + ) + elif self.periodic_checks == PeriodicChecksStatus.DISABLED: + request.periodic_checks_disabled.SetInParent() + + return request diff --git a/python/python/glide/constants.py b/python/python/glide/constants.py index 1526ee1167..a1c7c58445 100644 --- a/python/python/glide/constants.py +++ b/python/python/glide/constants.py @@ -14,13 +14,13 @@ TResult = Union[ TOK, str, - List[str], List[List[str]], int, None, Dict[str, T], float, Set[T], + List[T], ] TRequest = Union[RedisRequest, ConnectionRequest] # When routing to a single node, response will be T diff --git a/python/python/tests/test_async_client.py b/python/python/tests/test_async_client.py index 4256cdaa3a..a45c369fab 100644 --- a/python/python/tests/test_async_client.py +++ b/python/python/tests/test_async_client.py @@ -757,6 +757,25 @@ async def test_hlen(self, redis_client: TRedisClient): with pytest.raises(RequestError): await redis_client.hlen(key2) + @pytest.mark.parametrize("cluster_mode", [True, False]) + @pytest.mark.parametrize("protocol", [ProtocolVersion.RESP2, ProtocolVersion.RESP3]) + async def test_hvals(self, redis_client: TRedisClient): + key = get_random_string(10) + key2 = get_random_string(5) + field = get_random_string(5) + field2 = get_random_string(5) + field_value_map = {field: "value", field2: "value2"} + + assert await redis_client.hset(key, field_value_map) == 2 + assert await redis_client.hvals(key) == ["value", "value2"] + assert await redis_client.hdel(key, [field]) == 1 + assert await redis_client.hvals(key) == ["value2"] + assert await redis_client.hvals("non_existing_key") == [] + + assert await redis_client.set(key2, "value") == OK + with pytest.raises(RequestError): + await redis_client.hvals(key2) + @pytest.mark.parametrize("cluster_mode", [True, False]) @pytest.mark.parametrize("protocol", [ProtocolVersion.RESP2, ProtocolVersion.RESP3]) async def test_lpush_lpop_lrange(self, redis_client: TRedisClient): @@ -1077,6 +1096,16 @@ async def test_pttl(self, redis_client: TRedisClient): assert await redis_client.pexpireat(key, current_time * 1000 + 30000) assert 0 < await redis_client.pttl(key) <= 30000 + @pytest.mark.parametrize("cluster_mode", [True, False]) + @pytest.mark.parametrize("protocol", [ProtocolVersion.RESP2, ProtocolVersion.RESP3]) + async def test_persist(self, redis_client: TRedisClient): + key = get_random_string(10) + assert await redis_client.set(key, "value") == OK + assert not await redis_client.persist(key) + + assert await redis_client.expire(key, 10) + assert await redis_client.persist(key) + @pytest.mark.parametrize("cluster_mode", [True, False]) @pytest.mark.parametrize("protocol", [ProtocolVersion.RESP2, ProtocolVersion.RESP3]) async def test_zadd_zaddincr(self, redis_client: TRedisClient): @@ -1473,6 +1502,27 @@ async def test_zrange_different_types_of_keys(self, redis_client: TRedisClient): with pytest.raises(RequestError): await redis_client.zrange_withscores(key, RangeByIndex(start=0, stop=1)) + @pytest.mark.parametrize("cluster_mode", [True, False]) + @pytest.mark.parametrize("protocol", [ProtocolVersion.RESP2, ProtocolVersion.RESP3]) + async def test_zrank(self, redis_client: TRedisClient): + key = get_random_string(10) + members_scores = {"one": 1.5, "two": 2, "three": 3} + assert await redis_client.zadd(key, members_scores) == 3 + assert await redis_client.zrank(key, "one") == 0 + if not await check_if_server_version_lt(redis_client, "7.2.0"): + assert await redis_client.zrank_withscore(key, "one") == [0, 1.5] + assert await redis_client.zrank_withscore(key, "non_existing_field") is None + assert ( + await redis_client.zrank_withscore("non_existing_key", "field") is None + ) + + assert await redis_client.zrank(key, "non_existing_field") is None + assert await redis_client.zrank("non_existing_key", "field") is None + + assert await redis_client.set(key, "value") == OK + with pytest.raises(RequestError): + await redis_client.zrank(key, "one") + @pytest.mark.parametrize("cluster_mode", [True, False]) @pytest.mark.parametrize("protocol", [ProtocolVersion.RESP2, ProtocolVersion.RESP3]) async def test_type(self, redis_client: TRedisClient): diff --git a/python/python/tests/test_config.py b/python/python/tests/test_config.py index 5bcada20c2..ccd4d82a77 100644 --- a/python/python/tests/test_config.py +++ b/python/python/tests/test_config.py @@ -1,6 +1,13 @@ # Copyright GLIDE-for-Redis Project Contributors - SPDX Identifier: Apache-2.0 -from glide.config import BaseClientConfiguration, NodeAddress, ReadFrom +from glide.config import ( + BaseClientConfiguration, + ClusterClientConfiguration, + NodeAddress, + PeriodicChecksManualInterval, + PeriodicChecksStatus, + ReadFrom, +) from glide.protobuf.connection_request_pb2 import ConnectionRequest from glide.protobuf.connection_request_pb2 import ReadFrom as ProtobufReadFrom from glide.protobuf.connection_request_pb2 import TlsMode @@ -28,3 +35,20 @@ def test_convert_to_protobuf(): assert request.tls_mode is TlsMode.SecureTls assert request.read_from == ProtobufReadFrom.PreferReplica assert request.client_name == "TEST_CLIENT_NAME" + + +def test_periodic_checks_interval_to_protobuf(): + config = ClusterClientConfiguration( + [NodeAddress("127.0.0.1")], + ) + request = config._create_a_protobuf_conn_request(cluster_mode=True) + assert not request.HasField("periodic_checks_disabled") + assert not request.HasField("periodic_checks_manual_interval") + + config.periodic_checks = PeriodicChecksStatus.DISABLED + request = config._create_a_protobuf_conn_request(cluster_mode=True) + assert request.HasField("periodic_checks_disabled") + + config.periodic_checks = PeriodicChecksManualInterval(30) + request = config._create_a_protobuf_conn_request(cluster_mode=True) + assert request.periodic_checks_manual_interval.duration_in_sec == 30 diff --git a/python/python/tests/test_transaction.py b/python/python/tests/test_transaction.py index ebd1e44f35..5c07b71a09 100644 --- a/python/python/tests/test_transaction.py +++ b/python/python/tests/test_transaction.py @@ -15,11 +15,13 @@ from glide.constants import OK, TResult from glide.redis_client import RedisClient, RedisClusterClient, TRedisClient from tests.conftest import create_client -from tests.test_async_client import get_random_string +from tests.test_async_client import check_if_server_version_lt, get_random_string -def transaction_test( - transaction: Union[Transaction, ClusterTransaction], keyslot: str +async def transaction_test( + transaction: Union[Transaction, ClusterTransaction], + keyslot: str, + redis_client: TRedisClient, ) -> List[TResult]: key = "{{{}}}:{}".format(keyslot, get_random_string(3)) # to get the same slot key2 = "{{{}}}:{}".format(keyslot, get_random_string(3)) # to get the same slot @@ -43,6 +45,9 @@ def transaction_test( transaction.echo(value) args.append(value) + transaction.persist(key) + args.append(False) + transaction.exists([key]) args.append(1) @@ -86,6 +91,8 @@ def transaction_test( args.append(value2) transaction.hlen(key4) args.append(2) + transaction.hvals(key4) + args.append([value, value2]) transaction.hsetnx(key4, key, value) args.append(False) transaction.hincrby(key4, key3, 5) @@ -142,6 +149,11 @@ def transaction_test( transaction.zadd(key8, {"one": 1, "two": 2, "three": 3}) args.append(3) + transaction.zrank(key8, "one") + args.append(0) + if not await check_if_server_version_lt(redis_client, "7.2.0"): + transaction.zrank_withscore(key8, "one") + args.append([0, 1]) transaction.zadd_incr(key8, "one", 3) args.append(4) transaction.zrem(key8, ["one"]) @@ -247,7 +259,7 @@ async def test_cluster_transaction(self, redis_client: RedisClusterClient): keyslot = get_random_string(3) transaction = ClusterTransaction() transaction.info() - expected = transaction_test(transaction, keyslot) + expected = await transaction_test(transaction, keyslot, redis_client) result = await redis_client.exec(transaction) assert isinstance(result, list) assert isinstance(result[0], str) @@ -291,7 +303,7 @@ async def test_standalone_transaction(self, redis_client: RedisClient): transaction.get(key) transaction.select(0) transaction.get(key) - expected = transaction_test(transaction, keyslot) + expected = await transaction_test(transaction, keyslot, redis_client) result = await redis_client.exec(transaction) assert isinstance(result, list) assert isinstance(result[0], str) diff --git a/utils/cluster_manager.py b/utils/cluster_manager.py index 650d7f9095..8a65685d4c 100644 --- a/utils/cluster_manager.py +++ b/utils/cluster_manager.py @@ -426,7 +426,7 @@ def create_cluster( stderr=subprocess.PIPE, text=True, ) - output, err = p.communicate(timeout=20) + output, err = p.communicate(timeout=40) if err or "[OK] All 16384 slots covered." not in output: raise Exception(f"Failed to create cluster: {err if err else output}") @@ -963,7 +963,9 @@ def main(): tic = time.perf_counter() cluster_prefix = f"tls-{args.prefix}" if args.tls else args.prefix cluster_folder = create_cluster_folder(args.folder_path, cluster_prefix) - logging.info(f"{datetime.now(timezone.utc)} Starting script for cluster {cluster_folder}") + logging.info( + f"{datetime.now(timezone.utc)} Starting script for cluster {cluster_folder}" + ) logfile = ( f"{cluster_folder}/cluster_manager.log" if not args.logfile