From 71cf8a28431d2792128cb1364c7e75872b1c908b Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 15 Jan 2020 10:29:41 +0000 Subject: [PATCH 01/21] Drop Python <3.6; add Python 3.8 --- .travis.yml | 7 ++--- CHANGELOG | 9 ++++++ asyncio_redis/protocol.py | 2 +- asyncio_redis/replies.py | 9 +++--- docs/conf.py | 5 +-- examples/streaming-multi-bulk/test.py | 2 +- setup.py | 44 +++++++++++++-------------- tests.py | 16 +++++----- 8 files changed, 51 insertions(+), 43 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3fbc309..e591e3f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,15 @@ language: python python: - - "3.3" - - "3.4" - - "3.5" + - "3.6" + - "3.7" + - "3.8" before_install: - sudo apt-get install redis-server - sudo service redis-server start - redis-server --version - which redis-server - - /bin/sleep 10 install: - "pip install ." diff --git a/CHANGELOG b/CHANGELOG index 221eddb..0e756aa 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,15 @@ CHANGELOG ========= +0.16.0: unreleased +------------------ +- Added support for Python 3.8 +- Dropped support for Python <3.6 +- Added than support for zpopmin and zrangebylex methods +- Fix DeprecationWarning for inspect.formatargspec() +- Fix missing object asyncio.streams.IncompleteReadError + + 0.15.1: 2018-07-30 ------------------ diff --git a/asyncio_redis/protocol.py b/asyncio_redis/protocol.py index e6f01f1..152d568 100644 --- a/asyncio_redis/protocol.py +++ b/asyncio_redis/protocol.py @@ -940,7 +940,7 @@ def _reader_coroutine(self): yield from self._handle_item(self._push_answer) except ConnectionLostError: return - except asyncio.streams.IncompleteReadError: + except asyncio.IncompleteReadError: return @asyncio.coroutine diff --git a/asyncio_redis/replies.py b/asyncio_redis/replies.py index 00bfbef..e66eb24 100644 --- a/asyncio_redis/replies.py +++ b/asyncio_redis/replies.py @@ -1,5 +1,4 @@ import asyncio -from asyncio.tasks import gather __all__ = ( 'BlockingPopReply', @@ -56,12 +55,12 @@ def _parse(self, key, value): return key, value def __iter__(self): - """ Yield a list of futures that yield { key: value } tuples. """ - i = iter(self._result) - + """Yield a list of futures that yield { key: value } tuples + """ @asyncio.coroutine def getter(f): - """ Coroutine which processes one item. """ + """Coroutine which processes one item + """ key, value = yield from f key, value = self._parse(key, value) return key, value diff --git a/docs/conf.py b/docs/conf.py index efdf945..92f86e0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,7 +11,8 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import os + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -99,7 +100,7 @@ # a list of builtin themes. #html_theme = 'default' -import os + on_rtd = os.environ.get('READTHEDOCS', None) == 'True' if on_rtd: diff --git a/examples/streaming-multi-bulk/test.py b/examples/streaming-multi-bulk/test.py index caffb41..edec3c1 100755 --- a/examples/streaming-multi-bulk/test.py +++ b/examples/streaming-multi-bulk/test.py @@ -46,7 +46,7 @@ def run(): # Stream the items, this will probably wait for the next IP packets to come in. count = 0 for f in set_reply: - m = yield from f + yield from f count += 1 if count % 1000 == 0: print('Received %i items' % count) diff --git a/setup.py b/setup.py index 00c483f..4259e6a 100755 --- a/setup.py +++ b/setup.py @@ -1,27 +1,27 @@ #!/usr/bin/env python -try: - from setuptools import setup -except ImportError: - from distutils.core import setup -import sys +from setuptools import setup -if sys.version_info >= (3, 4): - install_requires = [] -else: - install_requires = ['asyncio'] setup( - name='asyncio_redis', - author='Jonathan Slenders', - version='0.15.1', - license='LICENSE.txt', - url='https://github.com/jonathanslenders/asyncio-redis', - - description='PEP 3156 implementation of the redis protocol.', - long_description=open("README.rst").read(), - packages=['asyncio_redis'], - install_requires=install_requires, - extras_require={ - 'hiredis': ['hiredis'], - }, + name='asyncio_redis', + author='Jonathan Slenders', + version='0.15.1', + license='LICENSE.txt', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Framework :: AsyncIO', + 'Intended Audience :: Developers', + 'Topic :: Database', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + ], + url='https://github.com/jonathanslenders/asyncio-redis', + description='PEP 3156 implementation of the redis protocol.', + long_description=open("README.rst").read(), + packages=['asyncio_redis'], + python_requires='>=3.6', + extras_require={'hiredis': ['hiredis']}, ) diff --git a/tests.py b/tests.py index bf96c07..020d02e 100755 --- a/tests.py +++ b/tests.py @@ -42,7 +42,7 @@ import unittest import os import gc -import warnings + try: import hiredis @@ -108,7 +108,7 @@ def c(): class TestCase(unittest.TestCase): def tearDown(self): # Collect garbage on tearDown. (This can print ResourceWarnings.) - result = gc.collect() + gc.collect() class RedisProtocolTest(TestCase): @@ -619,17 +619,17 @@ def test_blocking_timeout(self, transport, protocol): # brpoplpush with self.assertRaises(TimeoutError) as e: - result = yield from protocol.brpoplpush(u'from', u'to', 1) + yield from protocol.brpoplpush(u'from', u'to', 1) self.assertIn('Timeout in brpoplpush', e.exception.args[0]) # brpop with self.assertRaises(TimeoutError) as e: - result = yield from protocol.brpop([u'from'], 1) + yield from protocol.brpop([u'from'], 1) self.assertIn('Timeout in blocking pop', e.exception.args[0]) # blpop with self.assertRaises(TimeoutError) as e: - result = yield from protocol.blpop([u'from'], 1) + yield from protocol.blpop([u'from'], 1) self.assertIn('Timeout in blocking pop', e.exception.args[0]) @redis_test @@ -1915,7 +1915,7 @@ def listener(): @asyncio.coroutine def sender(): - value = yield from protocol.publish(b'our_channel', b'message1') + yield from protocol.publish(b'our_channel', b'message1') f = ensure_future(listener(), loop=self.loop) yield from asyncio.sleep(.5, loop=self.loop) @@ -2349,14 +2349,14 @@ def _start_redis_server(loop): return redis_srv -@unittest.skipIf(hiredis == None, 'Hiredis not found.') +@unittest.skipIf(hiredis is None, 'Hiredis not found.') class HiRedisProtocolTest(RedisProtocolTest): def setUp(self): super().setUp() self.protocol_class = HiRedisProtocol -@unittest.skipIf(hiredis == None, 'Hiredis not found.') +@unittest.skipIf(hiredis is None, 'Hiredis not found.') class HiRedisBytesProtocolTest(RedisBytesProtocolTest): def setUp(self): self.loop = asyncio.get_event_loop() From 520c999c308708b26d70b54b787090e105fc5a8e Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 15 Jan 2020 10:46:53 +0000 Subject: [PATCH 02/21] tests header cleanup --- tests.py | 61 +++++++++++++++++++++++--------------------------------- 1 file changed, 25 insertions(+), 36 deletions(-) diff --git a/tests.py b/tests.py index 020d02e..2a0c745 100755 --- a/tests.py +++ b/tests.py @@ -1,8 +1,15 @@ #!/usr/bin/env python -from asyncio.futures import Future -from asyncio.tasks import gather +import asyncio from asyncio.test_utils import run_briefly +import gc +import os +import unittest + +try: + import hiredis +except ImportError: + hiredis = None from asyncio_redis import ( Connection, @@ -38,29 +45,11 @@ from asyncio_redis.cursors import Cursor from asyncio_redis.encoders import BytesEncoder -import asyncio -import unittest -import os -import gc - - -try: - import hiredis -except ImportError: - hiredis = None - PORT = int(os.environ.get('REDIS_PORT', 6379)) HOST = os.environ.get('REDIS_HOST', 'localhost') START_REDIS_SERVER = bool(os.environ.get('START_REDIS_SERVER', False)) -# In Python 3.4.4, `async` was renamed to `ensure_future`. -try: - ensure_future = asyncio.ensure_future -except AttributeError: - ensure_future = getattr(asyncio, "async") - - @asyncio.coroutine def connect(loop, protocol=RedisProtocol): """ Connect to redis server. Return transport/protocol pair. """ @@ -571,7 +560,7 @@ def blpop(): self.assertEqual(value.list_name, u'my_list') self.assertEqual(value.value, u'value') test_order.append('#3') - f = ensure_future(blpop(), loop=self.loop) + f = asyncio.ensure_future(blpop(), loop=self.loop) transport2, protocol2 = yield from connect(self.loop) @@ -587,7 +576,7 @@ def blpop(): self.assertIsInstance(value, BlockingPopReply) self.assertEqual(value.list_name, u'my_list') self.assertEqual(value.value, u'value2') - f = ensure_future(blpop(), loop=self.loop) + f = asyncio.ensure_future(blpop(), loop=self.loop) yield from protocol2.rpush(u'my_list', [u'value2']) yield from f @@ -604,7 +593,7 @@ def test_brpoplpush(self, transport, protocol): def brpoplpush(): result = yield from protocol.brpoplpush(u'from', u'to') self.assertEqual(result, u'my_value') - f = ensure_future(brpoplpush(), loop=self.loop) + f = asyncio.ensure_future(brpoplpush(), loop=self.loop) transport2, protocol2 = yield from connect(self.loop) yield from protocol2.rpush(u'from', [u'my_value']) @@ -845,7 +834,7 @@ def listener(): return transport2 - f = ensure_future(listener(), loop=self.loop) + f = asyncio.ensure_future(listener(), loop=self.loop) @asyncio.coroutine def sender(): @@ -913,7 +902,7 @@ def listener(): transport2.close() - f = ensure_future(listener(), loop=self.loop) + f = asyncio.ensure_future(listener(), loop=self.loop) @asyncio.coroutine def sender(): @@ -955,7 +944,7 @@ def listener(): transport2.close() - f = ensure_future(listener(), loop=self.loop) + f = asyncio.ensure_future(listener(), loop=self.loop) @asyncio.coroutine def sender(): @@ -1448,7 +1437,7 @@ def run_while_true(): transport.close() # (start script) - f = ensure_future(run_while_true(), loop=self.loop) + f = asyncio.ensure_future(run_while_true(), loop=self.loop) yield from asyncio.sleep(.5, loop=self.loop) result = yield from protocol.script_kill() @@ -1480,7 +1469,7 @@ def test_transaction(self, transport, protocol): f5 = yield from transaction.hgetall('my_hash') for f in [ f1, f2, f3, f4, f5]: - self.assertIsInstance(f, Future) + self.assertIsInstance(f, asyncio.Future) # Calling subscribe inside transaction should fail. with self.assertRaises(Error) as e: @@ -1516,7 +1505,7 @@ def test_run_command_outside_transaction(self, transport, protocol): # Run command, but not as part of the transaction. # This should wait until the transaction finishes. - f = ensure_future(protocol.set('a', 'b'), loop=self.loop) + f = asyncio.ensure_future(protocol.set('a', 'b'), loop=self.loop) # Close transaction. yield from transaction.exec() @@ -1790,7 +1779,7 @@ def test_cancellation(self, transport, protocol): @asyncio.coroutine def run(): yield from protocol.brpop(['key'], 3) - f = ensure_future(run(), loop=self.loop) + f = asyncio.ensure_future(run(), loop=self.loop) # We cancel the coroutine before the answer arrives. yield from asyncio.sleep(.5, loop=self.loop) @@ -1917,7 +1906,7 @@ def listener(): def sender(): yield from protocol.publish(b'our_channel', b'message1') - f = ensure_future(listener(), loop=self.loop) + f = asyncio.ensure_future(listener(), loop=self.loop) yield from asyncio.sleep(.5, loop=self.loop) yield from sender() transport2 = yield from f @@ -2009,7 +1998,7 @@ def test(): # Wait for ever. (This blocking pop doesn't return.) yield from connection.delete([ 'unknown-key' ]) - f = ensure_future(connection.blpop(['unknown-key']), loop=self.loop) + f = asyncio.ensure_future(connection.blpop(['unknown-key']), loop=self.loop) yield from asyncio.sleep(.1, loop=self.loop) # Sleep to make sure that the above coroutine started executing. # Run command in other thread. @@ -2057,9 +2046,9 @@ def source(): yield from asyncio.sleep(.5, loop=self.loop) # Run both coroutines. - f1 = ensure_future(source(), loop=self.loop) - f2 = ensure_future(sink(), loop=self.loop) - yield from gather(f1, f2) + f1 = asyncio.ensure_future(source(), loop=self.loop) + f2 = asyncio.ensure_future(sink(), loop=self.loop) + yield from asyncio.gather(f1, f2) # Test results. self.assertEqual(results, [ str(i) for i in range(0, 5) ]) @@ -2112,7 +2101,7 @@ def sink(i): futures = [] for i in range(0, 10): self.assertEqual(connection.connections_in_use, i) - futures.append(ensure_future(sink(i), loop=self.loop)) + futures.append(asyncio.ensure_future(sink(i), loop=self.loop)) yield from asyncio.sleep(.1, loop=self.loop) # Sleep to make sure that the above coroutine started executing. # One more blocking call should fail. From ec5ea087dbd1d2f675876489e37f058389ca7579 Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 15 Jan 2020 11:41:55 +0000 Subject: [PATCH 03/21] Migrate tests.py to Python >=3.6 --- tests.py | 1561 ++++++++++++++++++++++++++---------------------------- 1 file changed, 762 insertions(+), 799 deletions(-) diff --git a/tests.py b/tests.py index 2a0c745..483f8e4 100755 --- a/tests.py +++ b/tests.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import asyncio -from asyncio.test_utils import run_briefly import gc import os import unittest @@ -50,17 +49,13 @@ START_REDIS_SERVER = bool(os.environ.get('START_REDIS_SERVER', False)) -@asyncio.coroutine -def connect(loop, protocol=RedisProtocol): +async def connect(protocol=RedisProtocol): """ Connect to redis server. Return transport/protocol pair. """ + loop = asyncio.get_event_loop() if PORT: - transport, protocol = yield from loop.create_connection( - lambda: protocol(loop=loop), HOST, PORT) - return transport, protocol + return await loop.create_connection(lambda: protocol(loop=loop), HOST, PORT) else: - transport, protocol = yield from loop.create_unix_connection( - lambda: protocol(loop=loop), HOST) - return transport, protocol + return await loop.create_unix_connection(lambda: protocol(loop=loop), HOST) def redis_test(function): @@ -69,28 +64,24 @@ def redis_test(function): Wraps the coroutine inside `run_until_complete`. """ - function = asyncio.coroutine(function) - def wrapper(self): - @asyncio.coroutine - def c(): + async def c(): # Create connection - transport, protocol = yield from connect(self.loop, self.protocol_class) + transport, protocol = await connect(self.protocol_class) # Run test try: - yield from function(self, transport, protocol) + await function(self, transport, protocol) # Close connection finally: transport.close() + # Run potential pending clean up callbacks + await asyncio.sleep(0) + self.loop.run_until_complete(c()) - # Run one more iteration of the event loop in order to run potential - # pending clean up callbacks. (We cannot use loop._run_once(), because - # that would block if there are no pending ready callbacks.) - self.loop.run_until_complete(asyncio.sleep(0, loop=self.loop)) return wrapper @@ -106,727 +97,721 @@ def setUp(self): self.protocol_class = RedisProtocol @redis_test - def test_ping(self, transport, protocol): - result = yield from protocol.ping() + async def test_ping(self, transport, protocol): + result = await protocol.ping() self.assertEqual(result, StatusReply('PONG')) self.assertEqual(repr(result), u"StatusReply(status='PONG')") @redis_test - def test_echo(self, transport, protocol): - result = yield from protocol.echo(u'my string') + async def test_echo(self, transport, protocol): + result = await protocol.echo(u'my string') self.assertEqual(result, u'my string') @redis_test - def test_set_and_get(self, transport, protocol): + async def test_set_and_get(self, transport, protocol): # Set - value = yield from protocol.set(u'my_key', u'my_value') + value = await protocol.set(u'my_key', u'my_value') self.assertEqual(value, StatusReply('OK')) # Get - value = yield from protocol.get(u'my_key') + value = await protocol.get(u'my_key') self.assertEqual(value, u'my_value') # Getset - value = yield from protocol.getset(u'my_key', u'new_value') + value = await protocol.getset(u'my_key', u'new_value') self.assertEqual(value, u'my_value') - value = yield from protocol.get(u'my_key') + value = await protocol.get(u'my_key') self.assertEqual(value, u'new_value') @redis_test - def test_extended_set(self, transport, protocol): - yield from protocol.delete([u'my_key', u'other_key']) + async def test_extended_set(self, transport, protocol): + await protocol.delete([u'my_key', u'other_key']) # set with expire only if not exists - value = yield from protocol.set(u'my_key', u'my_value', - expire=10, only_if_not_exists=True) + value = await protocol.set(u'my_key', u'my_value', + expire=10, only_if_not_exists=True) self.assertEqual(value, StatusReply('OK')) - value = yield from protocol.ttl(u'my_key') + value = await protocol.ttl(u'my_key') self.assertIn(value, (10, 9)) # check NX flag for SET command - value = yield from protocol.set(u'my_key', u'my_value', - expire=10, only_if_not_exists=True) + value = await protocol.set(u'my_key', u'my_value', + expire=10, only_if_not_exists=True) self.assertIsNone(value) # check XX flag for SET command - value = yield from protocol.set(u'other_key', 'some_value', only_if_exists=True) + value = await protocol.set(u'other_key', 'some_value', only_if_exists=True) self.assertIsNone(value) # set with pexpire only if key exists - value = yield from protocol.set(u'my_key', u'other_value', - pexpire=20000, only_if_exists=True) + value = await protocol.set(u'my_key', u'other_value', + pexpire=20000, only_if_exists=True) self.assertEqual(value, StatusReply('OK')) - value = yield from protocol.get(u'my_key') + value = await protocol.get(u'my_key') self.assertEqual(value, u'other_value') - value = yield from protocol.ttl(u'my_key') + value = await protocol.ttl(u'my_key') self.assertIn(value, (20, 19)) @redis_test - def test_setex(self, transport, protocol): + async def test_setex(self, transport, protocol): # Set - value = yield from protocol.setex(u'my_key', 10, u'my_value') + value = await protocol.setex(u'my_key', 10, u'my_value') self.assertEqual(value, StatusReply('OK')) # TTL - value = yield from protocol.ttl(u'my_key') + value = await protocol.ttl(u'my_key') self.assertIn(value, (10, 9)) # may be some delay # Get - value = yield from protocol.get(u'my_key') + value = await protocol.get(u'my_key') self.assertEqual(value, u'my_value') @redis_test - def test_setnx(self, transport, protocol): - yield from protocol.delete([u'my_key']) + async def test_setnx(self, transport, protocol): + await protocol.delete([u'my_key']) # Setnx while key does not exists - value = yield from protocol.setnx(u'my_key', u'my_value') + value = await protocol.setnx(u'my_key', u'my_value') self.assertEqual(value, True) # Get - value = yield from protocol.get(u'my_key') + value = await protocol.get(u'my_key') self.assertEqual(value, u'my_value') # Setnx if key exists - value = yield from protocol.setnx(u'my_key', u'other_value') + value = await protocol.setnx(u'my_key', u'other_value') self.assertEqual(value, False) # Get old value - value = yield from protocol.get(u'my_key') + value = await protocol.get(u'my_key') self.assertEqual(value, u'my_value') @redis_test - def test_special_characters(self, transport, protocol): + async def test_special_characters(self, transport, protocol): # Test some special unicode values and spaces. value = u'my value with special chars " # éçåø´¨åø´h ' - result = yield from protocol.set(u'my key with spaces', value) - result = yield from protocol.get(u'my key with spaces') + result = await protocol.set(u'my key with spaces', value) + result = await protocol.get(u'my key with spaces') self.assertEqual(result, value) # Test newlines value = u'ab\ncd\ref\r\ngh' - result = yield from protocol.set(u'my-key', value) - result = yield from protocol.get(u'my-key') + result = await protocol.set(u'my-key', value) + result = await protocol.get(u'my-key') self.assertEqual(result, value) @redis_test - def test_mget(self, transport, protocol): + async def test_mget(self, transport, protocol): # mget - yield from protocol.set(u'my_key', u'a') - yield from protocol.set(u'my_key2', u'b') - result = yield from protocol.mget([ u'my_key', u'my_key2', u'not_exists']) + await protocol.set(u'my_key', u'a') + await protocol.set(u'my_key2', u'b') + result = await protocol.mget([ u'my_key', u'my_key2', u'not_exists']) self.assertIsInstance(result, ListReply) - result = yield from result.aslist() + result = await result.aslist() self.assertEqual(result, [u'a', u'b', None]) @redis_test - def test_strlen(self, transport, protocol): - yield from protocol.delete([ u'my_key' ]) - yield from protocol.delete([ u'my_key2' ]) - yield from protocol.delete([ u'my_key3' ]) - yield from protocol.set(u'my_key', u'my_value') - yield from protocol.hset(u'my_key3', u'a', u'b') + async def test_strlen(self, transport, protocol): + await protocol.delete([ u'my_key' ]) + await protocol.delete([ u'my_key2' ]) + await protocol.delete([ u'my_key3' ]) + await protocol.set(u'my_key', u'my_value') + await protocol.hset(u'my_key3', u'a', u'b') # strlen - value = yield from protocol.strlen(u'my_key') + value = await protocol.strlen(u'my_key') self.assertEqual(value, len(u'my_value')) - value = yield from protocol.strlen(u'my_key2') + value = await protocol.strlen(u'my_key2') self.assertEqual(value, 0) with self.assertRaises(ErrorReply): - yield from protocol.strlen(u'my_key3') + await protocol.strlen(u'my_key3') # Redis exception: b'ERR Operation against a key holding the wrong kind of value') @redis_test - def test_exists_and_delete(self, transport, protocol): + async def test_exists_and_delete(self, transport, protocol): # Set - yield from protocol.set(u'my_key', u'aaa') - value = yield from protocol.append(u'my_key', u'bbb') + await protocol.set(u'my_key', u'aaa') + value = await protocol.append(u'my_key', u'bbb') self.assertEqual(value, 6) # Total length - value = yield from protocol.get(u'my_key') + value = await protocol.get(u'my_key') self.assertEqual(value, u'aaabbb') @redis_test - def test_exists_and_delete2(self, transport, protocol): + async def test_exists_and_delete2(self, transport, protocol): # Exists - value = yield from protocol.exists(u'unknown_key') + value = await protocol.exists(u'unknown_key') self.assertEqual(value, False) # Set - value = yield from protocol.set(u'known_key', u'value') - value = yield from protocol.exists(u'known_key') + value = await protocol.set(u'known_key', u'value') + value = await protocol.exists(u'known_key') self.assertEqual(value, True) # Delete - value = yield from protocol.set(u'known_key2', u'value') - value = yield from protocol.delete([ u'known_key', u'known_key2' ]) + value = await protocol.set(u'known_key2', u'value') + value = await protocol.delete([ u'known_key', u'known_key2' ]) self.assertEqual(value, 2) - value = yield from protocol.delete([ u'known_key' ]) + value = await protocol.delete([ u'known_key' ]) self.assertEqual(value, 0) - value = yield from protocol.exists(u'known_key') + value = await protocol.exists(u'known_key') self.assertEqual(value, False) @redis_test - def test_rename(self, transport, protocol): + async def test_rename(self, transport, protocol): # Set - value = yield from protocol.set(u'old_key', u'value') - value = yield from protocol.exists(u'old_key') + value = await protocol.set(u'old_key', u'value') + value = await protocol.exists(u'old_key') self.assertEqual(value, True) # Rename - value = yield from protocol.rename(u'old_key', u'new_key') + value = await protocol.rename(u'old_key', u'new_key') self.assertEqual(value, StatusReply('OK')) - value = yield from protocol.exists(u'old_key') + value = await protocol.exists(u'old_key') self.assertEqual(value, False) - value = yield from protocol.exists(u'new_key') + value = await protocol.exists(u'new_key') self.assertEqual(value, True) - value = yield from protocol.get(u'old_key') + value = await protocol.get(u'old_key') self.assertEqual(value, None) - value = yield from protocol.get(u'new_key') + value = await protocol.get(u'new_key') self.assertEqual(value, 'value') # RenameNX - yield from protocol.delete([ u'key3' ]) - value = yield from protocol.renamenx(u'new_key', u'key3') + await protocol.delete([ u'key3' ]) + value = await protocol.renamenx(u'new_key', u'key3') self.assertEqual(value, 1) - yield from protocol.set(u'key4', u'existing-value') - value = yield from protocol.renamenx(u'key3', u'key4') + await protocol.set(u'key4', u'existing-value') + value = await protocol.renamenx(u'key3', u'key4') self.assertEqual(value, 0) @redis_test - def test_expire(self, transport, protocol): + async def test_expire(self, transport, protocol): # Set - value = yield from protocol.set(u'key', u'value') + value = await protocol.set(u'key', u'value') # Expire (10s) - value = yield from protocol.expire(u'key', 10) + value = await protocol.expire(u'key', 10) self.assertEqual(value, 1) - value = yield from protocol.exists(u'key') + value = await protocol.exists(u'key') self.assertEqual(value, True) # TTL - value = yield from protocol.ttl(u'key') + value = await protocol.ttl(u'key') self.assertIsInstance(value, int) self.assertLessEqual(value, 10) # PTTL - value = yield from protocol.pttl(u'key') + value = await protocol.pttl(u'key') self.assertIsInstance(value, int) self.assertLessEqual(value, 10 * 1000) # Pexpire - value = yield from protocol.pexpire(u'key', 10*1000) + value = await protocol.pexpire(u'key', 10*1000) self.assertEqual(value, 1) # XXX: check this - value = yield from protocol.pttl(u'key') + value = await protocol.pttl(u'key') self.assertLessEqual(value, 10 * 1000) # Expire (1s) and wait - value = yield from protocol.expire(u'key', 1) - value = yield from protocol.exists(u'key') + value = await protocol.expire(u'key', 1) + value = await protocol.exists(u'key') self.assertEqual(value, True) - yield from asyncio.sleep(2, loop=self.loop) + await asyncio.sleep(2) - value = yield from protocol.exists(u'key') + value = await protocol.exists(u'key') self.assertEqual(value, False) # Test persist - yield from protocol.set(u'key', u'value') - yield from protocol.expire(u'key', 1) - value = yield from protocol.persist(u'key') + await protocol.set(u'key', u'value') + await protocol.expire(u'key', 1) + value = await protocol.persist(u'key') self.assertEqual(value, 1) - value = yield from protocol.persist(u'key') + value = await protocol.persist(u'key') self.assertEqual(value, 0) - yield from asyncio.sleep(2, loop=self.loop) + await asyncio.sleep(2) - value = yield from protocol.exists(u'key') + value = await protocol.exists(u'key') self.assertEqual(value, True) # Test expireat - value = yield from protocol.expireat(u'key', 1293840000) + value = await protocol.expireat(u'key', 1293840000) self.assertIsInstance(value, int) # Test pexpireat - value = yield from protocol.pexpireat(u'key', 1555555555005) + value = await protocol.pexpireat(u'key', 1555555555005) self.assertIsInstance(value, int) @redis_test - def test_set(self, transport, protocol): + async def test_set(self, transport, protocol): # Create set - value = yield from protocol.delete([ u'our_set' ]) - value = yield from protocol.sadd(u'our_set', [u'a', u'b']) - value = yield from protocol.sadd(u'our_set', [u'c']) + value = await protocol.delete([ u'our_set' ]) + value = await protocol.sadd(u'our_set', [u'a', u'b']) + value = await protocol.sadd(u'our_set', [u'c']) self.assertEqual(value, 1) # scard - value = yield from protocol.scard(u'our_set') + value = await protocol.scard(u'our_set') self.assertEqual(value, 3) # Smembers - value = yield from protocol.smembers(u'our_set') + value = await protocol.smembers(u'our_set') self.assertIsInstance(value, SetReply) self.assertEqual(repr(value), u"SetReply(length=3)") - value = yield from value.asset() + value = await value.asset() self.assertEqual(value, { u'a', u'b', u'c' }) # sismember - value = yield from protocol.sismember(u'our_set', 'a') + value = await protocol.sismember(u'our_set', 'a') self.assertEqual(value, True) - value = yield from protocol.sismember(u'our_set', 'd') + value = await protocol.sismember(u'our_set', 'd') self.assertEqual(value, False) # Intersection, union and diff - yield from protocol.delete([ u'set2' ]) - yield from protocol.sadd(u'set2', [u'b', u'c', u'd', u'e']) + await protocol.delete([ u'set2' ]) + await protocol.sadd(u'set2', [u'b', u'c', u'd', u'e']) - value = yield from protocol.sunion([ u'our_set', 'set2' ]) + value = await protocol.sunion([ u'our_set', 'set2' ]) self.assertIsInstance(value, SetReply) - value = yield from value.asset() + value = await value.asset() self.assertEqual(value, set([u'a', u'b', u'c', u'd', u'e'])) - value = yield from protocol.sinter([ u'our_set', 'set2' ]) - value = yield from value.asset() + value = await protocol.sinter([ u'our_set', 'set2' ]) + value = await value.asset() self.assertEqual(value, set([u'b', u'c'])) - value = yield from protocol.sdiff([ u'our_set', 'set2' ]) + value = await protocol.sdiff([ u'our_set', 'set2' ]) self.assertIsInstance(value, SetReply) - value = yield from value.asset() + value = await value.asset() self.assertEqual(value, set([u'a'])) - value = yield from protocol.sdiff([ u'set2', u'our_set' ]) - value = yield from value.asset() + value = await protocol.sdiff([ u'set2', u'our_set' ]) + value = await value.asset() self.assertEqual(value, set([u'd', u'e'])) # Interstore - value = yield from protocol.sinterstore(u'result', [u'our_set', 'set2']) + value = await protocol.sinterstore(u'result', [u'our_set', 'set2']) self.assertEqual(value, 2) - value = yield from protocol.smembers(u'result') + value = await protocol.smembers(u'result') self.assertIsInstance(value, SetReply) - value = yield from value.asset() + value = await value.asset() self.assertEqual(value, set([u'b', u'c'])) # Unionstore - value = yield from protocol.sunionstore(u'result', [u'our_set', 'set2']) + value = await protocol.sunionstore(u'result', [u'our_set', 'set2']) self.assertEqual(value, 5) - value = yield from protocol.smembers(u'result') + value = await protocol.smembers(u'result') self.assertIsInstance(value, SetReply) - value = yield from value.asset() + value = await value.asset() self.assertEqual(value, set([u'a', u'b', u'c', u'd', u'e'])) # Sdiffstore - value = yield from protocol.sdiffstore(u'result', [u'set2', 'our_set']) + value = await protocol.sdiffstore(u'result', [u'set2', 'our_set']) self.assertEqual(value, 2) - value = yield from protocol.smembers(u'result') + value = await protocol.smembers(u'result') self.assertIsInstance(value, SetReply) - value = yield from value.asset() + value = await value.asset() self.assertEqual(value, set([u'd', u'e'])) @redis_test - def test_srem(self, transport, protocol): - yield from protocol.delete([ u'our_set' ]) - yield from protocol.sadd(u'our_set', [u'a', u'b', u'c', u'd']) + async def test_srem(self, transport, protocol): + await protocol.delete([ u'our_set' ]) + await protocol.sadd(u'our_set', [u'a', u'b', u'c', u'd']) # Call srem - result = yield from protocol.srem(u'our_set', [u'b', u'c']) + result = await protocol.srem(u'our_set', [u'b', u'c']) self.assertEqual(result, 2) - result = yield from protocol.smembers(u'our_set') + result = await protocol.smembers(u'our_set') self.assertIsInstance(result, SetReply) - result = yield from result.asset() + result = await result.asset() self.assertEqual(result, set([u'a', u'd'])) @redis_test - def test_spop(self, transport, protocol): - @asyncio.coroutine - def setup(): - yield from protocol.delete([ u'my_set' ]) - yield from protocol.sadd(u'my_set', [u'value1']) - yield from protocol.sadd(u'my_set', [u'value2']) + async def test_spop(self, transport, protocol): + async def setup(): + await protocol.delete([ u'my_set' ]) + await protocol.sadd(u'my_set', [u'value1']) + await protocol.sadd(u'my_set', [u'value2']) # Test spop - yield from setup() - result = yield from protocol.spop(u'my_set') + await setup() + result = await protocol.spop(u'my_set') self.assertIn(result, [u'value1', u'value2']) - result = yield from protocol.smembers(u'my_set') + result = await protocol.smembers(u'my_set') self.assertIsInstance(result, SetReply) - result = yield from result.asset() + result = await result.asset() self.assertEqual(len(result), 1) # Test srandmember - yield from setup() - result = yield from protocol.srandmember(u'my_set') + await setup() + result = await protocol.srandmember(u'my_set') self.assertIsInstance(result, SetReply) - result = yield from result.asset() + result = await result.asset() self.assertIn(list(result)[0], [u'value1', u'value2']) - result = yield from protocol.smembers(u'my_set') + result = await protocol.smembers(u'my_set') self.assertIsInstance(result, SetReply) - result = yield from result.asset() + result = await result.asset() self.assertEqual(len(result), 2) # Popping from non-existing key should return None. - yield from protocol.delete([ u'my_set' ]) - result = yield from protocol.spop(u'my_set') + await protocol.delete([ u'my_set' ]) + result = await protocol.spop(u'my_set') self.assertEqual(result, None) @redis_test - def test_type(self, transport, protocol): + async def test_type(self, transport, protocol): # Setup - yield from protocol.delete([ u'key1' ]) - yield from protocol.delete([ u'key2' ]) - yield from protocol.delete([ u'key3' ]) + await protocol.delete([ u'key1' ]) + await protocol.delete([ u'key2' ]) + await protocol.delete([ u'key3' ]) - yield from protocol.set(u'key1', u'value') - yield from protocol.lpush(u'key2', [u'value']) - yield from protocol.sadd(u'key3', [u'value']) + await protocol.set(u'key1', u'value') + await protocol.lpush(u'key2', [u'value']) + await protocol.sadd(u'key3', [u'value']) # Test types - value = yield from protocol.type(u'key1') + value = await protocol.type(u'key1') self.assertEqual(value, StatusReply('string')) - value = yield from protocol.type(u'key2') + value = await protocol.type(u'key2') self.assertEqual(value, StatusReply('list')) - value = yield from protocol.type(u'key3') + value = await protocol.type(u'key3') self.assertEqual(value, StatusReply('set')) @redis_test - def test_list(self, transport, protocol): + async def test_list(self, transport, protocol): # Create list - yield from protocol.delete([ u'my_list' ]) - value = yield from protocol.lpush(u'my_list', [u'v1', u'v2']) - value = yield from protocol.rpush(u'my_list', [u'v3', u'v4']) + await protocol.delete([ u'my_list' ]) + value = await protocol.lpush(u'my_list', [u'v1', u'v2']) + value = await protocol.rpush(u'my_list', [u'v3', u'v4']) self.assertEqual(value, 4) # lrange - value = yield from protocol.lrange(u'my_list') + value = await protocol.lrange(u'my_list') self.assertIsInstance(value, ListReply) self.assertEqual(repr(value), u"ListReply(length=4)") - value = yield from value.aslist() + value = await value.aslist() self.assertEqual(value, [ u'v2', 'v1', 'v3', 'v4']) # lset - value = yield from protocol.lset(u'my_list', 3, 'new-value') + value = await protocol.lset(u'my_list', 3, 'new-value') self.assertEqual(value, StatusReply('OK')) - value = yield from protocol.lrange(u'my_list') + value = await protocol.lrange(u'my_list') self.assertIsInstance(value, ListReply) - value = yield from value.aslist() + value = await value.aslist() self.assertEqual(value, [ u'v2', 'v1', 'v3', 'new-value']) # lindex - value = yield from protocol.lindex(u'my_list', 1) + value = await protocol.lindex(u'my_list', 1) self.assertEqual(value, 'v1') - value = yield from protocol.lindex(u'my_list', 10) # Unknown index + value = await protocol.lindex(u'my_list', 10) # Unknown index self.assertEqual(value, None) # Length - value = yield from protocol.llen(u'my_list') + value = await protocol.llen(u'my_list') self.assertEqual(value, 4) # Remove element from list. - value = yield from protocol.lrem(u'my_list', value=u'new-value') + value = await protocol.lrem(u'my_list', value=u'new-value') self.assertEqual(value, 1) # Pop - value = yield from protocol.rpop(u'my_list') + value = await protocol.rpop(u'my_list') self.assertEqual(value, u'v3') - value = yield from protocol.lpop(u'my_list') + value = await protocol.lpop(u'my_list') self.assertEqual(value, u'v2') - value = yield from protocol.lpop(u'my_list') + value = await protocol.lpop(u'my_list') self.assertEqual(value, u'v1') - value = yield from protocol.lpop(u'my_list') + value = await protocol.lpop(u'my_list') self.assertEqual(value, None) # Blocking lpop test_order = [] - @asyncio.coroutine - def blpop(): + async def blpop(): test_order.append('#1') - value = yield from protocol.blpop([u'my_list']) + value = await protocol.blpop([u'my_list']) self.assertIsInstance(value, BlockingPopReply) self.assertEqual(value.list_name, u'my_list') self.assertEqual(value.value, u'value') test_order.append('#3') - f = asyncio.ensure_future(blpop(), loop=self.loop) + f = asyncio.ensure_future(blpop()) - transport2, protocol2 = yield from connect(self.loop) + transport2, protocol2 = await connect() test_order.append('#2') - yield from protocol2.rpush(u'my_list', [u'value']) - yield from f + await protocol2.rpush(u'my_list', [u'value']) + await f self.assertEqual(test_order, ['#1', '#2', '#3']) # Blocking rpop - @asyncio.coroutine - def blpop(): - value = yield from protocol.brpop([u'my_list']) + async def brpop(): + value = await protocol.brpop([u'my_list']) self.assertIsInstance(value, BlockingPopReply) self.assertEqual(value.list_name, u'my_list') self.assertEqual(value.value, u'value2') - f = asyncio.ensure_future(blpop(), loop=self.loop) + f = asyncio.ensure_future(brpop()) - yield from protocol2.rpush(u'my_list', [u'value2']) - yield from f + await protocol2.rpush(u'my_list', [u'value2']) + await f transport2.close() @redis_test - def test_brpoplpush(self, transport, protocol): - yield from protocol.delete([ u'from' ]) - yield from protocol.delete([ u'to' ]) - yield from protocol.lpush(u'to', [u'1']) + async def test_brpoplpush(self, transport, protocol): + await protocol.delete([ u'from' ]) + await protocol.delete([ u'to' ]) + await protocol.lpush(u'to', [u'1']) - @asyncio.coroutine - def brpoplpush(): - result = yield from protocol.brpoplpush(u'from', u'to') + async def brpoplpush(): + result = await protocol.brpoplpush(u'from', u'to') self.assertEqual(result, u'my_value') - f = asyncio.ensure_future(brpoplpush(), loop=self.loop) + f = asyncio.ensure_future(brpoplpush()) - transport2, protocol2 = yield from connect(self.loop) - yield from protocol2.rpush(u'from', [u'my_value']) - yield from f + transport2, protocol2 = await connect() + await protocol2.rpush(u'from', [u'my_value']) + await f transport2.close() @redis_test - def test_blocking_timeout(self, transport, protocol): - yield from protocol.delete([u'from']) - yield from protocol.delete([u'to']) + async def test_blocking_timeout(self, transport, protocol): + await protocol.delete([u'from']) + await protocol.delete([u'to']) # brpoplpush with self.assertRaises(TimeoutError) as e: - yield from protocol.brpoplpush(u'from', u'to', 1) + await protocol.brpoplpush(u'from', u'to', 1) self.assertIn('Timeout in brpoplpush', e.exception.args[0]) # brpop with self.assertRaises(TimeoutError) as e: - yield from protocol.brpop([u'from'], 1) + await protocol.brpop([u'from'], 1) self.assertIn('Timeout in blocking pop', e.exception.args[0]) # blpop with self.assertRaises(TimeoutError) as e: - yield from protocol.blpop([u'from'], 1) + await protocol.blpop([u'from'], 1) self.assertIn('Timeout in blocking pop', e.exception.args[0]) @redis_test - def test_linsert(self, transport, protocol): + async def test_linsert(self, transport, protocol): # Prepare - yield from protocol.delete([ u'my_list' ]) - yield from protocol.rpush(u'my_list', [u'1']) - yield from protocol.rpush(u'my_list', [u'2']) - yield from protocol.rpush(u'my_list', [u'3']) + await protocol.delete([ u'my_list' ]) + await protocol.rpush(u'my_list', [u'1']) + await protocol.rpush(u'my_list', [u'2']) + await protocol.rpush(u'my_list', [u'3']) # Insert after - result = yield from protocol.linsert(u'my_list', u'1', u'A') + result = await protocol.linsert(u'my_list', u'1', u'A') self.assertEqual(result, 4) - result = yield from protocol.lrange(u'my_list') + result = await protocol.lrange(u'my_list') self.assertIsInstance(result, ListReply) - result = yield from result.aslist() + result = await result.aslist() self.assertEqual(result, [u'1', u'A', u'2', u'3']) # Insert before - result = yield from protocol.linsert(u'my_list', u'3', u'B', before=True) + result = await protocol.linsert(u'my_list', u'3', u'B', before=True) self.assertEqual(result, 5) - result = yield from protocol.lrange(u'my_list') + result = await protocol.lrange(u'my_list') self.assertIsInstance(result, ListReply) - result = yield from result.aslist() + result = await result.aslist() self.assertEqual(result, [u'1', u'A', u'2', u'B', u'3']) @redis_test - def test_rpoplpush(self, transport, protocol): + async def test_rpoplpush(self, transport, protocol): # Prepare - yield from protocol.delete([ u'my_list' ]) - yield from protocol.delete([ u'my_list2' ]) - yield from protocol.lpush(u'my_list', [u'value']) - yield from protocol.lpush(u'my_list2', [u'value2']) + await protocol.delete([ u'my_list' ]) + await protocol.delete([ u'my_list2' ]) + await protocol.lpush(u'my_list', [u'value']) + await protocol.lpush(u'my_list2', [u'value2']) - value = yield from protocol.llen(u'my_list') - value2 = yield from protocol.llen(u'my_list2') + value = await protocol.llen(u'my_list') + value2 = await protocol.llen(u'my_list2') self.assertEqual(value, 1) self.assertEqual(value2, 1) # rpoplpush - result = yield from protocol.rpoplpush(u'my_list', u'my_list2') + result = await protocol.rpoplpush(u'my_list', u'my_list2') self.assertEqual(result, u'value') - result = yield from protocol.rpoplpush(u'my_list', u'my_list2') + result = await protocol.rpoplpush(u'my_list', u'my_list2') self.assertEqual(result, None) @redis_test - def test_pushx(self, transport, protocol): - yield from protocol.delete([ u'my_list' ]) + async def test_pushx(self, transport, protocol): + await protocol.delete([ u'my_list' ]) # rpushx - result = yield from protocol.rpushx(u'my_list', u'a') + result = await protocol.rpushx(u'my_list', u'a') self.assertEqual(result, 0) - yield from protocol.rpush(u'my_list', [u'a']) - result = yield from protocol.rpushx(u'my_list', u'a') + await protocol.rpush(u'my_list', [u'a']) + result = await protocol.rpushx(u'my_list', u'a') self.assertEqual(result, 2) # lpushx - yield from protocol.delete([ u'my_list' ]) - result = yield from protocol.lpushx(u'my_list', u'a') + await protocol.delete([ u'my_list' ]) + result = await protocol.lpushx(u'my_list', u'a') self.assertEqual(result, 0) - yield from protocol.rpush(u'my_list', [u'a']) - result = yield from protocol.lpushx(u'my_list', u'a') + await protocol.rpush(u'my_list', [u'a']) + result = await protocol.lpushx(u'my_list', u'a') self.assertEqual(result, 2) @redis_test - def test_ltrim(self, transport, protocol): - yield from protocol.delete([ u'my_list' ]) - yield from protocol.lpush(u'my_list', [u'a']) - yield from protocol.lpush(u'my_list', [u'b']) - result = yield from protocol.ltrim(u'my_list') + async def test_ltrim(self, transport, protocol): + await protocol.delete([ u'my_list' ]) + await protocol.lpush(u'my_list', [u'a']) + await protocol.lpush(u'my_list', [u'b']) + result = await protocol.ltrim(u'my_list') self.assertEqual(result, StatusReply('OK')) @redis_test - def test_hashes(self, transport, protocol): - yield from protocol.delete([ u'my_hash' ]) + async def test_hashes(self, transport, protocol): + await protocol.delete([ u'my_hash' ]) # Set in hash - result = yield from protocol.hset(u'my_hash', u'key', u'value') + result = await protocol.hset(u'my_hash', u'key', u'value') self.assertEqual(result, 1) - result = yield from protocol.hset(u'my_hash', u'key2', u'value2') + result = await protocol.hset(u'my_hash', u'key2', u'value2') self.assertEqual(result, 1) # hlen - result = yield from protocol.hlen(u'my_hash') + result = await protocol.hlen(u'my_hash') self.assertEqual(result, 2) # hexists - result = yield from protocol.hexists(u'my_hash', u'key') + result = await protocol.hexists(u'my_hash', u'key') self.assertEqual(result, True) - result = yield from protocol.hexists(u'my_hash', u'unknown_key') + result = await protocol.hexists(u'my_hash', u'unknown_key') self.assertEqual(result, False) # Get from hash - result = yield from protocol.hget(u'my_hash', u'key2') + result = await protocol.hget(u'my_hash', u'key2') self.assertEqual(result, u'value2') - result = yield from protocol.hget(u'my_hash', u'unknown-key') + result = await protocol.hget(u'my_hash', u'unknown-key') self.assertEqual(result, None) - result = yield from protocol.hgetall(u'my_hash') + result = await protocol.hgetall(u'my_hash') self.assertIsInstance(result, DictReply) self.assertEqual(repr(result), u"DictReply(length=2)") - result = yield from result.asdict() + result = await result.asdict() self.assertEqual(result, {u'key': u'value', u'key2': u'value2' }) - result = yield from protocol.hkeys(u'my_hash') + result = await protocol.hkeys(u'my_hash') self.assertIsInstance(result, SetReply) - result = yield from result.asset() + result = await result.asset() self.assertIsInstance(result, set) self.assertEqual(result, {u'key', u'key2' }) - result = yield from protocol.hvals(u'my_hash') + result = await protocol.hvals(u'my_hash') self.assertIsInstance(result, ListReply) - result = yield from result.aslist() + result = await result.aslist() self.assertIsInstance(result, list) self.assertEqual(set(result), {u'value', u'value2' }) # HDel - result = yield from protocol.hdel(u'my_hash', [u'key2']) + result = await protocol.hdel(u'my_hash', [u'key2']) self.assertEqual(result, 1) - result = yield from protocol.hdel(u'my_hash', [u'key2']) + result = await protocol.hdel(u'my_hash', [u'key2']) self.assertEqual(result, 0) - result = yield from protocol.hkeys(u'my_hash') + result = await protocol.hkeys(u'my_hash') self.assertIsInstance(result, SetReply) - result = yield from result.asset() + result = await result.asset() self.assertEqual(result, { u'key' }) @redis_test - def test_keys(self, transport, protocol): + async def test_keys(self, transport, protocol): # Create some keys in this 'namespace' - yield from protocol.set('our-keytest-key1', 'a') - yield from protocol.set('our-keytest-key2', 'a') - yield from protocol.set('our-keytest-key3', 'a') + await protocol.set('our-keytest-key1', 'a') + await protocol.set('our-keytest-key2', 'a') + await protocol.set('our-keytest-key3', 'a') # Test 'keys' - multibulk = yield from protocol.keys(u'our-keytest-key*') - generator = [ (yield from f) for f in multibulk ] - all_keys = yield from generator + multibulk = await protocol.keys(u'our-keytest-key*') + all_keys = await asyncio.gather(*multibulk) self.assertEqual(set(all_keys), { 'our-keytest-key1', 'our-keytest-key2', 'our-keytest-key3' }) @redis_test - def test_hmset_get(self, transport, protocol): - yield from protocol.delete([ u'my_hash' ]) - yield from protocol.hset(u'my_hash', u'a', u'1') + async def test_hmset_get(self, transport, protocol): + await protocol.delete([ u'my_hash' ]) + await protocol.hset(u'my_hash', u'a', u'1') # HMSet - result = yield from protocol.hmset(u'my_hash', { 'b':'2', 'c': '3'}) + result = await protocol.hmset(u'my_hash', { 'b':'2', 'c': '3'}) self.assertEqual(result, StatusReply('OK')) # HMGet - result = yield from protocol.hmget(u'my_hash', [u'a', u'b', u'c']) + result = await protocol.hmget(u'my_hash', [u'a', u'b', u'c']) self.assertIsInstance(result, ListReply) - result = yield from result.aslist() + result = await result.aslist() self.assertEqual(result, [ u'1', u'2', u'3']) - result = yield from protocol.hmget(u'my_hash', [u'c', u'b']) + result = await protocol.hmget(u'my_hash', [u'c', u'b']) self.assertIsInstance(result, ListReply) - result = yield from result.aslist() + result = await result.aslist() self.assertEqual(result, [ u'3', u'2' ]) # Hsetnx - result = yield from protocol.hsetnx(u'my_hash', u'b', '4') + result = await protocol.hsetnx(u'my_hash', u'b', '4') self.assertEqual(result, 0) # Existing key. Not set - result = yield from protocol.hget(u'my_hash', u'b') + result = await protocol.hget(u'my_hash', u'b') self.assertEqual(result, u'2') - result = yield from protocol.hsetnx(u'my_hash', u'd', '5') + result = await protocol.hsetnx(u'my_hash', u'd', '5') self.assertEqual(result, 1) # New key, set - result = yield from protocol.hget(u'my_hash', u'd') + result = await protocol.hget(u'my_hash', u'd') self.assertEqual(result, u'5') @redis_test - def test_hincr(self, transport, protocol): - yield from protocol.delete([ u'my_hash' ]) - yield from protocol.hset(u'my_hash', u'a', u'10') + async def test_hincr(self, transport, protocol): + await protocol.delete([ u'my_hash' ]) + await protocol.hset(u'my_hash', u'a', u'10') # hincrby - result = yield from protocol.hincrby(u'my_hash', u'a', 2) + result = await protocol.hincrby(u'my_hash', u'a', 2) self.assertEqual(result, 12) # hincrbyfloat - result = yield from protocol.hincrbyfloat(u'my_hash', u'a', 3.7) + result = await protocol.hincrbyfloat(u'my_hash', u'a', 3.7) self.assertEqual(result, 15.7) @redis_test - def test_pubsub(self, transport, protocol): - @asyncio.coroutine - def listener(): + async def test_pubsub(self, transport, protocol): + async def listener(): # Subscribe - transport2, protocol2 = yield from connect(self.loop) + transport2, protocol2 = await connect() self.assertEqual(protocol2.in_pubsub, False) - subscription = yield from protocol2.start_subscribe() + subscription = await protocol2.start_subscribe() self.assertIsInstance(subscription, Subscription) self.assertEqual(protocol2.in_pubsub, True) - yield from subscription.subscribe([u'our_channel']) + await subscription.subscribe([u'our_channel']) - value = yield from subscription.next_published() + value = await subscription.next_published() self.assertIsInstance(value, PubSubReply) self.assertEqual(value.channel, u'our_channel') self.assertEqual(value.value, u'message1') - value = yield from subscription.next_published() + value = await subscription.next_published() self.assertIsInstance(value, PubSubReply) self.assertEqual(value.channel, u'our_channel') self.assertEqual(value.value, u'message2') @@ -836,29 +821,28 @@ def listener(): f = asyncio.ensure_future(listener(), loop=self.loop) - @asyncio.coroutine - def sender(): - value = yield from protocol.publish(u'our_channel', 'message1') + async def sender(): + value = await protocol.publish(u'our_channel', 'message1') self.assertGreaterEqual(value, 1) # Nr of clients that received the message - value = yield from protocol.publish(u'our_channel', 'message2') + value = await protocol.publish(u'our_channel', 'message2') self.assertGreaterEqual(value, 1) # Test pubsub_channels - result = yield from protocol.pubsub_channels() + result = await protocol.pubsub_channels() self.assertIsInstance(result, ListReply) - result = yield from result.aslist() + result = await result.aslist() self.assertIn(u'our_channel', result) - result = yield from protocol.pubsub_channels_aslist(u'our_c*') + result = await protocol.pubsub_channels_aslist(u'our_c*') self.assertIn(u'our_channel', result) - result = yield from protocol.pubsub_channels_aslist(u'unknown-channel-prefix*') + result = await protocol.pubsub_channels_aslist(u'unknown-channel-prefix*') self.assertEqual(result, []) # Test pubsub numsub. - result = yield from protocol.pubsub_numsub([ u'our_channel', u'some_unknown_channel' ]) + result = await protocol.pubsub_numsub([ u'our_channel', u'some_unknown_channel' ]) self.assertIsInstance(result, DictReply) - result = yield from result.asdict() + result = await result.asdict() self.assertEqual(len(result), 2) self.assertGreater(int(result['our_channel']), 0) # XXX: the cast to int is required, because the redis @@ -868,30 +852,29 @@ def sender(): self.assertEqual(int(result['some_unknown_channel']), 0) # Test pubsub numpat - result = yield from protocol.pubsub_numpat() + result = await protocol.pubsub_numpat() self.assertIsInstance(result, int) - yield from asyncio.sleep(.5, loop=self.loop) - yield from sender() - transport2 = yield from f + await asyncio.sleep(.5, loop=self.loop) + await sender() + transport2 = await f transport2.close() @redis_test - def test_pubsub_many(self, transport, protocol): + async def test_pubsub_many(self, transport, protocol): """ Create a listener that listens to several channels. """ - @asyncio.coroutine - def listener(): + async def listener(): # Subscribe - transport2, protocol2 = yield from connect(self.loop) + transport2, protocol2 = await connect() self.assertEqual(protocol2.in_pubsub, False) - subscription = yield from protocol2.start_subscribe() - yield from subscription.subscribe(['channel1', 'channel2']) - yield from subscription.subscribe(['channel3', 'channel4']) + subscription = await protocol2.start_subscribe() + await subscription.subscribe(['channel1', 'channel2']) + await subscription.subscribe(['channel3', 'channel4']) results = [] for i in range(4): - results.append((yield from subscription.next_published())) + results.append((await subscription.next_published())) self.assertEqual(results, [ PubSubReply('channel1', 'message1'), @@ -902,38 +885,36 @@ def listener(): transport2.close() - f = asyncio.ensure_future(listener(), loop=self.loop) + f = asyncio.ensure_future(listener()) - @asyncio.coroutine - def sender(): + async def sender(): # Should not be received - yield from protocol.publish('channel5', 'message5') + await protocol.publish('channel5', 'message5') # These for should be received. - yield from protocol.publish('channel1', 'message1') - yield from protocol.publish('channel2', 'message2') - yield from protocol.publish('channel3', 'message3') - yield from protocol.publish('channel4', 'message4') + await protocol.publish('channel1', 'message1') + await protocol.publish('channel2', 'message2') + await protocol.publish('channel3', 'message3') + await protocol.publish('channel4', 'message4') - yield from asyncio.sleep(.5, loop=self.loop) - yield from sender() - yield from f + await asyncio.sleep(.5) + await sender() + await f @redis_test - def test_pubsub_patterns(self, transport, protocol): + async def test_pubsub_patterns(self, transport, protocol): """ Test a pubsub connection that subscribes to a pattern. """ - @asyncio.coroutine - def listener(): + async def listener(): # Subscribe to two patterns - transport2, protocol2 = yield from connect(self.loop) + transport2, protocol2 = await connect() - subscription = yield from protocol2.start_subscribe() - yield from subscription.psubscribe(['h*llo', 'w?rld']) + subscription = await protocol2.start_subscribe() + await subscription.psubscribe(['h*llo', 'w?rld']) # Receive messages results = [] for i in range(4): - results.append((yield from subscription.next_published())) + results.append((await subscription.next_published())) self.assertEqual(results, [ PubSubReply('hello', 'message1', pattern='h*llo'), @@ -944,49 +925,48 @@ def listener(): transport2.close() - f = asyncio.ensure_future(listener(), loop=self.loop) + f = asyncio.ensure_future(listener()) - @asyncio.coroutine - def sender(): + async def sender(): # Should not be received - yield from protocol.publish('other-channel', 'message5') + await protocol.publish('other-channel', 'message5') # These for should be received. - yield from protocol.publish('hello', 'message1') - yield from protocol.publish('heello', 'message2') - yield from protocol.publish('world', 'message3') - yield from protocol.publish('wArld', 'message4') + await protocol.publish('hello', 'message1') + await protocol.publish('heello', 'message2') + await protocol.publish('world', 'message3') + await protocol.publish('wArld', 'message4') - yield from asyncio.sleep(.5, loop=self.loop) - yield from sender() - yield from f + await asyncio.sleep(.5) + await sender() + await f @redis_test - def test_incr(self, transport, protocol): - yield from protocol.set(u'key1', u'3') + async def test_incr(self, transport, protocol): + await protocol.set(u'key1', u'3') # Incr - result = yield from protocol.incr(u'key1') + result = await protocol.incr(u'key1') self.assertEqual(result, 4) - result = yield from protocol.incr(u'key1') + result = await protocol.incr(u'key1') self.assertEqual(result, 5) # Incrby - result = yield from protocol.incrby(u'key1', 10) + result = await protocol.incrby(u'key1', 10) self.assertEqual(result, 15) # Decr - result = yield from protocol.decr(u'key1') + result = await protocol.decr(u'key1') self.assertEqual(result, 14) # Decrby - result = yield from protocol.decrby(u'key1', 4) + result = await protocol.decrby(u'key1', 4) self.assertEqual(result, 10) @redis_test - def test_bitops(self, transport, protocol): - yield from protocol.set('a', 'fff') - yield from protocol.set('b', '555') + async def test_bitops(self, transport, protocol): + await protocol.set('a', 'fff') + await protocol.set('b', '555') a = b'f'[0] b = b'5'[0] @@ -995,404 +975,405 @@ def test_bitops(self, transport, protocol): set_bits = len([ c for c in bin(a) if c == '1' ]) # Bitcount - result = yield from protocol.bitcount('a') + result = await protocol.bitcount('a') self.assertEqual(result, set_bits * 3) # And - result = yield from protocol.bitop_and('result', ['a', 'b']) + result = await protocol.bitop_and('result', ['a', 'b']) self.assertEqual(result, 3) - result = yield from protocol.get('result') + result = await protocol.get('result') self.assertEqual(result, chr(a & b) * 3) # Or - result = yield from protocol.bitop_or('result', ['a', 'b']) + result = await protocol.bitop_or('result', ['a', 'b']) self.assertEqual(result, 3) - result = yield from protocol.get('result') + result = await protocol.get('result') self.assertEqual(result, chr(a | b) * 3) # Xor - result = yield from protocol.bitop_xor('result', ['a', 'b']) + result = await protocol.bitop_xor('result', ['a', 'b']) self.assertEqual(result, 3) - result = yield from protocol.get('result') + result = await protocol.get('result') self.assertEqual(result, chr(a ^ b) * 3) # Not - result = yield from protocol.bitop_not('result', 'a') + result = await protocol.bitop_not('result', 'a') self.assertEqual(result, 3) # Check result using bytes protocol - bytes_transport, bytes_protocol = yield from connect(self.loop, lambda **kw: RedisProtocol(encoder=BytesEncoder(), **kw)) - result = yield from bytes_protocol.get(b'result') + bytes_transport, bytes_protocol = await connect( + lambda **kw: RedisProtocol(encoder=BytesEncoder(), **kw) + ) + result = await bytes_protocol.get(b'result') self.assertIsInstance(result, bytes) self.assertEqual(result, bytes((~a % 256, ~a % 256, ~a % 256))) bytes_transport.close() @redis_test - def test_setbit(self, transport, protocol): - yield from protocol.set('a', 'fff') + async def test_setbit(self, transport, protocol): + await protocol.set('a', 'fff') - value = yield from protocol.getbit('a', 3) + value = await protocol.getbit('a', 3) self.assertIsInstance(value, bool) self.assertEqual(value, False) - value = yield from protocol.setbit('a', 3, True) + value = await protocol.setbit('a', 3, True) self.assertIsInstance(value, bool) self.assertEqual(value, False) # Set returns the old value. - value = yield from protocol.getbit('a', 3) + value = await protocol.getbit('a', 3) self.assertIsInstance(value, bool) self.assertEqual(value, True) @redis_test - def test_zscore(self, transport, protocol): - yield from protocol.delete([ 'myzset' ]) + async def test_zscore(self, transport, protocol): + await protocol.delete([ 'myzset' ]) # Test zscore return value for NIL server response - value = yield from protocol.zscore('myzset', 'key') + value = await protocol.zscore('myzset', 'key') self.assertIsNone(value) # zadd key 4.0 - result = yield from protocol.zadd('myzset', { 'key': 4}) + result = await protocol.zadd('myzset', { 'key': 4}) self.assertEqual(result, 1) # Test zscore value for existing zset members - value = yield from protocol.zscore('myzset', 'key') + value = await protocol.zscore('myzset', 'key') self.assertEqual(value, 4.0) @redis_test - def test_zset(self, transport, protocol): - yield from protocol.delete([ 'myzset' ]) + async def test_zset(self, transport, protocol): + await protocol.delete([ 'myzset' ]) # Test zadd - result = yield from protocol.zadd('myzset', { 'key': 4, 'key2': 5, 'key3': 5.5 }) + result = await protocol.zadd('myzset', { 'key': 4, 'key2': 5, 'key3': 5.5 }) self.assertEqual(result, 3) # Test zcard - result = yield from protocol.zcard('myzset') + result = await protocol.zcard('myzset') self.assertEqual(result, 3) # Test zrank - result = yield from protocol.zrank('myzset', 'key') + result = await protocol.zrank('myzset', 'key') self.assertEqual(result, 0) - result = yield from protocol.zrank('myzset', 'key3') + result = await protocol.zrank('myzset', 'key3') self.assertEqual(result, 2) - result = yield from protocol.zrank('myzset', 'unknown-key') + result = await protocol.zrank('myzset', 'unknown-key') self.assertEqual(result, None) # Test revrank - result = yield from protocol.zrevrank('myzset', 'key') + result = await protocol.zrevrank('myzset', 'key') self.assertEqual(result, 2) - result = yield from protocol.zrevrank('myzset', 'key3') + result = await protocol.zrevrank('myzset', 'key3') self.assertEqual(result, 0) - result = yield from protocol.zrevrank('myzset', 'unknown-key') + result = await protocol.zrevrank('myzset', 'unknown-key') self.assertEqual(result, None) # Test zrange - result = yield from protocol.zrange('myzset') + result = await protocol.zrange('myzset') self.assertIsInstance(result, ZRangeReply) self.assertEqual(repr(result), u"ZRangeReply(length=3)") - self.assertEqual((yield from result.asdict()), + self.assertEqual((await result.asdict()), { 'key': 4.0, 'key2': 5.0, 'key3': 5.5 }) - result = yield from protocol.zrange('myzset') + result = await protocol.zrange('myzset') self.assertIsInstance(result, ZRangeReply) etalon = [ ('key', 4.0), ('key2', 5.0), ('key3', 5.5) ] for i, f in enumerate(result): # Ordering matter - d = yield from f + d = await f self.assertEqual(d, etalon[i]) # Test zrange_asdict - result = yield from protocol.zrange_asdict('myzset') + result = await protocol.zrange_asdict('myzset') self.assertEqual(result, { 'key': 4.0, 'key2': 5.0, 'key3': 5.5 }) # Test zrange with negative indexes - result = yield from protocol.zrange('myzset', -2, -1) - self.assertEqual((yield from result.asdict()), + result = await protocol.zrange('myzset', -2, -1) + self.assertEqual((await result.asdict()), {'key2': 5.0, 'key3': 5.5 }) - result = yield from protocol.zrange('myzset', -2, -1) + result = await protocol.zrange('myzset', -2, -1) self.assertIsInstance(result, ZRangeReply) for f in result: - d = yield from f + d = await f self.assertIn(d, [ ('key2', 5.0), ('key3', 5.5) ]) # Test zrangebyscore - result = yield from protocol.zrangebyscore('myzset') - self.assertEqual((yield from result.asdict()), + result = await protocol.zrangebyscore('myzset') + self.assertEqual((await result.asdict()), { 'key': 4.0, 'key2': 5.0, 'key3': 5.5 }) - result = yield from protocol.zrangebyscore('myzset', min=ZScoreBoundary(4.5)) - self.assertEqual((yield from result.asdict()), + result = await protocol.zrangebyscore('myzset', min=ZScoreBoundary(4.5)) + self.assertEqual((await result.asdict()), { 'key2': 5.0, 'key3': 5.5 }) - result = yield from protocol.zrangebyscore('myzset', max=ZScoreBoundary(5.5)) - self.assertEqual((yield from result.asdict()), + result = await protocol.zrangebyscore('myzset', max=ZScoreBoundary(5.5)) + self.assertEqual((await result.asdict()), { 'key': 4.0, 'key2': 5.0, 'key3': 5.5 }) - result = yield from protocol.zrangebyscore('myzset', + result = await protocol.zrangebyscore('myzset', max=ZScoreBoundary(5.5, exclude_boundary=True)) - self.assertEqual((yield from result.asdict()), + self.assertEqual((await result.asdict()), { 'key': 4.0, 'key2': 5.0 }) - result = yield from protocol.zrangebyscore('myzset', limit=1) - self.assertEqual((yield from result.asdict()), + result = await protocol.zrangebyscore('myzset', limit=1) + self.assertEqual((await result.asdict()), { 'key': 4.0 }) - result = yield from protocol.zrangebyscore('myzset', offset=1) - self.assertEqual((yield from result.asdict()), + result = await protocol.zrangebyscore('myzset', offset=1) + self.assertEqual((await result.asdict()), { 'key2': 5.0, 'key3': 5.5 }) # Test zrevrangebyscore (identical to zrangebyscore, unless we call aslist) - result = yield from protocol.zrevrangebyscore('myzset') + result = await protocol.zrevrangebyscore('myzset') self.assertIsInstance(result, DictReply) - self.assertEqual((yield from result.asdict()), + self.assertEqual((await result.asdict()), { 'key': 4.0, 'key2': 5.0, 'key3': 5.5 }) - self.assertEqual((yield from protocol.zrevrangebyscore_asdict('myzset')), + self.assertEqual((await protocol.zrevrangebyscore_asdict('myzset')), { 'key': 4.0, 'key2': 5.0, 'key3': 5.5 }) - result = yield from protocol.zrevrangebyscore('myzset', min=ZScoreBoundary(4.5)) - self.assertEqual((yield from result.asdict()), + result = await protocol.zrevrangebyscore('myzset', min=ZScoreBoundary(4.5)) + self.assertEqual((await result.asdict()), { 'key2': 5.0, 'key3': 5.5 }) - result = yield from protocol.zrevrangebyscore('myzset', max=ZScoreBoundary(5.5)) + result = await protocol.zrevrangebyscore('myzset', max=ZScoreBoundary(5.5)) self.assertIsInstance(result, DictReply) - self.assertEqual((yield from result.asdict()), + self.assertEqual((await result.asdict()), { 'key': 4.0, 'key2': 5.0, 'key3': 5.5 }) - result = yield from protocol.zrevrangebyscore('myzset', + result = await protocol.zrevrangebyscore('myzset', max=ZScoreBoundary(5.5, exclude_boundary=True)) - self.assertEqual((yield from result.asdict()), + self.assertEqual((await result.asdict()), { 'key': 4.0, 'key2': 5.0 }) - result = yield from protocol.zrevrangebyscore('myzset', limit=1) - self.assertEqual((yield from result.asdict()), + result = await protocol.zrevrangebyscore('myzset', limit=1) + self.assertEqual((await result.asdict()), { 'key3': 5.5 }) - result = yield from protocol.zrevrangebyscore('myzset', offset=1) - self.assertEqual((yield from result.asdict()), + result = await protocol.zrevrangebyscore('myzset', offset=1) + self.assertEqual((await result.asdict()), { 'key': 4.0, 'key2': 5.0 }) @redis_test - def test_zrevrange(self, transport, protocol): - yield from protocol.delete([ 'myzset' ]) + async def test_zrevrange(self, transport, protocol): + await protocol.delete([ 'myzset' ]) # Test zadd - result = yield from protocol.zadd('myzset', { 'key': 4, 'key2': 5, 'key3': 5.5 }) + result = await protocol.zadd('myzset', { 'key': 4, 'key2': 5, 'key3': 5.5 }) self.assertEqual(result, 3) # Test zrevrange - result = yield from protocol.zrevrange('myzset') + result = await protocol.zrevrange('myzset') self.assertIsInstance(result, ZRangeReply) self.assertEqual(repr(result), u"ZRangeReply(length=3)") - self.assertEqual((yield from result.asdict()), + self.assertEqual((await result.asdict()), { 'key': 4.0, 'key2': 5.0, 'key3': 5.5 }) - self.assertEqual((yield from protocol.zrevrange_asdict('myzset')), + self.assertEqual((await protocol.zrevrange_asdict('myzset')), { 'key': 4.0, 'key2': 5.0, 'key3': 5.5 }) - result = yield from protocol.zrevrange('myzset') + result = await protocol.zrevrange('myzset') self.assertIsInstance(result, ZRangeReply) etalon = [ ('key3', 5.5), ('key2', 5.0), ('key', 4.0) ] for i, f in enumerate(result): # Ordering matter - d = yield from f + d = await f self.assertEqual(d, etalon[i]) @redis_test - def test_zset_zincrby(self, transport, protocol): - yield from protocol.delete([ 'myzset' ]) - yield from protocol.zadd('myzset', { 'key': 4, 'key2': 5, 'key3': 5.5 }) + async def test_zset_zincrby(self, transport, protocol): + await protocol.delete([ 'myzset' ]) + await protocol.zadd('myzset', { 'key': 4, 'key2': 5, 'key3': 5.5 }) # Test zincrby - result = yield from protocol.zincrby('myzset', 1.1, 'key') + result = await protocol.zincrby('myzset', 1.1, 'key') self.assertEqual(result, 5.1) - result = yield from protocol.zrange('myzset') - self.assertEqual((yield from result.asdict()), + result = await protocol.zrange('myzset') + self.assertEqual((await result.asdict()), { 'key': 5.1, 'key2': 5.0, 'key3': 5.5 }) @redis_test - def test_zset_zrem(self, transport, protocol): - yield from protocol.delete([ 'myzset' ]) - yield from protocol.zadd('myzset', { 'key': 4, 'key2': 5, 'key3': 5.5 }) + async def test_zset_zrem(self, transport, protocol): + await protocol.delete([ 'myzset' ]) + await protocol.zadd('myzset', { 'key': 4, 'key2': 5, 'key3': 5.5 }) # Test zrem - result = yield from protocol.zrem('myzset', ['key']) + result = await protocol.zrem('myzset', ['key']) self.assertEqual(result, 1) - result = yield from protocol.zrem('myzset', ['key']) + result = await protocol.zrem('myzset', ['key']) self.assertEqual(result, 0) - result = yield from protocol.zrange('myzset') - self.assertEqual((yield from result.asdict()), + result = await protocol.zrange('myzset') + self.assertEqual((await result.asdict()), { 'key2': 5.0, 'key3': 5.5 }) @redis_test - def test_zset_zrembyscore(self, transport, protocol): + async def test_zset_zrembyscore(self, transport, protocol): # Test zremrangebyscore (1) - yield from protocol.delete([ 'myzset' ]) - yield from protocol.zadd('myzset', { 'key': 4, 'key2': 5, 'key3': 5.5 }) + await protocol.delete([ 'myzset' ]) + await protocol.zadd('myzset', { 'key': 4, 'key2': 5, 'key3': 5.5 }) - result = yield from protocol.zremrangebyscore('myzset', min=ZScoreBoundary(5.0)) + result = await protocol.zremrangebyscore('myzset', min=ZScoreBoundary(5.0)) self.assertEqual(result, 2) - result = yield from protocol.zrange('myzset') - self.assertEqual((yield from result.asdict()), { 'key': 4.0 }) + result = await protocol.zrange('myzset') + self.assertEqual((await result.asdict()), { 'key': 4.0 }) # Test zremrangebyscore (2) - yield from protocol.delete([ 'myzset' ]) - yield from protocol.zadd('myzset', { 'key': 4, 'key2': 5, 'key3': 5.5 }) + await protocol.delete([ 'myzset' ]) + await protocol.zadd('myzset', { 'key': 4, 'key2': 5, 'key3': 5.5 }) - result = yield from protocol.zremrangebyscore('myzset', max=ZScoreBoundary(5.0)) + result = await protocol.zremrangebyscore('myzset', max=ZScoreBoundary(5.0)) self.assertEqual(result, 2) - result = yield from protocol.zrange('myzset') - self.assertEqual((yield from result.asdict()), { 'key3': 5.5 }) + result = await protocol.zrange('myzset') + self.assertEqual((await result.asdict()), { 'key3': 5.5 }) @redis_test - def test_zset_zremrangebyrank(self, transport, protocol): - @asyncio.coroutine - def setup(): - yield from protocol.delete([ 'myzset' ]) - yield from protocol.zadd('myzset', { 'key': 4, 'key2': 5, 'key3': 5.5 }) + async def test_zset_zremrangebyrank(self, transport, protocol): + async def setup(): + await protocol.delete([ 'myzset' ]) + await protocol.zadd('myzset', { 'key': 4, 'key2': 5, 'key3': 5.5 }) # Test zremrangebyrank (1) - yield from setup() - result = yield from protocol.zremrangebyrank('myzset') + await setup() + result = await protocol.zremrangebyrank('myzset') self.assertEqual(result, 3) - result = yield from protocol.zrange('myzset') - self.assertEqual((yield from result.asdict()), { }) + result = await protocol.zrange('myzset') + self.assertEqual((await result.asdict()), { }) # Test zremrangebyrank (2) - yield from setup() - result = yield from protocol.zremrangebyrank('myzset', min=2) + await setup() + result = await protocol.zremrangebyrank('myzset', min=2) self.assertEqual(result, 1) - result = yield from protocol.zrange('myzset') - self.assertEqual((yield from result.asdict()), { 'key': 4.0, 'key2': 5.0 }) + result = await protocol.zrange('myzset') + self.assertEqual((await result.asdict()), { 'key': 4.0, 'key2': 5.0 }) # Test zremrangebyrank (3) - yield from setup() - result = yield from protocol.zremrangebyrank('myzset', max=1) + await setup() + result = await protocol.zremrangebyrank('myzset', max=1) self.assertEqual(result, 2) - result = yield from protocol.zrange('myzset') - self.assertEqual((yield from result.asdict()), { 'key3': 5.5 }) + result = await protocol.zrange('myzset') + self.assertEqual((await result.asdict()), { 'key3': 5.5 }) @redis_test - def test_zunionstore(self, transport, protocol): - yield from protocol.delete([ 'set_a', 'set_b' ]) - yield from protocol.zadd('set_a', { 'key': 4, 'key2': 5, 'key3': 5.5 }) - yield from protocol.zadd('set_b', { 'key': -1, 'key2': 1.1, 'key4': 9 }) + async def test_zunionstore(self, transport, protocol): + await protocol.delete([ 'set_a', 'set_b' ]) + await protocol.zadd('set_a', { 'key': 4, 'key2': 5, 'key3': 5.5 }) + await protocol.zadd('set_b', { 'key': -1, 'key2': 1.1, 'key4': 9 }) # Call zunionstore - result = yield from protocol.zunionstore('union_key', [ 'set_a', 'set_b' ]) + result = await protocol.zunionstore('union_key', [ 'set_a', 'set_b' ]) self.assertEqual(result, 4) - result = yield from protocol.zrange('union_key') - result = yield from result.asdict() + result = await protocol.zrange('union_key') + result = await result.asdict() self.assertEqual(result, { 'key': 3.0, 'key2': 6.1, 'key3': 5.5, 'key4': 9.0 }) # Call zunionstore with weights. - result = yield from protocol.zunionstore('union_key', [ 'set_a', 'set_b' ], [1, 1.5]) + result = await protocol.zunionstore('union_key', [ 'set_a', 'set_b' ], [1, 1.5]) self.assertEqual(result, 4) - result = yield from protocol.zrange('union_key') - result = yield from result.asdict() + result = await protocol.zrange('union_key') + result = await result.asdict() self.assertEqual(result, { 'key': 2.5, 'key2': 6.65, 'key3': 5.5, 'key4': 13.5 }) @redis_test - def test_zinterstore(self, transport, protocol): - yield from protocol.delete([ 'set_a', 'set_b' ]) - yield from protocol.zadd('set_a', { 'key': 4, 'key2': 5, 'key3': 5.5 }) - yield from protocol.zadd('set_b', { 'key': -1, 'key2': 1.5, 'key4': 9 }) + async def test_zinterstore(self, transport, protocol): + await protocol.delete([ 'set_a', 'set_b' ]) + await protocol.zadd('set_a', { 'key': 4, 'key2': 5, 'key3': 5.5 }) + await protocol.zadd('set_b', { 'key': -1, 'key2': 1.5, 'key4': 9 }) # Call zinterstore - result = yield from protocol.zinterstore('inter_key', [ 'set_a', 'set_b' ]) + result = await protocol.zinterstore('inter_key', [ 'set_a', 'set_b' ]) self.assertEqual(result, 2) - result = yield from protocol.zrange('inter_key') - result = yield from result.asdict() + result = await protocol.zrange('inter_key') + result = await result.asdict() self.assertEqual(result, { 'key': 3.0, 'key2': 6.5 }) # Call zinterstore with weights. - result = yield from protocol.zinterstore('inter_key', [ 'set_a', 'set_b' ], [1, 1.5]) + result = await protocol.zinterstore('inter_key', [ 'set_a', 'set_b' ], [1, 1.5]) self.assertEqual(result, 2) - result = yield from protocol.zrange('inter_key') - result = yield from result.asdict() + result = await protocol.zrange('inter_key') + result = await result.asdict() self.assertEqual(result, { 'key': 2.5, 'key2': 7.25, }) @redis_test - def test_randomkey(self, transport, protocol): - yield from protocol.set(u'key1', u'value') - result = yield from protocol.randomkey() + async def test_randomkey(self, transport, protocol): + await protocol.set(u'key1', u'value') + result = await protocol.randomkey() self.assertIsInstance(result, str) @redis_test - def test_dbsize(self, transport, protocol): - result = yield from protocol.dbsize() + async def test_dbsize(self, transport, protocol): + result = await protocol.dbsize() self.assertIsInstance(result, int) @redis_test - def test_client_names(self, transport, protocol): + async def test_client_names(self, transport, protocol): # client_setname - result = yield from protocol.client_setname(u'my-connection-name') + result = await protocol.client_setname(u'my-connection-name') self.assertEqual(result, StatusReply('OK')) # client_getname - result = yield from protocol.client_getname() + result = await protocol.client_getname() self.assertEqual(result, u'my-connection-name') # client list - result = yield from protocol.client_list() + result = await protocol.client_list() self.assertIsInstance(result, ClientListReply) @redis_test - def test_lua_script(self, transport, protocol): + async def test_lua_script(self, transport, protocol): code = """ local value = redis.call('GET', KEYS[1]) value = tonumber(value) return value * ARGV[1] """ - yield from protocol.set('foo', '2') + await protocol.set('foo', '2') # Register script - script = yield from protocol.register_script(code) + script = await protocol.register_script(code) self.assertIsInstance(script, Script) # Call script. - result = yield from script.run(keys=['foo'], args=['5']) + result = await script.run(keys=['foo'], args=['5']) self.assertIsInstance(result, EvalScriptReply) - result = yield from result.return_value() + result = await result.return_value() self.assertEqual(result, 10) # Test evalsha directly - result = yield from protocol.evalsha(script.sha, keys=['foo'], args=['5']) + result = await protocol.evalsha(script.sha, keys=['foo'], args=['5']) self.assertIsInstance(result, EvalScriptReply) - result = yield from result.return_value() + result = await result.return_value() self.assertEqual(result, 10) # Test script exists - result = yield from protocol.script_exists([ script.sha, script.sha, 'unknown-script' ]) + result = await protocol.script_exists([ script.sha, script.sha, 'unknown-script' ]) self.assertEqual(result, [ True, True, False ]) # Test script flush - result = yield from protocol.script_flush() + result = await protocol.script_flush() self.assertEqual(result, StatusReply('OK')) - result = yield from protocol.script_exists([ script.sha, script.sha, 'unknown-script' ]) + result = await protocol.script_exists([ script.sha, script.sha, 'unknown-script' ]) self.assertEqual(result, [ False, False, False ]) # Test another script where evalsha returns a string. code2 = """ return "text" """ - script2 = yield from protocol.register_script(code2) - result = yield from protocol.evalsha(script2.sha) + script2 = await protocol.register_script(code2) + result = await protocol.evalsha(script2.sha) self.assertIsInstance(result, EvalScriptReply) - result = yield from result.return_value() + result = await result.return_value() self.assertIsInstance(result, str) self.assertEqual(result, u'text') @redis_test - def test_script_return_types(self, transport, protocol): + async def test_script_return_types(self, transport, protocol): # Test whether LUA scripts are returning correct return values. script_and_return_values = { 'return "string" ': "string", # str @@ -1405,92 +1386,91 @@ def test_script_return_types(self, transport, protocol): } for code, return_value in script_and_return_values.items(): # Register script - script = yield from protocol.register_script(code) + script = await protocol.register_script(code) # Call script. - scriptreply = yield from script.run() - result = yield from scriptreply.return_value() + scriptreply = await script.run() + result = await scriptreply.return_value() self.assertEqual(result, return_value) @redis_test - def test_script_kill(self, transport, protocol): + async def test_script_kill(self, transport, protocol): # Test script kill (when nothing is running.) with self.assertRaises(NoRunningScriptError): - result = yield from protocol.script_kill() + result = await protocol.script_kill() # Test script kill (when a while/true is running.) - @asyncio.coroutine - def run_while_true(): + async def run_while_true(): code = """ local i = 0 while true do i = i + 1 end """ - transport, protocol = yield from connect(self.loop, RedisProtocol) + transport, protocol = await connect(RedisProtocol) - script = yield from protocol.register_script(code) + script = await protocol.register_script(code) with self.assertRaises(ScriptKilledError): - yield from script.run() + await script.run() transport.close() # (start script) f = asyncio.ensure_future(run_while_true(), loop=self.loop) - yield from asyncio.sleep(.5, loop=self.loop) + await asyncio.sleep(.5, loop=self.loop) - result = yield from protocol.script_kill() + result = await protocol.script_kill() self.assertEqual(result, StatusReply('OK')) # Wait for the other coroutine to finish. - yield from f + await f @redis_test - def test_transaction(self, transport, protocol): + async def test_transaction(self, transport, protocol): # Prepare - yield from protocol.set(u'my_key', u'a') - yield from protocol.set(u'my_key2', u'b') - yield from protocol.set(u'my_key3', u'c') - yield from protocol.delete([ u'my_hash' ]) - yield from protocol.hmset(u'my_hash', {'a':'1', 'b':'2', 'c':'3'}) + await protocol.set(u'my_key', u'a') + await protocol.set(u'my_key2', u'b') + await protocol.set(u'my_key3', u'c') + await protocol.delete([ u'my_hash' ]) + await protocol.hmset(u'my_hash', {'a':'1', 'b':'2', 'c':'3'}) # Start transaction self.assertEqual(protocol.in_transaction, False) - transaction = yield from protocol.multi() + transaction = await protocol.multi() self.assertIsInstance(transaction, Transaction) self.assertEqual(protocol.in_transaction, True) # Run commands - f1 = yield from transaction.get('my_key') - f2 = yield from transaction.mget(['my_key', 'my_key2']) - f3 = yield from transaction.get('my_key3') - f4 = yield from transaction.mget(['my_key2', 'my_key3']) - f5 = yield from transaction.hgetall('my_hash') + f1 = await transaction.get('my_key') + f2 = await transaction.mget(['my_key', 'my_key2']) + f3 = await transaction.get('my_key3') + f4 = await transaction.mget(['my_key2', 'my_key3']) + f5 = await transaction.hgetall('my_hash') for f in [ f1, f2, f3, f4, f5]: self.assertIsInstance(f, asyncio.Future) # Calling subscribe inside transaction should fail. with self.assertRaises(Error) as e: - yield from transaction.start_subscribe() + await transaction.start_subscribe() self.assertEqual(e.exception.args[0], 'Cannot start pubsub listener when a protocol is in use.') # Complete transaction - result = yield from transaction.exec() + result = await transaction.exec() self.assertEqual(result, None) self.assertEqual(protocol.in_transaction, False) # Read futures - r1 = yield from f1 - r3 = yield from f3 # 2 & 3 switched by purpose. (order shouldn't matter.) - r2 = yield from f2 - r4 = yield from f4 - r5 = yield from f5 + r1 = await f1 + r3 = await f3 # 2 & 3 switched by purpose. (order shouldn't matter.) + r2 = await f2 + r4 = await f4 + r5 = await f5 - r2 = yield from r2.aslist() - r4 = yield from r4.aslist() - r5 = yield from r5.asdict() + r2 = await r2.aslist() + r4 = await r4.aslist() + r5 = await r5.asdict() self.assertEqual(r1, u'a') self.assertEqual(r2, [u'a', u'b']) @@ -1499,145 +1479,147 @@ def test_transaction(self, transport, protocol): self.assertEqual(r5, { 'a': '1', 'b': '2', 'c': '3' }) @redis_test - def test_run_command_outside_transaction(self, transport, protocol): + async def test_run_command_outside_transaction(self, transport, protocol): # Start transaction. - transaction = yield from protocol.multi() + transaction = await protocol.multi() # Run command, but not as part of the transaction. # This should wait until the transaction finishes. f = asyncio.ensure_future(protocol.set('a', 'b'), loop=self.loop) # Close transaction. - yield from transaction.exec() + await transaction.exec() - result = yield from f + result = await f self.assertIsInstance(result, StatusReply) @redis_test - def test_discard_transaction(self, transport, protocol): - yield from protocol.set(u'my_key', u'a') + async def test_discard_transaction(self, transport, protocol): + await protocol.set(u'my_key', u'a') - transaction = yield from protocol.multi() - yield from transaction.set(u'my_key', 'b') + transaction = await protocol.multi() + await transaction.set(u'my_key', 'b') # Discard - result = yield from transaction.discard() + result = await transaction.discard() self.assertEqual(result, None) - result = yield from protocol.get(u'my_key') + result = await protocol.get(u'my_key') self.assertEqual(result, u'a') # Calling anything on the transaction after discard should fail. with self.assertRaises(Error) as e: - result = yield from transaction.get(u'my_key') + result = await transaction.get(u'my_key') self.assertEqual(e.exception.args[0], 'Transaction already finished or invalid.') @redis_test - def test_nesting_transactions(self, transport, protocol): + async def test_nesting_transactions(self, transport, protocol): # That should fail. - transaction = yield from protocol.multi() + transaction = await protocol.multi() with self.assertRaises(Error) as e: - transaction = yield from transaction.multi() + transaction = await transaction.multi() self.assertEqual(e.exception.args[0], 'Multi calls can not be nested.') @redis_test - def test_password(self, transport, protocol): + async def test_password(self, transport, protocol): # Set password - result = yield from protocol.config_set('requirepass', 'newpassword') + result = await protocol.config_set('requirepass', 'newpassword') self.assertIsInstance(result, StatusReply) # Further redis queries should fail without re-authenticating. with self.assertRaises(ErrorReply) as e: - yield from protocol.set('my-key', 'value') + await protocol.set('my-key', 'value') self.assertEqual(e.exception.args[0], 'NOAUTH Authentication required.') # Reconnect: - result = yield from protocol.auth('newpassword') + result = await protocol.auth('newpassword') self.assertIsInstance(result, StatusReply) # Redis queries should work again. - result = yield from protocol.set('my-key', 'value') + result = await protocol.set('my-key', 'value') self.assertIsInstance(result, StatusReply) # Try connecting through new Protocol instance. - transport2, protocol2 = yield from connect(self.loop, lambda **kw: RedisProtocol(password='newpassword', **kw)) - result = yield from protocol2.set('my-key', 'value') + transport2, protocol2 = await connect( + lambda **kw: RedisProtocol(password='newpassword', **kw) + ) + result = await protocol2.set('my-key', 'value') self.assertIsInstance(result, StatusReply) transport2.close() # Reset password - result = yield from protocol.config_set('requirepass', '') + result = await protocol.config_set('requirepass', '') self.assertIsInstance(result, StatusReply) @redis_test - def test_condfig(self, transport, protocol): + async def test_config(self, transport, protocol): # Config get - result = yield from protocol.config_get('loglevel') + result = await protocol.config_get('loglevel') self.assertIsInstance(result, ConfigPairReply) self.assertEqual(result.parameter, 'loglevel') self.assertIsInstance(result.value, str) # Config set - result = yield from protocol.config_set('loglevel', result.value) + result = await protocol.config_set('loglevel', result.value) self.assertIsInstance(result, StatusReply) # Resetstat - result = yield from protocol.config_resetstat() + result = await protocol.config_resetstat() self.assertIsInstance(result, StatusReply) # XXX: config_rewrite not tested. @redis_test - def test_info(self, transport, protocol): - result = yield from protocol.info() + async def test_info(self, transport, protocol): + result = await protocol.info() self.assertIsInstance(result, InfoReply) # TODO: implement and test InfoReply class - result = yield from protocol.info('CPU') + result = await protocol.info('CPU') self.assertIsInstance(result, InfoReply) @redis_test - def test_scan(self, transport, protocol): + async def test_scan(self, transport, protocol): # Run scan command - cursor = yield from protocol.scan(match='*') + cursor = await protocol.scan(match='*') self.assertIsInstance(cursor, Cursor) # Walk through cursor received = [] while True: - i = yield from cursor.fetchone() + i = await cursor.fetchone() if not i: break self.assertIsInstance(i, str) received.append(i) # The amount of keys should equal 'dbsize' - dbsize = yield from protocol.dbsize() + dbsize = await protocol.dbsize() self.assertEqual(dbsize, len(received)) # Test fetchall - cursor = yield from protocol.scan(match='*') - received2 = yield from cursor.fetchall() + cursor = await protocol.scan(match='*') + received2 = await cursor.fetchall() self.assertIsInstance(received2, list) self.assertEqual(set(received), set(received2)) @redis_test - def test_set_scan(self, transport, protocol): + async def test_set_scan(self, transport, protocol): """ Test sscan """ size = 1000 items = [ 'value-%i' % i for i in range(size) ] # Create a huge set - yield from protocol.delete(['my-set']) - yield from protocol.sadd('my-set', items) + await protocol.delete(['my-set']) + await protocol.sadd('my-set', items) # Scan this set. - cursor = yield from protocol.sscan('my-set') + cursor = await protocol.sscan('my-set') received = [] while True: - i = yield from cursor.fetchone() + i = await cursor.fetchone() if not i: break self.assertIsInstance(i, str) @@ -1648,27 +1630,27 @@ def test_set_scan(self, transport, protocol): self.assertEqual(set(received), set(items)) # Test fetchall - cursor = yield from protocol.sscan('my-set') - received2 = yield from cursor.fetchall() + cursor = await protocol.sscan('my-set') + received2 = await cursor.fetchall() self.assertIsInstance(received2, set) self.assertEqual(set(received), received2) @redis_test - def test_dict_scan(self, transport, protocol): + async def test_dict_scan(self, transport, protocol): """ Test hscan """ size = 1000 items = { 'key-%i' % i: 'values-%i' % i for i in range(size) } # Create a huge set - yield from protocol.delete(['my-dict']) - yield from protocol.hmset('my-dict', items) + await protocol.delete(['my-dict']) + await protocol.hmset('my-dict', items) # Scan this set. - cursor = yield from protocol.hscan('my-dict') + cursor = await protocol.hscan('my-dict') received = {} while True: - i = yield from cursor.fetchone() + i = await cursor.fetchone() if not i: break self.assertIsInstance(i, dict) @@ -1679,27 +1661,27 @@ def test_dict_scan(self, transport, protocol): self.assertEqual(received, items) # Test fetchall - cursor = yield from protocol.hscan('my-dict') - received2 = yield from cursor.fetchall() + cursor = await protocol.hscan('my-dict') + received2 = await cursor.fetchall() self.assertIsInstance(received2, dict) self.assertEqual(received, received2) @redis_test - def test_sorted_dict_scan(self, transport, protocol): + async def test_sorted_dict_scan(self, transport, protocol): """ Test zscan """ size = 1000 items = { 'key-%i' % i: (i + 0.1) for i in range(size) } # Create a huge set - yield from protocol.delete(['my-z']) - yield from protocol.zadd('my-z', items) + await protocol.delete(['my-z']) + await protocol.zadd('my-z', items) # Scan this set. - cursor = yield from protocol.zscan('my-z') + cursor = await protocol.zscan('my-z') received = {} while True: - i = yield from cursor.fetchone() + i = await cursor.fetchone() if not i: break self.assertIsInstance(i, dict) @@ -1710,56 +1692,56 @@ def test_sorted_dict_scan(self, transport, protocol): self.assertEqual(received, items) # Test fetchall - cursor = yield from protocol.zscan('my-z') - received2 = yield from cursor.fetchall() + cursor = await protocol.zscan('my-z') + received2 = await cursor.fetchall() self.assertIsInstance(received2, dict) self.assertEqual(received, received2) @redis_test - def test_alternate_gets(self, transport, protocol): + async def test_alternate_gets(self, transport, protocol): """ Test _asdict/_asset/_aslist suffixes. """ # Prepare - yield from protocol.set(u'my_key', u'a') - yield from protocol.set(u'my_key2', u'b') + await protocol.set(u'my_key', u'a') + await protocol.set(u'my_key2', u'b') - yield from protocol.delete([ u'my_set' ]) - yield from protocol.sadd(u'my_set', [u'value1']) - yield from protocol.sadd(u'my_set', [u'value2']) + await protocol.delete([ u'my_set' ]) + await protocol.sadd(u'my_set', [u'value1']) + await protocol.sadd(u'my_set', [u'value2']) - yield from protocol.delete([ u'my_hash' ]) - yield from protocol.hmset(u'my_hash', {'a':'1', 'b':'2', 'c':'3'}) + await protocol.delete([ u'my_hash' ]) + await protocol.hmset(u'my_hash', {'a':'1', 'b':'2', 'c':'3'}) # Test mget_aslist - result = yield from protocol.mget_aslist(['my_key', 'my_key2']) + result = await protocol.mget_aslist(['my_key', 'my_key2']) self.assertEqual(result, [u'a', u'b']) self.assertIsInstance(result, list) # Test keys_aslist - result = yield from protocol.keys_aslist('some-prefix-') + result = await protocol.keys_aslist('some-prefix-') self.assertIsInstance(result, list) # Test smembers - result = yield from protocol.smembers_asset(u'my_set') + result = await protocol.smembers_asset(u'my_set') self.assertEqual(result, { u'value1', u'value2' }) self.assertIsInstance(result, set) # Test hgetall_asdict - result = yield from protocol.hgetall_asdict('my_hash') + result = await protocol.hgetall_asdict('my_hash') self.assertEqual(result, {'a':'1', 'b':'2', 'c':'3'}) self.assertIsInstance(result, dict) # test all inside a transaction. - transaction = yield from protocol.multi() - f1 = yield from transaction.mget_aslist(['my_key', 'my_key2']) - f2 = yield from transaction.smembers_asset(u'my_set') - f3 = yield from transaction.hgetall_asdict('my_hash') - yield from transaction.exec() + transaction = await protocol.multi() + f1 = await transaction.mget_aslist(['my_key', 'my_key2']) + f2 = await transaction.smembers_asset(u'my_set') + f3 = await transaction.hgetall_asdict('my_hash') + await transaction.exec() - result1 = yield from f1 - result2 = yield from f2 - result3 = yield from f3 + result1 = await f1 + result2 = await f2 + result3 = await f3 self.assertEqual(result1, [u'a', u'b']) self.assertIsInstance(result1, list) @@ -1771,96 +1753,93 @@ def test_alternate_gets(self, transport, protocol): self.assertIsInstance(result3, dict) @redis_test - def test_cancellation(self, transport, protocol): + async def test_cancellation(self, transport, protocol): """ Test CancelledError: when a query gets cancelled. """ - yield from protocol.delete(['key']) + await protocol.delete(['key']) - # Start a coroutine that runs a blocking command for 3seconds - @asyncio.coroutine - def run(): - yield from protocol.brpop(['key'], 3) - f = asyncio.ensure_future(run(), loop=self.loop) + # Start a task that runs a blocking command for 3seconds + f = self.loop.create_task(protocol.brpop(['key'], 3)) - # We cancel the coroutine before the answer arrives. - yield from asyncio.sleep(.5, loop=self.loop) + # We cancel the task before the answer arrives. + await asyncio.sleep(.5) f.cancel() # Now there's a cancelled future in protocol._queue, the # protocol._push_answer function should notice that and ignore the # incoming result from our `brpop` in this case. - yield from protocol.set('key', 'value') + await protocol.set('key', 'value') @redis_test - def test_watch_1(self, transport, protocol): + async def test_watch_1(self, transport, protocol): """ Test a transaction using watch. (Retrieve the watched value then use it inside the transaction.) """ - yield from protocol.set(u'key', u'val') + await protocol.set(u'key', u'val') # Test - yield from protocol.watch([u'key']) - value = yield from protocol.get(u'key') + await protocol.watch([u'key']) + value = await protocol.get(u'key') - t = yield from protocol.multi() + t = await protocol.multi() - yield from t.set(u'key', value + u'ue') + await t.set(u'key', value + u'ue') - yield from t.exec() + await t.exec() # Check - result = yield from protocol.get(u'key') + result = await protocol.get(u'key') self.assertEqual(result, u'value') @redis_test - def test_multi_watch_1(self, transport, protocol): + async def test_multi_watch_1(self, transport, protocol): """ Test a transaction, using watch (Test using the watched key inside the transaction.) """ - yield from protocol.set(u'key', u'0') - yield from protocol.set(u'other_key', u'0') + await protocol.set(u'key', u'0') + await protocol.set(u'other_key', u'0') # Test self.assertEqual(protocol.in_transaction, False) - t = yield from protocol.multi(watch=['other_key']) + t = await protocol.multi(watch=['other_key']) self.assertEqual(protocol.in_transaction, True) - yield from t.set(u'key', u'value') - yield from t.set(u'other_key', u'my_value') - yield from t.exec() + await t.set(u'key', u'value') + await t.set(u'other_key', u'my_value') + await t.exec() # Check self.assertEqual(protocol.in_transaction, False) - result = yield from protocol.get(u'key') + result = await protocol.get(u'key') self.assertEqual(result, u'value') - result = yield from protocol.get(u'other_key') + result = await protocol.get(u'other_key') self.assertEqual(result, u'my_value') @redis_test - def test_multi_watch_2(self, transport, protocol): + async def test_multi_watch_2(self, transport, protocol): """ Test using the watched key outside the transaction. (the transaction should fail in this case.) """ # Setup - transport2, protocol2 = yield from connect(self.loop) + transport2, protocol2 = await connect() - yield from protocol.set(u'key', u'0') - yield from protocol.set(u'other_key', u'0') + await protocol.set(u'key', u'0') + await protocol.set(u'other_key', u'0') # Test - t = yield from protocol.multi(watch=['other_key']) - yield from protocol2.set('other_key', 'other_value') - yield from t.set(u'other_key', u'value') + t = await protocol.multi(watch=['other_key']) + await protocol2.set('other_key', 'other_value') + await t.set(u'other_key', u'value') with self.assertRaises(TransactionError): - yield from t.exec() + await t.exec() # Check self.assertEqual(protocol.in_transaction, False) - result = yield from protocol.get(u'other_key') + result = await protocol.get(u'other_key') self.assertEqual(result, u'other_value') transport2.close() @@ -1872,44 +1851,43 @@ def setUp(self): self.protocol_class = lambda **kw: RedisProtocol(encoder=BytesEncoder(), **kw) @redis_test - def test_bytes_protocol(self, transport, protocol): + async def test_bytes_protocol(self, transport, protocol): # When passing string instead of bytes, this protocol should raise an exception. with self.assertRaises(TypeError): - result = yield from protocol.set('key', 'value') + result = await protocol.set('key', 'value') # Setting bytes - result = yield from protocol.set(b'key', b'value') + result = await protocol.set(b'key', b'value') self.assertEqual(result, StatusReply('OK')) # Getting bytes - result = yield from protocol.get(b'key') + result = await protocol.get(b'key') self.assertEqual(result, b'value') @redis_test - def test_pubsub(self, transport, protocol): + async def test_pubsub(self, transport, protocol): """ Test pubsub with BytesEncoder. Channel names and data are now bytes. """ - @asyncio.coroutine - def listener(): + async def listener(): # Subscribe - transport2, protocol2 = yield from connect(self.loop, - protocol = lambda **kw: RedisProtocol(encoder=BytesEncoder(), **kw)) + transport2, protocol2 = await connect( + lambda **kw: RedisProtocol(encoder=BytesEncoder(), **kw), + ) - subscription = yield from protocol2.start_subscribe() - yield from subscription.subscribe([b'our_channel']) - value = yield from subscription.next_published() + subscription = await protocol2.start_subscribe() + await subscription.subscribe([b'our_channel']) + value = await subscription.next_published() self.assertEqual(value.channel, b'our_channel') self.assertEqual(value.value, b'message1') return transport2 - @asyncio.coroutine - def sender(): - yield from protocol.publish(b'our_channel', b'message1') + async def sender(): + await protocol.publish(b'our_channel', b'message1') f = asyncio.ensure_future(listener(), loop=self.loop) - yield from asyncio.sleep(.5, loop=self.loop) - yield from sender() - transport2 = yield from f + await asyncio.sleep(.5, loop=self.loop) + await sender() + transport2 = await f transport2.close() @@ -1919,18 +1897,16 @@ def test_protocol(self): def factory(**kw): return RedisProtocol(encoder=BytesEncoder(), enable_typechecking=False, **kw) - loop = asyncio.get_event_loop() - - @asyncio.coroutine - def test(): - transport, protocol = yield from connect(loop, protocol=factory) + async def test(): + transport, protocol = await connect(factory) # Setting values should still work. - result = yield from protocol.set(b'key', b'value') + result = await protocol.set(b'key', b'value') self.assertEqual(result, StatusReply('OK')) transport.close() + loop = asyncio.get_event_loop() loop.run_until_complete(test()) @@ -1940,16 +1916,17 @@ def setUp(self): self.loop = asyncio.get_event_loop() def test_connection(self): - @asyncio.coroutine - def test(): + async def test(): # Create connection - connection = yield from Connection.create(host=HOST, port=PORT) - self.assertEqual(repr(connection), "Connection(host=%r, port=%r)" % (HOST, PORT)) + connection = await Connection.create(host=HOST, port=PORT) + self.assertEqual( + repr(connection), f"Connection(host='{HOST}', port={PORT})" + ) self.assertEqual(connection._closing, False) # Test get/set - yield from connection.set('key', 'value') - result = yield from connection.get('key') + await connection.set('key', 'value') + result = await connection.get('key') self.assertEqual(result, 'value') connection.close() @@ -1967,15 +1944,16 @@ def setUp(self): def test_pool(self): """ Test creation of Connection instance. """ - @asyncio.coroutine - def test(): + async def test(): # Create pool - connection = yield from Pool.create(host=HOST, port=PORT) - self.assertEqual(repr(connection), "Pool(host=%r, port=%r, poolsize=1)" % (HOST, PORT)) + connection = await Pool.create(host=HOST, port=PORT) + self.assertEqual( + repr(connection), f"Pool(host='{HOST}', port={PORT}, poolsize=1)" + ) # Test get/set - yield from connection.set('key', 'value') - result = yield from connection.get('key') + await connection.set('key', 'value') + result = await connection.get('key') self.assertEqual(result, 'value') # Test default poolsize @@ -1990,20 +1968,20 @@ def test_connection_in_use(self): When a blocking call is running, it's impossible to use the same protocol for another call. """ - @asyncio.coroutine - def test(): + async def test(): # Create connection - connection = yield from Pool.create(host=HOST, port=PORT) + connection = await Pool.create(host=HOST, port=PORT) self.assertEqual(connection.connections_in_use, 0) # Wait for ever. (This blocking pop doesn't return.) - yield from connection.delete([ 'unknown-key' ]) - f = asyncio.ensure_future(connection.blpop(['unknown-key']), loop=self.loop) - yield from asyncio.sleep(.1, loop=self.loop) # Sleep to make sure that the above coroutine started executing. + await connection.delete([ 'unknown-key' ]) + f = self.loop.create_task(connection.blpop(['unknown-key'])) + # Sleep to make sure that the above task started executing + await asyncio.sleep(.1) # Run command in other thread. with self.assertRaises(NoAvailableConnectionsInPoolError) as e: - yield from connection.set('key', 'value') + await connection.set('key', 'value') self.assertIn('No available connections in the pool', e.exception.args[0]) self.assertEqual(connection.connections_in_use, 1) @@ -2012,7 +1990,7 @@ def test(): # Consume this future (which now contains ConnectionLostError) with self.assertRaises(ConnectionLostError): - yield from f + await f self.loop.run_until_complete(test()) @@ -2020,35 +1998,30 @@ def test_parallel_requests(self): """ Test a blocking pop and a set using a connection pool. """ - @asyncio.coroutine - def test(): + async def test(): # Create connection - connection = yield from Pool.create(host=HOST, port=PORT, poolsize=2) - yield from connection.delete([ 'my-list' ]) + connection = await Pool.create(host=HOST, port=PORT, poolsize=2) + await connection.delete([ 'my-list' ]) results = [] # Sink: receive items using blocking pop - @asyncio.coroutine - def sink(): + async def sink(): for i in range(0, 5): - reply = yield from connection.blpop(['my-list']) + reply = await connection.blpop(['my-list']) self.assertIsInstance(reply, BlockingPopReply) self.assertIsInstance(reply.value, str) results.append(reply.value) self.assertIn(u"BlockingPopReply(list_name='my-list', value='", repr(reply)) # Source: Push items on the queue - @asyncio.coroutine - def source(): + async def source(): for i in range(0, 5): - yield from connection.rpush('my-list', [str(i)]) - yield from asyncio.sleep(.5, loop=self.loop) + await connection.rpush('my-list', [str(i)]) + await asyncio.sleep(.5) - # Run both coroutines. - f1 = asyncio.ensure_future(source(), loop=self.loop) - f2 = asyncio.ensure_future(sink(), loop=self.loop) - yield from asyncio.gather(f1, f2) + # Run both coroutines + await asyncio.gather(source(), sink()) # Test results. self.assertEqual(results, [ str(i) for i in range(0, 5) ]) @@ -2061,19 +2034,18 @@ def test_select_db(self): """ Connect to two different DBs. """ - @asyncio.coroutine - def test(): - c1 = yield from Pool.create(host=HOST, port=PORT, poolsize=10, db=1) - c2 = yield from Pool.create(host=HOST, port=PORT, poolsize=10, db=2) + async def test(): + c1 = await Pool.create(host=HOST, port=PORT, poolsize=10, db=1) + c2 = await Pool.create(host=HOST, port=PORT, poolsize=10, db=2) - c3 = yield from Pool.create(host=HOST, port=PORT, poolsize=10, db=1) - c4 = yield from Pool.create(host=HOST, port=PORT, poolsize=10, db=2) + c3 = await Pool.create(host=HOST, port=PORT, poolsize=10, db=1) + c4 = await Pool.create(host=HOST, port=PORT, poolsize=10, db=2) - yield from c1.set('key', 'A') - yield from c2.set('key', 'B') + await c1.set('key', 'A') + await c2.set('key', 'B') - r1 = yield from c3.get('key') - r2 = yield from c4.get('key') + r1 = await c3.get('key') + r2 = await c4.get('key') self.assertEqual(r1, 'A') self.assertEqual(r2, 'B') @@ -2087,50 +2059,48 @@ def test_in_use_flag(self): """ Do several blocking calls and see whether in_use increments. """ - @asyncio.coroutine - def test(): + async def test(): # Create connection - connection = yield from Pool.create(host=HOST, port=PORT, poolsize=10) + connection = await Pool.create(host=HOST, port=PORT, poolsize=10) for i in range(0, 10): - yield from connection.delete([ 'my-list-%i' % i ]) + await connection.delete([ 'my-list-%i' % i ]) - @asyncio.coroutine - def sink(i): - the_list, result = yield from connection.blpop(['my-list-%i' % i]) + async def sink(i): + await connection.blpop(['my-list-%i' % i]) futures = [] for i in range(0, 10): self.assertEqual(connection.connections_in_use, i) - futures.append(asyncio.ensure_future(sink(i), loop=self.loop)) - yield from asyncio.sleep(.1, loop=self.loop) # Sleep to make sure that the above coroutine started executing. + futures.append(self.loop.create_task(sink(i))) + # Sleep to make sure that the above coroutine started executing + await asyncio.sleep(.1) # One more blocking call should fail. with self.assertRaises(NoAvailableConnectionsInPoolError) as e: - yield from connection.delete([ 'my-list-one-more' ]) - yield from connection.blpop(['my-list-one-more']) + await connection.delete([ 'my-list-one-more' ]) + await connection.blpop(['my-list-one-more']) self.assertIn('No available connections in the pool', e.exception.args[0]) connection.close() # Consume this futures (which now contain ConnectionLostError) with self.assertRaises(ConnectionLostError): - yield from asyncio.gather(*futures) + await asyncio.gather(*futures) self.loop.run_until_complete(test()) def test_lua_script_in_pool(self): - @asyncio.coroutine - def test(): + async def test(): # Create connection - connection = yield from Pool.create(host=HOST, port=PORT, poolsize=3) + connection = await Pool.create(host=HOST, port=PORT, poolsize=3) # Register script - script = yield from connection.register_script("return 100") + script = await connection.register_script("return 100") self.assertIsInstance(script, Script) # Run script - scriptreply = yield from script.run() - result = yield from scriptreply.return_value() + scriptreply = await script.run() + result = await scriptreply.return_value() self.assertEqual(result, 100) connection.close() @@ -2141,31 +2111,30 @@ def test_transactions(self): """ Do several transactions in parallel. """ - @asyncio.coroutine - def test(): + async def test(): # Create connection - connection = yield from Pool.create(host=HOST, port=PORT, poolsize=3) + connection = await Pool.create(host=HOST, port=PORT, poolsize=3) - t1 = yield from connection.multi() - t2 = yield from connection.multi() - yield from connection.multi() + t1 = await connection.multi() + t2 = await connection.multi() + await connection.multi() # Fourth transaction should fail. (Pool is full) with self.assertRaises(NoAvailableConnectionsInPoolError) as e: - yield from connection.multi() + await connection.multi() self.assertIn('No available connections in the pool', e.exception.args[0]) # Run commands in transaction - yield from t1.set(u'key', u'value') - yield from t2.set(u'key2', u'value2') + await t1.set(u'key', u'value') + await t2.set(u'key2', u'value2') # Commit. - yield from t1.exec() - yield from t2.exec() + await t1.exec() + await t2.exec() # Check - result1 = yield from connection.get(u'key') - result2 = yield from connection.get(u'key2') + result1 = await connection.get(u'key') + result2 = await connection.get(u'key2') self.assertEqual(result1, u'value') self.assertEqual(result2, u'value2') @@ -2179,10 +2148,9 @@ def test_connection_reconnect(self): Test whether the connection reconnects. (needs manual interaction.) """ - @asyncio.coroutine - def test(): - connection = yield from Pool.create(host=HOST, port=PORT, poolsize=1) - yield from connection.set('key', 'value') + async def test(): + connection = await Pool.create(host=HOST, port=PORT, poolsize=1) + await connection.set('key', 'value') # Try the reconnect cycle several times. (Be sure that the # `connection_lost` callback doesn't set variables that avoid @@ -2191,10 +2159,10 @@ def test(): transport = connection._connections[0].transport transport.close() - yield from asyncio.sleep(1, loop=self.loop) # Give asyncio time to reconnect. + await asyncio.sleep(1) # Give asyncio time to reconnect # Test get/set - yield from connection.set('key', 'value') + await connection.set('key', 'value') connection.close() @@ -2206,41 +2174,39 @@ def test_connection_lost(self): NotConnectedError. (Unless the transport would be auto-reconnecting and have established a new connection.) """ - @asyncio.coroutine - def test(): + async def test(): # Create connection - transport, protocol = yield from connect(self.loop, RedisProtocol) - yield from protocol.set('key', 'value') + transport, protocol = await connect(RedisProtocol) + await protocol.set('key', 'value') # Close transport self.assertEqual(protocol.is_connected, True) transport.close() - yield from asyncio.sleep(.5, loop=self.loop) + await asyncio.sleep(.5) self.assertEqual(protocol.is_connected, False) # Test get/set with self.assertRaises(NotConnectedError): - yield from protocol.set('key', 'value') + await protocol.set('key', 'value') transport.close() self.loop.run_until_complete(test()) - # Test connection lost in connection pool. - @asyncio.coroutine - def test(): + def test_connection_lost_pool(self): + async def test(): # Create connection - connection = yield from Pool.create(host=HOST, port=PORT, poolsize=1, auto_reconnect=False) - yield from connection.set('key', 'value') + connection = await Pool.create(host=HOST, port=PORT, poolsize=1, auto_reconnect=False) + await connection.set('key', 'value') # Close transport transport = connection._connections[0].transport transport.close() - yield from asyncio.sleep(.5, loop=self.loop) + await asyncio.sleep(.5) # Test get/set with self.assertRaises(NoAvailableConnectionsInPoolError) as e: - yield from connection.set('key', 'value') + await connection.set('key', 'value') self.assertIn('No available connections in the pool: size=1, in_use=0, connected=0', e.exception.args[0]) connection.close() @@ -2254,11 +2220,10 @@ class NoGlobalLoopTest(TestCase): """ def test_no_global_loop(self): old_loop = asyncio.get_event_loop() + # Remove global loop and create a new one. + asyncio.set_event_loop(None) + new_loop = asyncio.new_event_loop() try: - # Remove global loop and create a new one. - asyncio.set_event_loop(None) - new_loop = asyncio.new_event_loop() - # ** Run code on the new loop. ** # Create connection @@ -2281,9 +2246,8 @@ def test_no_global_loop(self): finally: connection.close() finally: - # Run loop briefly until socket has been closed. (call_soon behind the scenes.) - run_briefly(new_loop) - + # Run loop briefly until socket has been closed. + new_loop.run_until_complete(asyncio.sleep(0.1)) new_loop.close() asyncio.set_event_loop(old_loop) @@ -2321,7 +2285,7 @@ def tearDown(self): def _start_redis_server(loop): - print('Running Redis server REDIS_HOST=%r REDIS_PORT=%r...' % (HOST, PORT)) + print(f'Running Redis server REDIS_HOST={HOST} REDIS_PORT={PORT}...') redis_srv = loop.run_until_complete( asyncio.create_subprocess_exec( @@ -2331,10 +2295,9 @@ def _start_redis_server(loop): '--maxclients', '100', '--save', '""', '--loglevel', 'warning', - loop=loop, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL)) - loop.run_until_complete(asyncio.sleep(.05, loop=loop)) + loop.run_until_complete(asyncio.sleep(.05)) return redis_srv From d5e4dcadea4908a70ce99ea518964c3caed8bbd0 Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 15 Jan 2020 12:03:17 +0000 Subject: [PATCH 04/21] Get rid of nested test methods --- tests.py | 516 +++++++++++++++++++++++++++---------------------------- 1 file changed, 256 insertions(+), 260 deletions(-) diff --git a/tests.py b/tests.py index 483f8e4..cd11ff3 100755 --- a/tests.py +++ b/tests.py @@ -59,10 +59,7 @@ async def connect(protocol=RedisProtocol): def redis_test(function): - """ - Decorator for methods (which are coroutines) in RedisProtocolTest - - Wraps the coroutine inside `run_until_complete`. + """Decorator for async test methods in RedisProtocolTest """ def wrapper(self): async def c(): @@ -85,6 +82,16 @@ async def c(): return wrapper +def async_test(function): + """Decorator for other async test methods + """ + def wrapper(self): + loop = getattr(self, "loop", asyncio.get_event_loop()) + loop.run_until_complete(function(self)) + + return wrapper + + class TestCase(unittest.TestCase): def tearDown(self): # Collect garbage on tearDown. (This can print ResourceWarnings.) @@ -1892,22 +1899,19 @@ async def sender(): class NoTypeCheckingTest(TestCase): - def test_protocol(self): - # Override protocol, disabling type checking. - def factory(**kw): - return RedisProtocol(encoder=BytesEncoder(), enable_typechecking=False, **kw) - - async def test(): - transport, protocol = await connect(factory) + @async_test + async def test_protocol(self): + transport, protocol = await connect( + lambda **kw: RedisProtocol( + encoder=BytesEncoder(), enable_typechecking=False, **kw + ), + ) - # Setting values should still work. - result = await protocol.set(b'key', b'value') - self.assertEqual(result, StatusReply('OK')) + # Setting values should still work. + result = await protocol.set(b'key', b'value') + self.assertEqual(result, StatusReply('OK')) - transport.close() - - loop = asyncio.get_event_loop() - loop.run_until_complete(test()) + transport.close() class RedisConnectionTest(TestCase): @@ -1915,26 +1919,24 @@ class RedisConnectionTest(TestCase): def setUp(self): self.loop = asyncio.get_event_loop() - def test_connection(self): - async def test(): - # Create connection - connection = await Connection.create(host=HOST, port=PORT) - self.assertEqual( - repr(connection), f"Connection(host='{HOST}', port={PORT})" - ) - self.assertEqual(connection._closing, False) - - # Test get/set - await connection.set('key', 'value') - result = await connection.get('key') - self.assertEqual(result, 'value') + @async_test + async def test_connection(self): + # Create connection + connection = await Connection.create(host=HOST, port=PORT) + self.assertEqual( + repr(connection), f"Connection(host='{HOST}', port={PORT})" + ) + self.assertEqual(connection._closing, False) - connection.close() + # Test get/set + await connection.set('key', 'value') + result = await connection.get('key') + self.assertEqual(result, 'value') - # Test closing flag - self.assertEqual(connection._closing, True) + connection.close() - self.loop.run_until_complete(test()) + # Test closing flag + self.assertEqual(connection._closing, True) class RedisPoolTest(TestCase): @@ -1942,276 +1944,256 @@ class RedisPoolTest(TestCase): def setUp(self): self.loop = asyncio.get_event_loop() - def test_pool(self): + @async_test + async def test_pool(self): """ Test creation of Connection instance. """ - async def test(): - # Create pool - connection = await Pool.create(host=HOST, port=PORT) - self.assertEqual( - repr(connection), f"Pool(host='{HOST}', port={PORT}, poolsize=1)" - ) - - # Test get/set - await connection.set('key', 'value') - result = await connection.get('key') - self.assertEqual(result, 'value') + # Create pool + connection = await Pool.create(host=HOST, port=PORT) + self.assertEqual( + repr(connection), f"Pool(host='{HOST}', port={PORT}, poolsize=1)" + ) - # Test default poolsize - self.assertEqual(connection.poolsize, 1) + # Test get/set + await connection.set('key', 'value') + result = await connection.get('key') + self.assertEqual(result, 'value') - connection.close() + # Test default poolsize + self.assertEqual(connection.poolsize, 1) - self.loop.run_until_complete(test()) + connection.close() - def test_connection_in_use(self): + @async_test + async def test_connection_in_use(self): """ When a blocking call is running, it's impossible to use the same protocol for another call. """ - async def test(): - # Create connection - connection = await Pool.create(host=HOST, port=PORT) - self.assertEqual(connection.connections_in_use, 0) - - # Wait for ever. (This blocking pop doesn't return.) - await connection.delete([ 'unknown-key' ]) - f = self.loop.create_task(connection.blpop(['unknown-key'])) - # Sleep to make sure that the above task started executing - await asyncio.sleep(.1) - - # Run command in other thread. - with self.assertRaises(NoAvailableConnectionsInPoolError) as e: - await connection.set('key', 'value') - self.assertIn('No available connections in the pool', e.exception.args[0]) - - self.assertEqual(connection.connections_in_use, 1) + # Create connection + connection = await Pool.create(host=HOST, port=PORT) + self.assertEqual(connection.connections_in_use, 0) + + # Wait for ever. (This blocking pop doesn't return.) + await connection.delete([ 'unknown-key' ]) + f = self.loop.create_task(connection.blpop(['unknown-key'])) + # Sleep to make sure that the above task started executing + await asyncio.sleep(.1) + + # Run command in other thread. + with self.assertRaises(NoAvailableConnectionsInPoolError) as e: + await connection.set('key', 'value') + self.assertIn('No available connections in the pool', e.exception.args[0]) - connection.close() + self.assertEqual(connection.connections_in_use, 1) - # Consume this future (which now contains ConnectionLostError) - with self.assertRaises(ConnectionLostError): - await f + connection.close() - self.loop.run_until_complete(test()) + # Consume this future (which now contains ConnectionLostError) + with self.assertRaises(ConnectionLostError): + await f - def test_parallel_requests(self): + @async_test + async def test_parallel_requests(self): """ Test a blocking pop and a set using a connection pool. """ - async def test(): - # Create connection - connection = await Pool.create(host=HOST, port=PORT, poolsize=2) - await connection.delete([ 'my-list' ]) - - results = [] + # Create connection + connection = await Pool.create(host=HOST, port=PORT, poolsize=2) + await connection.delete([ 'my-list' ]) - # Sink: receive items using blocking pop - async def sink(): - for i in range(0, 5): - reply = await connection.blpop(['my-list']) - self.assertIsInstance(reply, BlockingPopReply) - self.assertIsInstance(reply.value, str) - results.append(reply.value) - self.assertIn(u"BlockingPopReply(list_name='my-list', value='", repr(reply)) + results = [] - # Source: Push items on the queue - async def source(): - for i in range(0, 5): - await connection.rpush('my-list', [str(i)]) - await asyncio.sleep(.5) + # Sink: receive items using blocking pop + async def sink(): + for i in range(0, 5): + reply = await connection.blpop(['my-list']) + self.assertIsInstance(reply, BlockingPopReply) + self.assertIsInstance(reply.value, str) + results.append(reply.value) + self.assertIn(u"BlockingPopReply(list_name='my-list', value='", repr(reply)) - # Run both coroutines - await asyncio.gather(source(), sink()) + # Source: Push items on the queue + async def source(): + for i in range(0, 5): + await connection.rpush('my-list', [str(i)]) + await asyncio.sleep(.5) - # Test results. - self.assertEqual(results, [ str(i) for i in range(0, 5) ]) + # Run both coroutines + await asyncio.gather(source(), sink()) - connection.close() + # Test results. + self.assertEqual(results, [ str(i) for i in range(0, 5) ]) - self.loop.run_until_complete(test()) + connection.close() - def test_select_db(self): + @async_test + async def test_select_db(self): """ Connect to two different DBs. """ - async def test(): - c1 = await Pool.create(host=HOST, port=PORT, poolsize=10, db=1) - c2 = await Pool.create(host=HOST, port=PORT, poolsize=10, db=2) - - c3 = await Pool.create(host=HOST, port=PORT, poolsize=10, db=1) - c4 = await Pool.create(host=HOST, port=PORT, poolsize=10, db=2) + c1 = await Pool.create(host=HOST, port=PORT, poolsize=10, db=1) + c2 = await Pool.create(host=HOST, port=PORT, poolsize=10, db=2) - await c1.set('key', 'A') - await c2.set('key', 'B') + c3 = await Pool.create(host=HOST, port=PORT, poolsize=10, db=1) + c4 = await Pool.create(host=HOST, port=PORT, poolsize=10, db=2) - r1 = await c3.get('key') - r2 = await c4.get('key') + await c1.set('key', 'A') + await c2.set('key', 'B') - self.assertEqual(r1, 'A') - self.assertEqual(r2, 'B') + r1 = await c3.get('key') + r2 = await c4.get('key') - for c in [ c1, c2, c3, c4]: - c.close() + self.assertEqual(r1, 'A') + self.assertEqual(r2, 'B') - self.loop.run_until_complete(test()) + for c in [ c1, c2, c3, c4]: + c.close() - def test_in_use_flag(self): + @async_test + async def test_in_use_flag(self): """ Do several blocking calls and see whether in_use increments. """ - async def test(): - # Create connection - connection = await Pool.create(host=HOST, port=PORT, poolsize=10) - for i in range(0, 10): - await connection.delete([ 'my-list-%i' % i ]) - - async def sink(i): - await connection.blpop(['my-list-%i' % i]) - - futures = [] - for i in range(0, 10): - self.assertEqual(connection.connections_in_use, i) - futures.append(self.loop.create_task(sink(i))) - # Sleep to make sure that the above coroutine started executing - await asyncio.sleep(.1) - - # One more blocking call should fail. - with self.assertRaises(NoAvailableConnectionsInPoolError) as e: - await connection.delete([ 'my-list-one-more' ]) - await connection.blpop(['my-list-one-more']) - self.assertIn('No available connections in the pool', e.exception.args[0]) - - connection.close() + # Create connection + connection = await Pool.create(host=HOST, port=PORT, poolsize=10) + for i in range(0, 10): + await connection.delete([ 'my-list-%i' % i ]) + + async def sink(i): + await connection.blpop(['my-list-%i' % i]) + + futures = [] + for i in range(0, 10): + self.assertEqual(connection.connections_in_use, i) + futures.append(self.loop.create_task(sink(i))) + # Sleep to make sure that the above coroutine started executing + await asyncio.sleep(.1) - # Consume this futures (which now contain ConnectionLostError) - with self.assertRaises(ConnectionLostError): - await asyncio.gather(*futures) + # One more blocking call should fail. + with self.assertRaises(NoAvailableConnectionsInPoolError) as e: + await connection.delete([ 'my-list-one-more' ]) + await connection.blpop(['my-list-one-more']) + self.assertIn('No available connections in the pool', e.exception.args[0]) - self.loop.run_until_complete(test()) + connection.close() - def test_lua_script_in_pool(self): - async def test(): - # Create connection - connection = await Pool.create(host=HOST, port=PORT, poolsize=3) + # Consume this futures (which now contain ConnectionLostError) + with self.assertRaises(ConnectionLostError): + await asyncio.gather(*futures) - # Register script - script = await connection.register_script("return 100") - self.assertIsInstance(script, Script) + @async_test + async def test_lua_script_in_pool(self): + # Create connection + connection = await Pool.create(host=HOST, port=PORT, poolsize=3) - # Run script - scriptreply = await script.run() - result = await scriptreply.return_value() - self.assertEqual(result, 100) + # Register script + script = await connection.register_script("return 100") + self.assertIsInstance(script, Script) - connection.close() + # Run script + scriptreply = await script.run() + result = await scriptreply.return_value() + self.assertEqual(result, 100) - self.loop.run_until_complete(test()) + connection.close() - def test_transactions(self): + @async_test + async def test_transactions(self): """ Do several transactions in parallel. """ - async def test(): - # Create connection - connection = await Pool.create(host=HOST, port=PORT, poolsize=3) - - t1 = await connection.multi() - t2 = await connection.multi() - await connection.multi() + # Create connection + connection = await Pool.create(host=HOST, port=PORT, poolsize=3) - # Fourth transaction should fail. (Pool is full) - with self.assertRaises(NoAvailableConnectionsInPoolError) as e: - await connection.multi() - self.assertIn('No available connections in the pool', e.exception.args[0]) + t1 = await connection.multi() + t2 = await connection.multi() + await connection.multi() - # Run commands in transaction - await t1.set(u'key', u'value') - await t2.set(u'key2', u'value2') + # Fourth transaction should fail. (Pool is full) + with self.assertRaises(NoAvailableConnectionsInPoolError) as e: + await connection.multi() + self.assertIn('No available connections in the pool', e.exception.args[0]) - # Commit. - await t1.exec() - await t2.exec() + # Run commands in transaction + await t1.set(u'key', u'value') + await t2.set(u'key2', u'value2') - # Check - result1 = await connection.get(u'key') - result2 = await connection.get(u'key2') + # Commit. + await t1.exec() + await t2.exec() - self.assertEqual(result1, u'value') - self.assertEqual(result2, u'value2') + # Check + result1 = await connection.get(u'key') + result2 = await connection.get(u'key2') - connection.close() + self.assertEqual(result1, u'value') + self.assertEqual(result2, u'value2') - self.loop.run_until_complete(test()) + connection.close() - def test_connection_reconnect(self): + @async_test + async def test_connection_reconnect(self): """ Test whether the connection reconnects. (needs manual interaction.) """ - async def test(): - connection = await Pool.create(host=HOST, port=PORT, poolsize=1) - await connection.set('key', 'value') + connection = await Pool.create(host=HOST, port=PORT, poolsize=1) + await connection.set('key', 'value') - # Try the reconnect cycle several times. (Be sure that the - # `connection_lost` callback doesn't set variables that avoid - # reconnection a second time.) - for i in range(3): - transport = connection._connections[0].transport - transport.close() - - await asyncio.sleep(1) # Give asyncio time to reconnect + # Try the reconnect cycle several times. (Be sure that the + # `connection_lost` callback doesn't set variables that avoid + # reconnection a second time.) + for i in range(3): + transport = connection._connections[0].transport + transport.close() - # Test get/set - await connection.set('key', 'value') + await asyncio.sleep(1) # Give asyncio time to reconnect - connection.close() + # Test get/set + await connection.set('key', 'value') - self.loop.run_until_complete(test()) + connection.close() - def test_connection_lost(self): + @async_test + async def test_connection_lost(self): """ When the transport is closed, any further commands should raise NotConnectedError. (Unless the transport would be auto-reconnecting and have established a new connection.) """ - async def test(): - # Create connection - transport, protocol = await connect(RedisProtocol) - await protocol.set('key', 'value') - - # Close transport - self.assertEqual(protocol.is_connected, True) - transport.close() - await asyncio.sleep(.5) - self.assertEqual(protocol.is_connected, False) - - # Test get/set - with self.assertRaises(NotConnectedError): - await protocol.set('key', 'value') + # Create connection + transport, protocol = await connect(RedisProtocol) + await protocol.set('key', 'value') - transport.close() + # Close transport + self.assertEqual(protocol.is_connected, True) + transport.close() + await asyncio.sleep(.5) + self.assertEqual(protocol.is_connected, False) - self.loop.run_until_complete(test()) + # Test get/set + with self.assertRaises(NotConnectedError): + await protocol.set('key', 'value') - def test_connection_lost_pool(self): - async def test(): - # Create connection - connection = await Pool.create(host=HOST, port=PORT, poolsize=1, auto_reconnect=False) - await connection.set('key', 'value') + transport.close() - # Close transport - transport = connection._connections[0].transport - transport.close() - await asyncio.sleep(.5) + @async_test + async def test_connection_lost_pool(self): + # Create connection + connection = await Pool.create(host=HOST, port=PORT, poolsize=1, auto_reconnect=False) + await connection.set('key', 'value') - # Test get/set - with self.assertRaises(NoAvailableConnectionsInPoolError) as e: - await connection.set('key', 'value') - self.assertIn('No available connections in the pool: size=1, in_use=0, connected=0', e.exception.args[0]) + # Close transport + transport = connection._connections[0].transport + transport.close() + await asyncio.sleep(.5) - connection.close() + # Test get/set + with self.assertRaises(NoAvailableConnectionsInPoolError) as e: + await connection.set('key', 'value') + self.assertIn('No available connections in the pool: size=1, in_use=0, connected=0', e.exception.args[0]) - self.loop.run_until_complete(test()) + connection.close() class NoGlobalLoopTest(TestCase): @@ -2219,35 +2201,49 @@ class NoGlobalLoopTest(TestCase): If we set the global loop variable to None, everything should still work. """ def test_no_global_loop(self): - old_loop = asyncio.get_event_loop() - # Remove global loop and create a new one. - asyncio.set_event_loop(None) - new_loop = asyncio.new_event_loop() - try: - # ** Run code on the new loop. ** - - # Create connection - connection = new_loop.run_until_complete(Connection.create(host=HOST, port=PORT, loop=new_loop)) + async def test(): + connection = await Connection.create(host=HOST, port=PORT) self.assertIsInstance(connection, Connection) try: # Delete keys - new_loop.run_until_complete(connection.delete(['key1', 'key2'])) + await connection.delete(['key1', 'key2']) # Get/set - new_loop.run_until_complete(connection.set('key1', 'value')) - result = new_loop.run_until_complete(connection.get('key1')) + await connection.set('key1', 'value') + result = await connection.get('key1') self.assertEqual(result, 'value') # hmset/hmget (something that uses a MultiBulkReply) - new_loop.run_until_complete(connection.hmset('key2', { 'a': 'b', 'c': 'd' })) - result = new_loop.run_until_complete(connection.hgetall_asdict('key2')) + await connection.hmset('key2', {'a': 'b', 'c': 'd'}) + result = await connection.hgetall_asdict('key2') + self.assertEqual(result, {'a': 'b', 'c': 'd'}) + + # Delete keys + await connection.delete(['key1', 'key2']) + + # Get/set + await connection.set('key1', 'value') + result = await connection.get('key1') + self.assertEqual(result, 'value') + + # hmset/hmget (something that uses a MultiBulkReply) + await connection.hmset('key2', { 'a': 'b', 'c': 'd' }) + result = await connection.hgetall_asdict('key2') self.assertEqual(result, { 'a': 'b', 'c': 'd' }) + finally: connection.close() + # Run loop briefly until socket has been closed. + await asyncio.sleep(0.1) + + # Remove global loop and create a new one. + old_loop = asyncio.get_event_loop() + asyncio.set_event_loop(None) + new_loop = asyncio.new_event_loop() + try: + new_loop.run_until_complete(test()) finally: - # Run loop briefly until socket has been closed. - new_loop.run_until_complete(asyncio.sleep(0.1)) new_loop.close() asyncio.set_event_loop(old_loop) @@ -2284,20 +2280,20 @@ def tearDown(self): super().tearDown() -def _start_redis_server(loop): +async def start_redis_server(): print(f'Running Redis server REDIS_HOST={HOST} REDIS_PORT={PORT}...') - redis_srv = loop.run_until_complete( - asyncio.create_subprocess_exec( - 'redis-server', - '--port', str(PORT), - ('--bind' if PORT else '--unixsocket'), HOST, - '--maxclients', '100', - '--save', '""', - '--loglevel', 'warning', - stdout=asyncio.subprocess.DEVNULL, - stderr=asyncio.subprocess.DEVNULL)) - loop.run_until_complete(asyncio.sleep(.05)) + redis_srv = await asyncio.create_subprocess_exec( + 'redis-server', + '--port', str(PORT), + ('--bind' if PORT else '--unixsocket'), HOST, + '--maxclients', '100', + '--save', '""', + '--loglevel', 'warning', + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + ) + await asyncio.sleep(.05) return redis_srv @@ -2317,11 +2313,11 @@ def setUp(self): if __name__ == '__main__': if START_REDIS_SERVER: - redis_srv = _start_redis_server(asyncio.get_event_loop()) + loop = asyncio.get_event_loop() + redis_srv = loop.run_until_complete(start_redis_server()) try: unittest.main() finally: if START_REDIS_SERVER: redis_srv.terminate() - From 0bf39b873832a9488b3b1aeebf8b3488c86cdc34 Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 15 Jan 2020 12:14:42 +0000 Subject: [PATCH 05/21] Documentation --- README.rst | 7 +++---- docs/index.rst | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 520a947..abcefb4 100644 --- a/README.rst +++ b/README.rst @@ -13,10 +13,9 @@ Redis client for the `PEP 3156`_ Python event loop. .. _PEP 3156: http://legacy.python.org/dev/peps/pep-3156/ -This Redis library is a completely asynchronous, non-blocking client for a -Redis server. It depends on asyncio (PEP 3156) and therefor it requires Python -3.3 or greater. If you're new to asyncio, it can be helpful to check out -`the asyncio documentation`_ first. +This Redis library is a completely asynchronous, non-blocking client for a Redis server. +It depends on asyncio (PEP 3156) and requires Python 3.6 or greater. If you're new to +asyncio, it can be helpful to check out `the asyncio documentation`_ first. .. _the asyncio documentation: http://docs.python.org/dev/library/asyncio.html diff --git a/docs/index.rst b/docs/index.rst index c2613ae..fc5e5d8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,10 +11,9 @@ Asynchronous Redis client for Python. .. _PEP 3156: http://legacy.python.org/dev/peps/pep-3156/ .. _GitHub: https://github.com/jonathanslenders/asyncio-redis -This Redis library is a completely asynchronous, non-blocking client for a -Redis server. It depends on asyncio (PEP 3156) and therefor it requires Python -3.3 or 3.4. If you're new to asyncio, it can be helpful to check out -`the asyncio documentation`_ first. +This Redis library is a completely asynchronous, non-blocking client for a Redis server. +It depends on asyncio (PEP 3156) and requires Python 3.6 or greater. If you're new to +asyncio, it can be helpful to check out `the asyncio documentation`_ first. .. _the asyncio documentation: http://docs.python.org/dev/library/asyncio.html From ba45aeb3ec74c1b586f2216f3cffa705726d07b7 Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 15 Jan 2020 12:15:17 +0000 Subject: [PATCH 06/21] Drop legacy backports --- asyncio_redis/connection.py | 9 +-------- asyncio_redis/exceptions.py | 2 +- asyncio_redis/protocol.py | 27 ++++++++++----------------- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/asyncio_redis/connection.py b/asyncio_redis/connection.py index eb944e2..9baa2fd 100644 --- a/asyncio_redis/connection.py +++ b/asyncio_redis/connection.py @@ -7,13 +7,6 @@ __all__ = ('Connection', ) -# In Python 3.4.4, `async` was renamed to `ensure_future`. -try: - ensure_future = asyncio.ensure_future -except AttributeError: - ensure_future = getattr(asyncio, "async") - - class Connection: """ Wrapper around the protocol and transport which takes care of establishing @@ -61,7 +54,7 @@ def create(cls, host='localhost', port=6379, *, password=None, db=0, # Create protocol instance def connection_lost(): if connection._auto_reconnect and not connection._closing: - ensure_future(connection._reconnect(), loop=connection._loop) + asyncio.ensure_future(connection._reconnect(), loop=connection._loop) # Create protocol instance connection.protocol = protocol_class(password=password, db=db, encoder=encoder, diff --git a/asyncio_redis/exceptions.py b/asyncio_redis/exceptions.py index 7117d1f..e4fb3f1 100644 --- a/asyncio_redis/exceptions.py +++ b/asyncio_redis/exceptions.py @@ -12,7 +12,7 @@ # See following link for the proper way to create user defined exceptions: -# http://docs.python.org/3.3/tutorial/errors.html#user-defined-exceptions +# http://docs.python.org/3.8/tutorial/errors.html#user-defined-exceptions class Error(Exception): diff --git a/asyncio_redis/protocol.py b/asyncio_redis/protocol.py index 152d568..ed041d4 100644 --- a/asyncio_redis/protocol.py +++ b/asyncio_redis/protocol.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import asyncio +import enum import logging import types @@ -58,11 +59,6 @@ NoneType = type(None) -# In Python 3.4.4, `async` was renamed to `ensure_future`. -try: - ensure_future = asyncio.ensure_future -except AttributeError: - ensure_future = getattr(asyncio, "async") class _NoTransactionType(object): """ @@ -71,8 +67,11 @@ class _NoTransactionType(object): from None. (None could be a valid input for a @_command, so there is no way to see whether this would be an extra 'transaction' value.) """ + + _NoTransaction = _NoTransactionType() + class ZScoreBoundary: """ Score boundary for a sorted set. @@ -96,17 +95,11 @@ def __repr__(self): ZScoreBoundary.MAX_VALUE = ZScoreBoundary('+inf') -class ZAggregate: # TODO: use the Python 3.4 enum type. +class ZAggregate(enum.Enum): + """Aggregation method for zinterstore and zunionstore """ - Aggregation method for zinterstore and zunionstore. - """ - #: Sum aggregation. SUM = 'SUM' - - #: Min aggregation. MIN = 'MIN' - - #: Max aggregation. MAX = 'MAX' @@ -653,7 +646,7 @@ def done(result): typecheck_return(protocol_self, result) future2.set_result(result) - future.add_done_callback(lambda f: ensure_future(done(f.result()), loop=protocol_self._loop)) + future.add_done_callback(lambda f: asyncio.ensure_future(done(f.result()), loop=protocol_self._loop)) return future2 @@ -825,7 +818,7 @@ def connection_made(self, transport): # Start parsing reader stream. self._reader = StreamReader(loop=self._loop) self._reader.set_transport(transport) - self._reader_f = ensure_future(self._reader_coroutine(), loop=self._loop) + self._reader_f = asyncio.ensure_future(self._reader_coroutine(), loop=self._loop) @asyncio.coroutine def initialize(): @@ -844,7 +837,7 @@ def initialize(): if self._pubsub_patterns: yield from self._psubscribe(self._subscription, list(self._pubsub_patterns)) - ensure_future(initialize(), loop=self._loop) + asyncio.ensure_future(initialize(), loop=self._loop) def data_received(self, data): """ Process data received from Redis server. """ @@ -999,7 +992,7 @@ def _handle_multi_bulk_reply(self, cb): # Return the empty queue immediately as an answer. if self._in_pubsub: - ensure_future(self._handle_pubsub_multibulk_reply(reply), loop=self._loop) + asyncio.ensure_future(self._handle_pubsub_multibulk_reply(reply), loop=self._loop) else: cb(reply) From ff0b1956bd12a37a9b936b59aeb0dbc9966ca81b Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 15 Jan 2020 12:21:17 +0000 Subject: [PATCH 07/21] __init__ cleanup --- asyncio_redis/__init__.py | 55 +++++++++++++++++++++++++++++++++---- asyncio_redis/connection.py | 3 -- asyncio_redis/exceptions.py | 13 --------- asyncio_redis/pool.py | 3 -- asyncio_redis/protocol.py | 10 ------- tests.py | 3 +- 6 files changed, 51 insertions(+), 36 deletions(-) diff --git a/asyncio_redis/__init__.py b/asyncio_redis/__init__.py index 1f46792..9682542 100644 --- a/asyncio_redis/__init__.py +++ b/asyncio_redis/__init__.py @@ -1,7 +1,50 @@ +"""Redis protocol implementation for asyncio (PEP 3156) """ -Redis protocol implementation for asyncio (PEP 3156) -""" -from .connection import * -from .exceptions import * -from .pool import * -from .protocol import * +from .connection import Connection +from .exceptions import ( + ConnectionLostError, + Error, + ErrorReply, + NoAvailableConnectionsInPoolError, + NoRunningScriptError, + NotConnectedError, + ScriptKilledError, + TimeoutError, + TransactionError, +) +from .pool import Pool +from .protocol import ( + RedisProtocol, + HiRedisProtocol, + Transaction, + Subscription, + Script, + ZAggregate, + ZScoreBoundary, +) + + +__all__ = ( + 'Connection', + 'Pool', + + # Protocols + 'RedisProtocol', + 'HiRedisProtocol', + 'Transaction', + 'Subscription', + 'Script', + 'ZAggregate', + 'ZScoreBoundary', + + # Exceptions + 'ConnectionLostError', + 'Error', + 'ErrorReply', + 'NoAvailableConnectionsInPoolError', + 'NoRunningScriptError', + 'NotConnectedError', + 'ScriptKilledError', + 'TimeoutError', + 'TransactionError', +) diff --git a/asyncio_redis/connection.py b/asyncio_redis/connection.py index 9baa2fd..f22bb51 100644 --- a/asyncio_redis/connection.py +++ b/asyncio_redis/connection.py @@ -4,9 +4,6 @@ import logging -__all__ = ('Connection', ) - - class Connection: """ Wrapper around the protocol and transport which takes care of establishing diff --git a/asyncio_redis/exceptions.py b/asyncio_redis/exceptions.py index e4fb3f1..bb785aa 100644 --- a/asyncio_redis/exceptions.py +++ b/asyncio_redis/exceptions.py @@ -1,16 +1,3 @@ -__all__ = ( - 'ConnectionLostError', - 'Error', - 'ErrorReply', - 'NoAvailableConnectionsInPoolError', - 'NoRunningScriptError', - 'NotConnectedError', - 'ScriptKilledError', - 'TimeoutError', - 'TransactionError', -) - - # See following link for the proper way to create user defined exceptions: # http://docs.python.org/3.8/tutorial/errors.html#user-defined-exceptions diff --git a/asyncio_redis/pool.py b/asyncio_redis/pool.py index 82188b1..ec1f7b8 100644 --- a/asyncio_redis/pool.py +++ b/asyncio_redis/pool.py @@ -6,9 +6,6 @@ import asyncio -__all__ = ('Pool', ) - - class Pool: """ Pool of connections. Each diff --git a/asyncio_redis/protocol.py b/asyncio_redis/protocol.py index ed041d4..a20af9d 100644 --- a/asyncio_redis/protocol.py +++ b/asyncio_redis/protocol.py @@ -46,16 +46,6 @@ from .cursors import Cursor, SetCursor, DictCursor, ZCursor -__all__ = ( - 'RedisProtocol', - 'HiRedisProtocol', - 'Transaction', - 'Subscription', - 'Script', - - 'ZAggregate', - 'ZScoreBoundary', -) NoneType = type(None) diff --git a/tests.py b/tests.py index cd11ff3..f766e7b 100755 --- a/tests.py +++ b/tests.py @@ -12,6 +12,7 @@ from asyncio_redis import ( Connection, + ConnectionLostError, Error, ErrorReply, HiRedisProtocol, @@ -23,6 +24,7 @@ Script, ScriptKilledError, Subscription, + TimeoutError, Transaction, TransactionError, ZScoreBoundary, @@ -40,7 +42,6 @@ StatusReply, ZRangeReply, ) -from asyncio_redis.exceptions import TimeoutError, ConnectionLostError from asyncio_redis.cursors import Cursor from asyncio_redis.encoders import BytesEncoder From 1e3e20a1fbe1ee142d35900e8a2362360a4ec614 Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 15 Jan 2020 15:03:07 +0000 Subject: [PATCH 08/21] Clean up modules (all except protocol) --- asyncio_redis/connection.py | 69 +++++++++++++++------------- asyncio_redis/cursors.py | 37 ++++++--------- asyncio_redis/exceptions.py | 1 + asyncio_redis/pool.py | 89 +++++++++++++++++++++---------------- asyncio_redis/replies.py | 40 +++++++---------- tests.py | 18 ++++---- 6 files changed, 127 insertions(+), 127 deletions(-) diff --git a/asyncio_redis/connection.py b/asyncio_redis/connection.py index f22bb51..40182e3 100644 --- a/asyncio_redis/connection.py +++ b/asyncio_redis/connection.py @@ -2,6 +2,7 @@ from .protocol import RedisProtocol, _all_commands import asyncio import logging +import warnings class Connection: @@ -11,30 +12,33 @@ class Connection: :: - - connection = yield from Connection.create(host='localhost', port=6379) - result = yield from connection.set('key', 'value') + connection = await Connection.create(host='localhost', port=6379) + result = await connection.set('key', 'value') """ @classmethod - @asyncio.coroutine - def create(cls, host='localhost', port=6379, *, password=None, db=0, + async def create(cls, host='localhost', port=6379, *, password=None, db=0, encoder=None, auto_reconnect=True, loop=None, protocol_class=RedisProtocol): """ - :param host: Address, either host or unix domain socket path - :type host: str - :param port: TCP port. If port is 0 then host assumed to be unix socket path - :type port: int - :param password: Redis database password - :type password: bytes - :param db: Redis database - :type db: int - :param encoder: Encoder to use for encoding to or decoding from redis bytes to a native type. - :type encoder: :class:`~asyncio_redis.encoders.BaseEncoder` instance. - :param auto_reconnect: Enable auto reconnect - :type auto_reconnect: bool - :param loop: (optional) asyncio event loop. - :type protocol_class: :class:`~asyncio_redis.RedisProtocol` - :param protocol_class: (optional) redis protocol implementation + :param str host: + Address, either host or unix domain socket path + :param int port: + TCP port. If port is 0 then host assumed to be unix socket path + :param bytes password: + Redis database password + :param int db: + Redis database + :param encoder: + Encoder to use for encoding to or decoding from redis bytes to a native type. + :type encoder: + :class:`~asyncio_redis.encoders.BaseEncoder` + :param bool auto_reconnect: + Enable auto reconnect + :param loop: + (optional) asyncio event loop. + :param protocol_class: + (optional) redis protocol implementation + :type protocol_class: + :class:`~asyncio_redis.RedisProtocol` """ assert port >= 0, "Unexpected port value: %r" % (port, ) connection = cls() @@ -51,7 +55,7 @@ def create(cls, host='localhost', port=6379, *, password=None, db=0, # Create protocol instance def connection_lost(): if connection._auto_reconnect and not connection._closing: - asyncio.ensure_future(connection._reconnect(), loop=connection._loop) + connection._loop.create_task(connection._reconnect()) # Create protocol instance connection.protocol = protocol_class(password=password, db=db, encoder=encoder, @@ -59,9 +63,9 @@ def connection_lost(): # Connect if connection._auto_reconnect: - yield from connection._reconnect() + await connection._reconnect() else: - yield from connection._connect() + await connection._connect() return connection @@ -82,33 +86,34 @@ def _increase_retry_interval(self): """ When a connection failed. Increase the interval.""" self._retry_interval = min(60, 1.5 * self._retry_interval) - @asyncio.coroutine - def _connect(self): + async def _connect(self): """ Set up Redis connection. """ logger.log(logging.INFO, 'Connecting to redis') if self.port: - yield from self._loop.create_connection(lambda: self.protocol, self.host, self.port) + await self._loop.create_connection(lambda: self.protocol, self.host, self.port) else: - yield from self._loop.create_unix_connection(lambda: self.protocol, self.host) + await self._loop.create_unix_connection(lambda: self.protocol, self.host) - @asyncio.coroutine - def _reconnect(self): + async def _reconnect(self): """ Set up Redis re-connection. """ while True: try: - yield from self._connect() + await self._connect() self._reset_retry_interval() return except OSError: # Sleep and try again self._increase_retry_interval() interval = self._get_retry_interval() - logger.log(logging.INFO, 'Connecting to redis failed. Retrying in %i seconds' % interval) - yield from asyncio.sleep(interval, loop=self._loop) + logger.log( + logging.INFO, + f'Connecting to redis failed. Retrying in {interval} seconds', + ) + await asyncio.sleep(interval) def __getattr__(self, name): # Only proxy commands. diff --git a/asyncio_redis/cursors.py b/asyncio_redis/cursors.py index 15c9d57..b6e7bd7 100644 --- a/asyncio_redis/cursors.py +++ b/asyncio_redis/cursors.py @@ -1,4 +1,3 @@ -import asyncio from collections import deque __all__ = ( @@ -30,21 +29,17 @@ def __init__(self, name, scanfunc): def __repr__(self): return '<%s %s>' % (self.__class__.__name__, self._name) - @asyncio.coroutine - def _fetch_more(self): + async def _fetch_more(self): """ Get next chunk of keys from Redis """ if not self._done: - chunk = yield from self._scanfunc(self._cursor, self.count) + chunk = await self._scanfunc(self._cursor, self.count) self._cursor = chunk.new_cursor_pos - - if chunk.new_cursor_pos == 0: - self._done = True + self._done = chunk.new_cursor_pos == 0 for i in chunk.items: self._queue.append(i) - @asyncio.coroutine - def fetchone(self): + async def fetchone(self): """ Coroutines that returns the next item. It returns `None` after the last item. @@ -54,19 +49,18 @@ def fetchone(self): # Redis can return a chunk of zero items, even when we're not yet finished. # See: https://github.com/jonathanslenders/asyncio-redis/issues/65#issuecomment-127026408 while not self._queue and not self._done: - yield from self._fetch_more() + await self._fetch_more() # Return the next item. if self._queue: return self._queue.popleft() - @asyncio.coroutine - def fetchall(self): + async def fetchall(self): """ Coroutine that reads all the items in one list. """ results = [] while not self._done: - yield from self._fetch_more() + await self._fetch_more() results.extend(self._queue) self._queue.clear() @@ -78,9 +72,8 @@ class SetCursor(Cursor): Cursor for walking through the results of a :func:`sscan ` query. """ - @asyncio.coroutine - def fetchall(self): - result = yield from super().fetchall() + async def fetchall(self): + result = await super().fetchall() return set(result) @@ -92,26 +85,24 @@ class DictCursor(Cursor): def _parse(self, key, value): return key, value - @asyncio.coroutine - def fetchone(self): + async def fetchone(self): """ Get next { key: value } tuple It returns `None` after the last item. """ - key = yield from super().fetchone() - value = yield from super().fetchone() + key = await super().fetchone() + value = await super().fetchone() if key is not None: key, value = self._parse(key, value) return { key: value } - @asyncio.coroutine - def fetchall(self): + async def fetchall(self): """ Coroutine that reads all the items in one dictionary. """ results = {} while True: - i = yield from self.fetchone() + i = await self.fetchone() if i is None: break else: diff --git a/asyncio_redis/exceptions.py b/asyncio_redis/exceptions.py index bb785aa..527c20f 100644 --- a/asyncio_redis/exceptions.py +++ b/asyncio_redis/exceptions.py @@ -38,6 +38,7 @@ class NoAvailableConnectionsInPoolError(NotConnectedError): When the connection pool has no available connections. """ + class ScriptKilledError(Error): """ Script was killed during an evalsha call. """ diff --git a/asyncio_redis/pool.py b/asyncio_redis/pool.py index ec1f7b8..b13d670 100644 --- a/asyncio_redis/pool.py +++ b/asyncio_redis/pool.py @@ -16,34 +16,38 @@ class Pool: :: - connection = yield from Pool.create(host='localhost', port=6379, poolsize=10) - result = yield from connection.set('key', 'value') + connection = await Pool.create(host='localhost', port=6379, poolsize=10) + result = await connection.set('key', 'value') """ @classmethod - @asyncio.coroutine - def create(cls, host='localhost', port=6379, *, password=None, db=0, + async def create(cls, host='localhost', port=6379, *, password=None, db=0, encoder=None, poolsize=1, auto_reconnect=True, loop=None, protocol_class=RedisProtocol): """ Create a new connection pool instance. - :param host: Address, either host or unix domain socket path - :type host: str - :param port: TCP port. If port is 0 then host assumed to be unix socket path - :type port: int - :param password: Redis database password - :type password: bytes - :param db: Redis database - :type db: int - :param encoder: Encoder to use for encoding to or decoding from redis bytes to a native type. - :type encoder: :class:`~asyncio_redis.encoders.BaseEncoder` instance. - :param poolsize: The number of parallel connections. - :type poolsize: int - :param auto_reconnect: Enable auto reconnect - :type auto_reconnect: bool - :param loop: (optional) asyncio event loop. - :type protocol_class: :class:`~asyncio_redis.RedisProtocol` - :param protocol_class: (optional) redis protocol implementation + :param str host: + Address, either host or unix domain socket path + :param int port: + TCP port. If port is 0 then host assumed to be unix socket path + :param bytes password: + Redis database password + :param int db: + Redis database + :param encoder: + Encoder to use for encoding to or decoding from redis bytes to a native type + :type encoder: + :class:`~asyncio_redis.encoders.BaseEncoder` + :param int poolsize: + The number of parallel connections. + :param bool auto_reconnect: + Enable auto reconnect + :param loop: + (optional) asyncio event loop + :param protocol_class: + (optional) redis protocol implementation + :type protocol_class: + :class:`~asyncio_redis.RedisProtocol` """ self = cls() self._host = host @@ -51,19 +55,25 @@ def create(cls, host='localhost', port=6379, *, password=None, db=0, self._poolsize = poolsize # Create connections - self._connections = [] - - for i in range(poolsize): - connection = yield from Connection.create(host=host, port=port, - password=password, db=db, encoder=encoder, - auto_reconnect=auto_reconnect, loop=loop, - protocol_class=protocol_class) - self._connections.append(connection) + conn_coros = [ + Connection.create( + host=host, + port=port, + password=password, + db=db, + encoder=encoder, + auto_reconnect=auto_reconnect, + loop=loop, + protocol_class=protocol_class, + ) + for _ in range(poolsize) + ] + self._connections = list(await asyncio.gather(*conn_coros)) return self def __repr__(self): - return 'Pool(host=%r, port=%r, poolsize=%r)' % (self._host, self._port, self._poolsize) + return f"Pool(host='{self._host}', port={self._port}, poolsize={self._poolsize})" @property def poolsize(self): @@ -75,14 +85,14 @@ def connections_in_use(self): """ Return how many protocols are in use. """ - return sum([ 1 for c in self._connections if c.protocol.in_use ]) + return sum(int(c.protocol.in_use) for c in self._connections) @property def connections_connected(self): """ The amount of open TCP connections. """ - return sum([ 1 for c in self._connections if c.protocol.is_connected ]) + return sum(int(c.protocol.is_connected) for c in self._connections) def _get_free_connection(self): """ @@ -98,7 +108,8 @@ def _get_free_connection(self): def _shuffle_connections(self): """ - 'shuffle' protocols. Make sure that we devide the load equally among the protocols. + 'shuffle' protocols. Make sure that we divide the load equally among the + protocols. """ self._connections = self._connections[1:] + self._connections[:1] @@ -111,18 +122,18 @@ def __getattr__(self, name): if connection: return getattr(connection, name) - else: - raise NoAvailableConnectionsInPoolError('No available connections in the pool: size=%s, in_use=%s, connected=%s' % ( - self.poolsize, self.connections_in_use, self.connections_connected)) + raise NoAvailableConnectionsInPoolError( + f'No available connections in the pool: size={self.poolsize}, ' + f'in_use={self.connections_in_use}, connected={self.connections_connected}' + ) # Proxy the register_script method, so that the returned object will # execute on any available connection in the pool. - @asyncio.coroutine @wraps(RedisProtocol.register_script) - def register_script(self, script:str) -> Script: + async def register_script(self, script:str) -> Script: # Call register_script from the Protocol. - script = yield from self.__getattr__('register_script')(script) + script = await self.__getattr__('register_script')(script) assert isinstance(script, Script) # Return a new script instead that runs it on any connection of the pool. diff --git a/asyncio_redis/replies.py b/asyncio_redis/replies.py index e66eb24..02bad11 100644 --- a/asyncio_redis/replies.py +++ b/asyncio_redis/replies.py @@ -45,7 +45,7 @@ class DictReply: :: for f in dict_reply: - key, value = yield from f + key, value = await f print(key, value) """ def __init__(self, multibulk_reply): @@ -57,24 +57,22 @@ def _parse(self, key, value): def __iter__(self): """Yield a list of futures that yield { key: value } tuples """ - @asyncio.coroutine - def getter(f): + async def getter(f): """Coroutine which processes one item """ - key, value = yield from f + key, value = await f key, value = self._parse(key, value) return key, value for _ in range(self._result.count // 2): read_future = self._result._read(count=2) - yield ensure_future(getter(read_future), loop=self._result._loop) + yield ensure_future(getter(read_future)) - @asyncio.coroutine - def asdict(self): + async def asdict(self): """ Return the result as a Python dictionary. """ - data = yield from self._result._read(count=self._result.count) + data = await self._result._read(count=self._result.count) return dict(self._parse(k, v) for k, v in zip(data[::2], data[1::2])) def __repr__(self): @@ -99,7 +97,7 @@ class SetReply: :: for f in set_reply: - item = yield from f + item = await f print(item) """ def __init__(self, multibulk_reply): @@ -109,10 +107,9 @@ def __iter__(self): """ Yield a list of futures. """ return iter(self._result) - @asyncio.coroutine - def asset(self): + async def asset(self): """ Return the result as a Python ``set``. """ - data = yield from self._result._read(count=self._result.count) + data = await self._result._read(count=self._result.count) return set(data) def __repr__(self): @@ -129,7 +126,7 @@ class ListReply: :: for f in list_reply: - item = yield from f + item = await f print(item) """ def __init__(self, multibulk_reply): @@ -139,11 +136,9 @@ def __iter__(self): """ Yield a list of futures. """ return iter(self._result) - @asyncio.coroutine def aslist(self): - """ Return the result as a Python ``list``. """ - data = yield from self._result._read(count=self._result.count) - return data + """ Return a coroutine with the result as a Python ``list``. """ + return self._result._read(count=self._result.count) def __repr__(self): return 'ListReply(length=%r)' % (self._result.count, ) @@ -246,7 +241,6 @@ def __init__(self, protocol, value): self._protocol = protocol self._value = value - @asyncio.coroutine def return_value(self): """ Coroutine that returns a Python representation of the script's return @@ -254,8 +248,7 @@ def return_value(self): """ from asyncio_redis.protocol import MultiBulkReply - @asyncio.coroutine - def decode(obj): + async def decode(obj): if isinstance(obj, int): return obj @@ -266,13 +259,12 @@ def decode(obj): # Unpack MultiBulkReply recursively as Python list. result = [] for f in obj: - item = yield from f - result.append((yield from decode(item))) + item = await f + result.append((await decode(item))) return result else: # Nonetype, or decoded bytes. return obj - return (yield from decode(self._value)) - + return decode(self._value) diff --git a/tests.py b/tests.py index f766e7b..3065860 100755 --- a/tests.py +++ b/tests.py @@ -54,9 +54,9 @@ async def connect(protocol=RedisProtocol): """ Connect to redis server. Return transport/protocol pair. """ loop = asyncio.get_event_loop() if PORT: - return await loop.create_connection(lambda: protocol(loop=loop), HOST, PORT) + return await loop.create_connection(lambda: protocol(), HOST, PORT) else: - return await loop.create_unix_connection(lambda: protocol(loop=loop), HOST) + return await loop.create_unix_connection(lambda: protocol(), HOST) def redis_test(function): @@ -827,7 +827,7 @@ async def listener(): return transport2 - f = asyncio.ensure_future(listener(), loop=self.loop) + f = asyncio.ensure_future(listener()) async def sender(): value = await protocol.publish(u'our_channel', 'message1') @@ -863,7 +863,7 @@ async def sender(): result = await protocol.pubsub_numpat() self.assertIsInstance(result, int) - await asyncio.sleep(.5, loop=self.loop) + await asyncio.sleep(.5) await sender() transport2 = await f transport2.close() @@ -1425,8 +1425,8 @@ async def run_while_true(): transport.close() # (start script) - f = asyncio.ensure_future(run_while_true(), loop=self.loop) - await asyncio.sleep(.5, loop=self.loop) + f = asyncio.ensure_future(run_while_true()) + await asyncio.sleep(.5) result = await protocol.script_kill() self.assertEqual(result, StatusReply('OK')) @@ -1493,7 +1493,7 @@ async def test_run_command_outside_transaction(self, transport, protocol): # Run command, but not as part of the transaction. # This should wait until the transaction finishes. - f = asyncio.ensure_future(protocol.set('a', 'b'), loop=self.loop) + f = asyncio.ensure_future(protocol.set('a', 'b')) # Close transaction. await transaction.exec() @@ -1892,8 +1892,8 @@ async def listener(): async def sender(): await protocol.publish(b'our_channel', b'message1') - f = asyncio.ensure_future(listener(), loop=self.loop) - await asyncio.sleep(.5, loop=self.loop) + f = asyncio.ensure_future(listener()) + await asyncio.sleep(.5) await sender() transport2 = await f transport2.close() From 9bbfa59feda2718bec5ac252054666d33b9f4395 Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 15 Jan 2020 15:06:00 +0000 Subject: [PATCH 09/21] Reinstate NoGlobalLoopTest --- tests.py | 48 ++++++++++++++++++------------------------------ 1 file changed, 18 insertions(+), 30 deletions(-) diff --git a/tests.py b/tests.py index 3065860..ef489d1 100755 --- a/tests.py +++ b/tests.py @@ -2202,49 +2202,37 @@ class NoGlobalLoopTest(TestCase): If we set the global loop variable to None, everything should still work. """ def test_no_global_loop(self): + old_loop = asyncio.get_event_loop() + try: + # Remove global loop and create a new one. + asyncio.set_event_loop(None) + new_loop = asyncio.new_event_loop() - async def test(): - connection = await Connection.create(host=HOST, port=PORT) - self.assertIsInstance(connection, Connection) - try: - # Delete keys - await connection.delete(['key1', 'key2']) - - # Get/set - await connection.set('key1', 'value') - result = await connection.get('key1') - self.assertEqual(result, 'value') + # ** Run code on the new loop. ** - # hmset/hmget (something that uses a MultiBulkReply) - await connection.hmset('key2', {'a': 'b', 'c': 'd'}) - result = await connection.hgetall_asdict('key2') - self.assertEqual(result, {'a': 'b', 'c': 'd'}) + # Create connection + connection = new_loop.run_until_complete(Connection.create(host=HOST, port=PORT, loop=new_loop)) + self.assertIsInstance(connection, Connection) + try: # Delete keys - await connection.delete(['key1', 'key2']) + new_loop.run_until_complete(connection.delete(['key1', 'key2'])) # Get/set - await connection.set('key1', 'value') - result = await connection.get('key1') + new_loop.run_until_complete(connection.set('key1', 'value')) + result = new_loop.run_until_complete(connection.get('key1')) self.assertEqual(result, 'value') # hmset/hmget (something that uses a MultiBulkReply) - await connection.hmset('key2', { 'a': 'b', 'c': 'd' }) - result = await connection.hgetall_asdict('key2') + new_loop.run_until_complete(connection.hmset('key2', { 'a': 'b', 'c': 'd' })) + result = new_loop.run_until_complete(connection.hgetall_asdict('key2')) self.assertEqual(result, { 'a': 'b', 'c': 'd' }) - finally: connection.close() - # Run loop briefly until socket has been closed. - await asyncio.sleep(0.1) - - # Remove global loop and create a new one. - old_loop = asyncio.get_event_loop() - asyncio.set_event_loop(None) - new_loop = asyncio.new_event_loop() - try: - new_loop.run_until_complete(test()) finally: + # Run loop briefly until socket has been closed. (call_soon behind the scenes.) + new_loop.run_until_complete(asyncio.sleep(.1)) + new_loop.close() asyncio.set_event_loop(old_loop) From 8a95ee4bb9375b4d66034eed09a8894575e9c989 Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 15 Jan 2020 15:50:55 +0000 Subject: [PATCH 10/21] protocol (failing) --- asyncio_redis/protocol.py | 361 ++++++++++++++++---------------------- 1 file changed, 148 insertions(+), 213 deletions(-) diff --git a/asyncio_redis/protocol.py b/asyncio_redis/protocol.py index a20af9d..da143ef 100644 --- a/asyncio_redis/protocol.py +++ b/asyncio_redis/protocol.py @@ -4,10 +4,6 @@ import logging import types -from asyncio.futures import Future -from asyncio.queues import Queue -from asyncio.streams import StreamReader - try: import hiredis except ImportError: @@ -106,9 +102,7 @@ class MultiBulkReply: """ Container for a multi bulk reply. """ - def __init__(self, protocol, count, loop=None): - self._loop = loop or asyncio.get_event_loop() - + def __init__(self, protocol, count): #: Buffer of incoming, undelivered data, received from the parser. self._data_queue = [] @@ -165,7 +159,7 @@ def _decode(self, result): def _read(self, decode=True, count=1, _one=False): """ Do read operation on the queue. Return future. """ - f = Future(loop=self.protocol._loop) + f = asyncio.Future() self._f_queue.append((count, f, decode, _one)) # If there is enough data on the queue, answer future immediately. @@ -262,72 +256,61 @@ def get_alternate_post_processor(cls, return_type): original_post_processor = cls.get_default(return_type) if return_type == ListReply: - @asyncio.coroutine - def as_list(protocol, result): - result = yield from original_post_processor(protocol, result) - return (yield from result.aslist()) + async def as_list(protocol, result): + result = await original_post_processor(protocol, result) + return await result.aslist() return '_aslist', list, as_list elif return_type == SetReply: - @asyncio.coroutine - def as_set(protocol, result): - result = yield from original_post_processor(protocol, result) - return (yield from result.asset()) + async def as_set(protocol, result): + result = await original_post_processor(protocol, result) + return await result.asset() return '_asset', set, as_set elif return_type in (DictReply, ZRangeReply): - @asyncio.coroutine - def as_dict(protocol, result): - result = yield from original_post_processor(protocol, result) - return (yield from result.asdict()) + async def as_dict(protocol, result): + result = await original_post_processor(protocol, result) + return await result.asdict() return '_asdict', dict, as_dict # === Post processor handlers below. === - @asyncio.coroutine - def multibulk_as_list(protocol, result): + async def multibulk_as_list(protocol, result): assert isinstance(result, MultiBulkReply) return ListReply(result) - @asyncio.coroutine - def multibulk_as_boolean_list(protocol, result): + async def multibulk_as_boolean_list(protocol, result): # Turn the array of integers into booleans. assert isinstance(result, MultiBulkReply) - values = yield from ListReply(result).aslist() + values = await ListReply(result).aslist() return [ bool(v) for v in values ] - @asyncio.coroutine - def multibulk_as_set(protocol, result): + async def multibulk_as_set(protocol, result): assert isinstance(result, MultiBulkReply) return SetReply(result) - @asyncio.coroutine - def multibulk_as_dict(protocol, result): + async def multibulk_as_dict(protocol, result): assert isinstance(result, MultiBulkReply) return DictReply(result) - @asyncio.coroutine - def multibulk_as_zrangereply(protocol, result): + async def multibulk_as_zrangereply(protocol, result): assert isinstance(result, MultiBulkReply) return ZRangeReply(result) - @asyncio.coroutine - def multibulk_as_blocking_pop_reply(protocol, result): + async def multibulk_as_blocking_pop_reply(protocol, result): if result is None: raise TimeoutError('Timeout in blocking pop') else: assert isinstance(result, MultiBulkReply) - list_name, value = yield from ListReply(result).aslist() + list_name, value = await ListReply(result).aslist() return BlockingPopReply(list_name, value) - @asyncio.coroutine - def multibulk_as_configpair(protocol, result): + async def multibulk_as_configpair(protocol, result): assert isinstance(result, MultiBulkReply) - parameter, value = yield from ListReply(result).aslist() + parameter, value = await ListReply(result).aslist() return ConfigPairReply(parameter, value) - @asyncio.coroutine - def multibulk_as_scanpart(protocol, result): + async def multibulk_as_scanpart(protocol, result): """ Process scanpart result. This is a multibulk reply of length two, where the first item is the @@ -336,72 +319,61 @@ def multibulk_as_scanpart(protocol, result): """ # Get outer multi bulk reply. assert isinstance(result, MultiBulkReply) - new_cursor_pos, items_bulk = yield from ListReply(result).aslist() + new_cursor_pos, items_bulk = await ListReply(result).aslist() assert isinstance(items_bulk, MultiBulkReply) # Read all items for scan chunk in memory. This is fine, because it's # transmitted in chunks of about 10. - items = yield from ListReply(items_bulk).aslist() + items = await ListReply(items_bulk).aslist() return _ScanPart(int(new_cursor_pos), items) - @asyncio.coroutine - def bytes_to_info(protocol, result): + async def bytes_to_info(protocol, result): assert isinstance(result, bytes) return InfoReply(result) - @asyncio.coroutine - def bytes_to_status_reply(protocol, result): + async def bytes_to_status_reply(protocol, result): assert isinstance(result, bytes) return StatusReply(result.decode('utf-8')) - @asyncio.coroutine - def bytes_to_status_reply_or_none(protocol, result): + async def bytes_to_status_reply_or_none(protocol, result): assert isinstance(result, (bytes, NoneType)) if result: return StatusReply(result.decode('utf-8')) - @asyncio.coroutine - def bytes_to_clientlist(protocol, result): + async def bytes_to_clientlist(protocol, result): assert isinstance(result, bytes) return ClientListReply(result) - @asyncio.coroutine - def int_to_bool(protocol, result): + async def int_to_bool(protocol, result): assert isinstance(result, int) return bool(result) # Convert int to bool - @asyncio.coroutine - def bytes_to_native(protocol, result): + async def bytes_to_native(protocol, result): assert isinstance(result, bytes) return protocol.decode_to_native(result) - @asyncio.coroutine - def bytes_to_str(protocol, result): + async def bytes_to_str(protocol, result): assert isinstance(result, bytes) return result.decode('ascii') - @asyncio.coroutine - def bytes_to_native_or_none(protocol, result): + async def bytes_to_native_or_none(protocol, result): if result is None: return result else: assert isinstance(result, bytes) return protocol.decode_to_native(result) - @asyncio.coroutine - def bytes_to_float_or_none(protocol, result): + async def bytes_to_float_or_none(protocol, result): if result is None: return result assert isinstance(result, bytes) return float(result) - @asyncio.coroutine - def bytes_to_float(protocol, result): + async def bytes_to_float(protocol, result): assert isinstance(result, bytes) return float(result) - @asyncio.coroutine - def any_to_evalscript(protocol, result): + async def any_to_evalscript(protocol, result): # Result can be native, int, MultiBulkReply or even a nested structure assert isinstance(result, (int, bytes, MultiBulkReply, NoneType)) return EvalScriptReply(protocol, result) @@ -612,8 +584,7 @@ def _get_wrapped_method(self, post_process, suffix, return_type): # directly on the protocol, outside of transactions or from the # transaction object. @wraps(method) - @asyncio.coroutine - def wrapper(protocol_self, *a, **kw): + async def wrapper(protocol_self, *a, **kw): if a and isinstance(a[0], (Transaction, _NoTransactionType)): transaction = a[0] a = a[1:] @@ -624,19 +595,18 @@ def wrapper(protocol_self, *a, **kw): if transaction != _NoTransaction: # In case of a transaction, we receive a Future from the command. typecheck_input(protocol_self, *a, **kw) - future = yield from method(protocol_self, transaction, *a, **kw) - future2 = Future(loop=protocol_self._loop) + future = await method(protocol_self, transaction, *a, **kw) + future2 = asyncio.Future() # Typecheck the future when the result is available. - @asyncio.coroutine - def done(result): + async def done(result): if post_process: - result = yield from post_process(protocol_self, result) + result = await post_process(protocol_self, result) typecheck_return(protocol_self, result) future2.set_result(result) - future.add_done_callback(lambda f: asyncio.ensure_future(done(f.result()), loop=protocol_self._loop)) + future.add_done_callback(lambda f: protocol_self._loop.create_task(done(f.result()))) return future2 @@ -647,17 +617,17 @@ def done(result): else: typecheck_input(protocol_self, *a[1:], **kw) - result = yield from method(protocol_self, _NoTransaction, *a[1:], **kw) + result = await method(protocol_self, _NoTransaction, *a[1:], **kw) if post_process: - result = yield from post_process(protocol_self, result) + result = await post_process(protocol_self, result) typecheck_return(protocol_self, result) return (result) else: typecheck_input(protocol_self, *a, **kw) - result = yield from method(protocol_self, _NoTransaction, *a, **kw) + result = await method(protocol_self, _NoTransaction, *a, **kw) if post_process: - result = yield from post_process(protocol_self, result) + result = await post_process(protocol_self, result) typecheck_return(protocol_self, result) return result @@ -739,7 +709,7 @@ class RedisProtocol(asyncio.Protocol, metaclass=_RedisProtocolMeta): :: self.loop = asyncio.get_event_loop() - transport, protocol = yield from loop.create_connection(RedisProtocol, 'localhost', 6379) + transport, protocol = await loop.create_connection(RedisProtocol, 'localhost', 6379) :param password: Redis database password :type password: Native Python type as defined by the ``encoder`` parameter @@ -785,7 +755,7 @@ def __init__(self, *, password=None, db=0, encoder=None, connection_lost_callbac self._pubsub_patterns = set() # Set of patterns # Transaction related stuff. - self._transaction_lock = asyncio.Lock(loop=loop) + self._transaction_lock = asyncio.Lock() self._transaction = None self._transaction_response_queue = None # Transaction answer queue @@ -806,28 +776,27 @@ def connection_made(self, transport): self._pipelined_calls = set() # Set of all the pipelined calls. # Start parsing reader stream. - self._reader = StreamReader(loop=self._loop) + self._reader = asyncio.StreamReader() self._reader.set_transport(transport) - self._reader_f = asyncio.ensure_future(self._reader_coroutine(), loop=self._loop) + self._reader_f = self._loop.create_task(self._reader_coroutine()) - @asyncio.coroutine - def initialize(): + async def initialize(): # If a password or database was been given, first connect to that one. if self.password: - yield from self.auth(self.password) + await self.auth(self.password) if self.db: - yield from self.select(self.db) + await self.select(self.db) # If we are in pubsub mode, send channel subscriptions again. if self._in_pubsub: if self._pubsub_channels: - yield from self._subscribe(self._subscription, list(self._pubsub_channels)) # TODO: unittest this + await self._subscribe(self._subscription, list(self._pubsub_channels)) # TODO: unittest this if self._pubsub_patterns: - yield from self._psubscribe(self._subscription, list(self._pubsub_patterns)) + await self._psubscribe(self._subscription, list(self._pubsub_patterns)) - asyncio.ensure_future(initialize(), loop=self._loop) + self._loop.create_task(initialize()) def data_received(self, data): """ Process data received from Redis server. """ @@ -913,64 +882,57 @@ def is_connected(self): # Handle replies - @asyncio.coroutine - def _reader_coroutine(self): + async def _reader_coroutine(self): """ Coroutine which reads input from the stream reader and processes it. """ while True: try: - yield from self._handle_item(self._push_answer) + await self._handle_item(self._push_answer) except ConnectionLostError: return except asyncio.IncompleteReadError: return - @asyncio.coroutine - def _handle_item(self, cb): - c = yield from self._reader.readexactly(1) + async def _handle_item(self, cb): + c = await self._reader.readexactly(1) if c: - yield from self._line_received_handlers[c](cb) + await self._line_received_handlers[c](cb) else: raise ConnectionLostError(None) - @asyncio.coroutine - def _handle_status_reply(self, cb): - line = (yield from self._reader.readline()).rstrip(b'\r\n') + async def _handle_status_reply(self, cb): + line = (await self._reader.readline()).rstrip(b'\r\n') cb(line) - @asyncio.coroutine - def _handle_int_reply(self, cb): - line = (yield from self._reader.readline()).rstrip(b'\r\n') + async def _handle_int_reply(self, cb): + line = (await self._reader.readline()).rstrip(b'\r\n') cb(int(line)) - @asyncio.coroutine - def _handle_error_reply(self, cb): - line = (yield from self._reader.readline()).rstrip(b'\r\n') + async def _handle_error_reply(self, cb): + line = (await self._reader.readline()).rstrip(b'\r\n') cb(ErrorReply(line.decode('ascii'))) - @asyncio.coroutine - def _handle_bulk_reply(self, cb): - length = int((yield from self._reader.readline()).rstrip(b'\r\n')) + async def _handle_bulk_reply(self, cb): + length = int((await self._reader.readline()).rstrip(b'\r\n')) if length == -1: # None bulk reply cb(None) else: # Read data - data = yield from self._reader.readexactly(length) + data = await self._reader.readexactly(length) cb(data) # Ignore trailing newline. - remaining = yield from self._reader.readline() + remaining = await self._reader.readline() assert remaining.rstrip(b'\r\n') == b'' - @asyncio.coroutine - def _handle_multi_bulk_reply(self, cb): + async def _handle_multi_bulk_reply(self, cb): # NOTE: the reason for passing the callback `cb` in here is # mainly because we want to return the result object # especially in this case before the input is read # completely. This allows a streaming API. - count = int((yield from self._reader.readline()).rstrip(b'\r\n')) + count = int((await self._reader.readline()).rstrip(b'\r\n')) # Handle multi-bulk none. # (Used when a transaction exec fails.) @@ -978,31 +940,30 @@ def _handle_multi_bulk_reply(self, cb): cb(None) return - reply = MultiBulkReply(self, count, loop=self._loop) + reply = MultiBulkReply(self, count) # Return the empty queue immediately as an answer. if self._in_pubsub: - asyncio.ensure_future(self._handle_pubsub_multibulk_reply(reply), loop=self._loop) + self._loop.create_task(self._handle_pubsub_multibulk_reply(reply)) else: cb(reply) # Wait for all multi bulk reply content. for i in range(count): - yield from self._handle_item(reply._feed_received) + await self._handle_item(reply._feed_received) - @asyncio.coroutine - def _handle_pubsub_multibulk_reply(self, multibulk_reply): + async def _handle_pubsub_multibulk_reply(self, multibulk_reply): # Read first item of the multi bulk reply raw. - type = yield from multibulk_reply._read(decode=False, _one=True) + type = await multibulk_reply._read(decode=False, _one=True) assert type in (b'message', b'subscribe', b'unsubscribe', b'pmessage', b'psubscribe', b'punsubscribe') if type == b'message': - channel, value = yield from multibulk_reply._read(count=2) - yield from self._subscription._messages_queue.put(PubSubReply(channel, value)) + channel, value = await multibulk_reply._read(count=2) + await self._subscription._messages_queue.put(PubSubReply(channel, value)) elif type == b'pmessage': - pattern, channel, value = yield from multibulk_reply._read(count=3) - yield from self._subscription._messages_queue.put(PubSubReply(channel, value, pattern=pattern)) + pattern, channel, value = await multibulk_reply._read(count=3) + await self._subscription._messages_queue.put(PubSubReply(channel, value, pattern=pattern)) # We can safely ignore 'subscribe'/'unsubscribe' replies at this point, # they don't contain anything really useful. @@ -1034,14 +995,13 @@ def _send_command(self, args): # Flush the last part self.transport.write(b''.join(data)) - @asyncio.coroutine - def _get_answer(self, transaction, answer_f, _bypass=False, call=None): # XXX: rename _bypass to not_queued + async def _get_answer(self, transaction, answer_f, _bypass=False, call=None): # XXX: rename _bypass to not_queued """ Return an answer to the pipelined query. (Or when we are in a transaction, return a future for the answer.) """ # Wait for the answer to come in - result = yield from answer_f + result = await answer_f if transaction != _NoTransaction and not _bypass: # When the connection is inside a transaction, the query will be queued. @@ -1049,7 +1009,7 @@ def _get_answer(self, transaction, answer_f, _bypass=False, call=None): # XXX: raise Error('Expected to receive QUEUED for query in transaction, received %r.' % result) # Return a future which will contain the result when it arrives. - f = Future(loop=self._loop) + f = asyncio.Future() self._transaction_response_queue.append( (f, call) ) return f else: @@ -1073,8 +1033,7 @@ def _push_answer(self, answer): else: f.set_result(answer) - @asyncio.coroutine - def _query(self, transaction, *args, _bypass=False, set_blocking=False): + async def _query(self, transaction, *args, _bypass=False, set_blocking=False): """ Wrapper around both _send_command and _get_answer. @@ -1090,7 +1049,7 @@ def _query(self, transaction, *args, _bypass=False, set_blocking=False): # Get lock. if transaction == _NoTransaction: - yield from self._transaction_lock.acquire() + await self._transaction_lock.acquire() else: assert transaction == self._transaction @@ -1099,7 +1058,7 @@ def _query(self, transaction, *args, _bypass=False, set_blocking=False): self._pipelined_calls.add(call) # Add a new future to our answer queue. - answer_f = Future(loop=self._loop) + answer_f = asyncio.Future() self._queue.append(answer_f) # Send command @@ -1114,7 +1073,7 @@ def _query(self, transaction, *args, _bypass=False, set_blocking=False): # Receive answer. - result = yield from self._get_answer(transaction, answer_f, _bypass=_bypass, call=call) + result = await self._get_answer(transaction, answer_f, _bypass=_bypass, call=call) return result # Internal @@ -1142,22 +1101,22 @@ def set(self, tr, key:NativeType, value:NativeType, :: - yield from protocol.set('key', 'value') - result = yield from protocol.get('key') + await protocol.set('key', 'value') + result = await protocol.get('key') assert result == 'value' To set a value and its expiration, only if key not exists, do: :: - yield from protocol.set('key', 'value', expire=1, only_if_not_exists=True) + await protocol.set('key', 'value', expire=1, only_if_not_exists=True) This will send: ``SET key value EX 1 NX`` at the network. To set value and its expiration in milliseconds, but only if key already exists: :: - yield from protocol.set('key', 'value', pexpire=1000, only_if_exists=True) + await protocol.set('key', 'value', pexpire=1000, only_if_exists=True) """ params = [ b'set', @@ -1511,10 +1470,9 @@ def _blocking_pop(self, tr, command, keys, timeout:int=0): return self._query(tr, command, *([ self.encode_from_native(k) for k in keys ] + [self._encode_int(timeout)]), set_blocking=True) @_command - @asyncio.coroutine - def brpoplpush(self, tr, source:NativeType, destination:NativeType, timeout:int=0) -> NativeType: + async def brpoplpush(self, tr, source:NativeType, destination:NativeType, timeout:int=0) -> NativeType: """ Pop a value from a list, push it to another list and return it; or block until one is available """ - result = yield from self._query(tr, b'brpoplpush', self.encode_from_native(source), self.encode_from_native(destination), + result = await self._query(tr, b'brpoplpush', self.encode_from_native(source), self.encode_from_native(destination), self._encode_int(timeout), set_blocking=True) if result is None: @@ -1840,28 +1798,23 @@ def hincrbyfloat(self, tr, key:NativeType, field:NativeType, increment:(int,floa # (subscribe, unsubscribe, etc... should be called through the Subscription class.) @_command - def start_subscribe(self, tr, *a) -> 'Subscription': + async def start_subscribe(self, tr, *a) -> 'Subscription': """ Start a pubsub listener. :: # Create subscription - subscription = yield from protocol.start_subscribe() - yield from subscription.subscribe(['key']) - yield from subscription.psubscribe(['pattern*']) + subscription = await protocol.start_subscribe() + await subscription.subscribe(['key']) + await subscription.psubscribe(['pattern*']) while True: - result = yield from subscription.next_published() + result = await subscription.next_published() print(result) :returns: :class:`~asyncio_redis.Subscription` """ - # (Make coroutine. @asyncio.coroutine breaks documentation. It uses - # @functools.wraps to make a generator for this function. But _command - # will no longer be able to read the signature.) - if False: yield - if self.in_use: raise Error('Cannot start pubsub listener when a protocol is in use.') @@ -1895,8 +1848,7 @@ def _punsubscribe(self, tr, patterns:ListOf(NativeType)) -> NoneType: # XXX: uni self._pubsub_patterns -= set(patterns) return self._pubsub_method('punsubscribe', patterns) - @asyncio.coroutine - def _pubsub_method(self, method, params): + async def _pubsub_method(self, method, params): if not self._in_pubsub: raise Error('Cannot call pubsub methods without calling start_subscribe') @@ -1988,11 +1940,6 @@ def flushdb(self, tr) -> StatusReply: """ Delete all the keys of the currently selected DB. This command never fails. """ return self._query(tr, b'flushdb') -# @_query_command -# def object(self, subcommand, args): -# """ Inspect the internals of Redis objects """ -# raise NotImplementedError - @_query_command def type(self, tr, key:NativeType) -> StatusReply: """ Determine the type stored at key """ @@ -2058,19 +2005,18 @@ def client_kill(self, tr, address:str) -> StatusReply: # LUA scripting @_command - @asyncio.coroutine - def register_script(self, tr, script:str) -> 'Script': + async def register_script(self, tr, script:str) -> 'Script': """ Register a LUA script. :: - script = yield from protocol.register_script(lua_code) - result = yield from script.run(keys=[...], args=[...]) + script = await protocol.register_script(lua_code) + result = await script.run(keys=[...], args=[...]) """ # The register_script APi was made compatible with the redis.py library: # https://github.com/andymccurdy/redis-py - sha = yield from self.script_load(tr, script) + sha = await self.script_load(tr, script) return Script(sha, script, lambda:self.evalsha) @_query_command @@ -2084,15 +2030,14 @@ def script_flush(self, tr) -> StatusReply: return self._query(tr, b'script', b'flush') @_query_command - @asyncio.coroutine - def script_kill(self, tr) -> StatusReply: + async def script_kill(self, tr) -> StatusReply: """ Kill the script currently in execution. This raises :class:`~asyncio_redis.exceptions.NoRunningScriptError` when there are no scrips running. """ try: - return (yield from self._query(tr, b'script', b'kill')) + return await self._query(tr, b'script', b'kill') except ErrorReply as e: if 'NOTBUSY' in e.args[0]: raise NoRunningScriptError @@ -2100,8 +2045,7 @@ def script_kill(self, tr) -> StatusReply: raise @_query_command - @asyncio.coroutine - def evalsha(self, tr, sha:str, + async def evalsha(self, tr, sha:str, keys:(ListOf(NativeType), NoneType)=None, args:(ListOf(NativeType), NoneType)=None) -> EvalScriptReply: """ @@ -2117,7 +2061,7 @@ def evalsha(self, tr, sha:str, if not args: args = [] try: - result = yield from self._query(tr, b'evalsha', sha.encode('ascii'), + result = await self._query(tr, b'evalsha', sha.encode('ascii'), self._encode_int(len(keys)), *map(self.encode_from_native, keys + args)) @@ -2140,9 +2084,9 @@ def scan(self, tr, match:(NativeType, NoneType)=None) -> Cursor: :: - cursor = yield from protocol.scan(match='*') + cursor = await protocol.scan(match='*') while True: - item = yield from cursor.fetchone() + item = await cursor.fetchone() if item is None: break else: @@ -2150,8 +2094,8 @@ def scan(self, tr, match:(NativeType, NoneType)=None) -> Cursor: :: - cursor = yield from protocol.scan(match='*') - items = yield from cursor.fetchall() + cursor = await protocol.scan(match='*') + items = await cursor.fetchall() It's possible to alter the COUNT-parameter, by assigning a value to ``cursor.count``, before calling ``fetchone`` or ``fetchall``. For @@ -2236,59 +2180,56 @@ def _do_scan(self, tr, verb:bytes, key:NativeType, cursor:int, match:(NativeType # Transaction @_command - @asyncio.coroutine - def watch(self, tr, keys:ListOf(NativeType)) -> NoneType: + async def watch(self, tr, keys:ListOf(NativeType)) -> NoneType: """ Watch keys. :: # Watch keys for concurrent updates - yield from protocol.watch(['key', 'other_key']) + await protocol.watch(['key', 'other_key']) - value = yield from protocol.get('key') - another_value = yield from protocol.get('another_key') + value = await protocol.get('key') + another_value = await protocol.get('another_key') - transaction = yield from protocol.multi() + transaction = await protocol.multi() - f1 = yield from transaction.set('key', another_value) - f2 = yield from transaction.set('another_key', value) + f1 = await transaction.set('key', another_value) + f2 = await transaction.set('another_key', value) # Commit transaction - yield from transaction.exec() + await transaction.exec() # Retrieve results - yield from f1 - yield from f2 + await f1 + await f2 """ return self._watch(tr, keys) - @asyncio.coroutine - def _watch(self, tr, keys:ListOf(NativeType)) -> NoneType: - result = yield from self._query(tr, b'watch', *map(self.encode_from_native, keys), _bypass=True) + async def _watch(self, tr, keys:ListOf(NativeType)) -> NoneType: + result = await self._query(tr, b'watch', *map(self.encode_from_native, keys), _bypass=True) assert result == b'OK' @_command - @asyncio.coroutine - def multi(self, tr, watch:(ListOf(NativeType),NoneType)=None) -> 'Transaction': + async def multi(self, tr, watch:(ListOf(NativeType),NoneType)=None) -> 'Transaction': """ Start of transaction. :: - transaction = yield from protocol.multi() + transaction = await protocol.multi() # Run commands in transaction - f1 = yield from transaction.set('key', 'value') - f2 = yield from transaction.set('another_key', 'another_value') + f1 = await transaction.set('key', 'value') + f2 = await transaction.set('another_key', 'another_value') # Commit transaction - yield from transaction.exec() + await transaction.exec() # Retrieve results (you can also use asyncio.tasks.gather) - result1 = yield from f1 - result2 = yield from f2 + result1 = await f1 + result2 = await f2 :returns: A :class:`asyncio_redis.Transaction` instance. """ @@ -2296,25 +2237,23 @@ def multi(self, tr, watch:(ListOf(NativeType),NoneType)=None) -> 'Transaction': if tr != _NoTransaction: raise Error('Multi calls can not be nested.') else: - yield from self._transaction_lock.acquire() + await self._transaction_lock.acquire() tr = Transaction(self) self._transaction = tr # Call watch if watch is not None: - yield from self._watch(tr, watch) -# yield from asyncio.sleep(.015) + await self._watch(tr, watch) # Call multi - result = yield from self._query(tr, b'multi', _bypass=True) + result = await self._query(tr, b'multi', _bypass=True) assert result == b'OK' self._transaction_response_queue = deque() return tr - @asyncio.coroutine - def _exec(self, tr): + async def _exec(self, tr): """ Execute all commands issued after MULTI """ @@ -2325,7 +2264,7 @@ def _exec(self, tr): self._transaction_response_queue = None # Get transaction answers. - multi_bulk_reply = yield from self._query(tr, b'exec', _bypass=True) + multi_bulk_reply = await self._query(tr, b'exec', _bypass=True) if multi_bulk_reply is None: # We get None when a transaction failed. @@ -2334,7 +2273,7 @@ def _exec(self, tr): assert isinstance(multi_bulk_reply, MultiBulkReply) for f in multi_bulk_reply.iter_raw(): - answer = yield from f + answer = await f f2, call = futures_and_postprocessors.popleft() if isinstance(answer, Exception): @@ -2349,8 +2288,7 @@ def _exec(self, tr): self._transaction = None self._transaction_lock.release() - @asyncio.coroutine - def _discard(self, tr): + async def _discard(self, tr): """ Discard all commands issued after MULTI """ @@ -2358,22 +2296,21 @@ def _discard(self, tr): raise Error('Not in transaction') try: - result = yield from self._query(tr, b'discard', _bypass=True) + result = await self._query(tr, b'discard', _bypass=True) assert result == b'OK' finally: self._transaction_response_queue = deque() self._transaction = None self._transaction_lock.release() - @asyncio.coroutine - def _unwatch(self, tr): + async def _unwatch(self, tr): """ Forget about all watched keys """ if not self._transaction or self._transaction != tr: raise Error('Not in transaction') - result = yield from self._query(tr, b'unwatch') # XXX: should be _bypass??? + result = await self._query(tr, b'unwatch') # XXX: should be _bypass??? assert result == b'OK' @@ -2390,10 +2327,10 @@ def run(self, keys=[], args=[]): :: - script_reply = yield from script.run(keys=[], args=[]) + script_reply = await script.run(keys=[], args=[]) # If the LUA script returns something, retrieve the return value - result = yield from script_reply.return_value() + result = await script_reply.return_value() This will raise a :class:`~asyncio_redis.exceptions.ScriptKilledError` exception if the script was killed. @@ -2460,7 +2397,7 @@ class Subscription: """ def __init__(self, protocol): self.protocol = protocol - self._messages_queue = Queue(loop=protocol._loop) # Pubsub queue + self._messages_queue = asyncio.Queue() # Pubsub queue @wraps(RedisProtocol._subscribe) def subscribe(self, channels): @@ -2478,15 +2415,14 @@ def psubscribe(self, patterns): def punsubscribe(self, patterns): return self.protocol._punsubscribe(self, patterns) - @asyncio.coroutine - def next_published(self): + async def next_published(self): """ Coroutine which waits for next pubsub message to be received and returns it. :returns: instance of :class:`PubSubReply ` """ - return (yield from self._messages_queue.get()) + return await self._messages_queue.get() class HiRedisProtocol(RedisProtocol, metaclass=_RedisProtocolMeta): @@ -2531,7 +2467,7 @@ def _process_hiredis_item(self, item, cb): if isinstance(item, (bytes, int)): cb(item) elif isinstance(item, list): - reply = MultiBulkReply(self, len(item), loop=self._loop) + reply = MultiBulkReply(self, len(item)) for i in item: self._process_hiredis_item(i, reply._feed_received) @@ -2542,7 +2478,6 @@ def _process_hiredis_item(self, item, cb): elif isinstance(item, NoneType): cb(item) - @asyncio.coroutine - def _reader_coroutine(self): + async def _reader_coroutine(self): # We don't need this one. return From ee9abbdc2217a0dc4cf8e152f076c4610238fb7b Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 15 Jan 2020 15:55:19 +0000 Subject: [PATCH 11/21] protocol fixes --- asyncio_redis/protocol.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/asyncio_redis/protocol.py b/asyncio_redis/protocol.py index da143ef..3869f91 100644 --- a/asyncio_redis/protocol.py +++ b/asyncio_redis/protocol.py @@ -2077,7 +2077,7 @@ def script_load(self, tr, script:str) -> str: # Scanning @_command - def scan(self, tr, match:(NativeType, NoneType)=None) -> Cursor: + async def scan(self, tr, match:(NativeType, NoneType)=None) -> Cursor: """ Walk through the keys space. You can either fetch the items one by one or in bulk. @@ -2111,8 +2111,6 @@ def scan(self, tr, match:(NativeType, NoneType)=None) -> Cursor: Redis reference: http://redis.io/commands/scan """ - if False: yield - def scanfunc(cursor, count): return self._scan(tr, cursor, match, count) @@ -2127,13 +2125,12 @@ def _scan(self, tr, cursor:int, match:(NativeType,NoneType), count:int) -> _Scan b'count', self._encode_int(count)) @_command - def sscan(self, tr, key:NativeType, match:(NativeType,NoneType)=None) -> SetCursor: + async def sscan(self, tr, key:NativeType, match:(NativeType,NoneType)=None) -> SetCursor: """ Incrementally iterate set elements Also see: :func:`~asyncio_redis.RedisProtocol.scan` """ - if False: yield name = 'sscan(key=%r match=%r)' % (key, match) def scan(cursor, count): @@ -2142,12 +2139,11 @@ def scan(cursor, count): return SetCursor(name=name, scanfunc=scan) @_command - def hscan(self, tr, key:NativeType, match:(NativeType,NoneType)=None) -> DictCursor: + async def hscan(self, tr, key:NativeType, match:(NativeType,NoneType)=None) -> DictCursor: """ Incrementally iterate hash fields and associated values Also see: :func:`~asyncio_redis.RedisProtocol.scan` """ - if False: yield name = 'hscan(key=%r match=%r)' % (key, match) def scan(cursor, count): @@ -2156,12 +2152,11 @@ def scan(cursor, count): return DictCursor(name=name, scanfunc=scan) @_command - def zscan(self, tr, key:NativeType, match:(NativeType,NoneType)=None) -> DictCursor: + async def zscan(self, tr, key:NativeType, match:(NativeType,NoneType)=None) -> DictCursor: """ Incrementally iterate sorted sets elements and associated scores Also see: :func:`~asyncio_redis.RedisProtocol.scan` """ - if False: yield name = 'zscan(key=%r match=%r)' % (key, match) def scan(cursor, count): From e971060210fa7e48d7b5515ab3f74d1186d724f0 Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 15 Jan 2020 16:00:15 +0000 Subject: [PATCH 12/21] bugfix --- asyncio_redis/protocol.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/asyncio_redis/protocol.py b/asyncio_redis/protocol.py index 3869f91..12d5d02 100644 --- a/asyncio_redis/protocol.py +++ b/asyncio_redis/protocol.py @@ -621,7 +621,7 @@ async def done(result): if post_process: result = await post_process(protocol_self, result) typecheck_return(protocol_self, result) - return (result) + return result else: typecheck_input(protocol_self, *a, **kw) @@ -656,6 +656,7 @@ def get_methods(self): return result + _SMALL_INTS = list(str(i).encode('ascii') for i in range(1000)) @@ -2200,7 +2201,7 @@ async def watch(self, tr, keys:ListOf(NativeType)) -> NoneType: await f2 """ - return self._watch(tr, keys) + return await self._watch(tr, keys) async def _watch(self, tr, keys:ListOf(NativeType)) -> NoneType: result = await self._query(tr, b'watch', *map(self.encode_from_native, keys), _bypass=True) From 28406b0fd98f57ab965152ce53a0d214292be4ab Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 15 Jan 2020 16:17:50 +0000 Subject: [PATCH 13/21] Remove unicode markers; set literal --- docs/conf.py | 16 +- tests.py | 654 +++++++++++++++++++++++++-------------------------- 2 files changed, 335 insertions(+), 335 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 92f86e0..99ef769 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -44,8 +44,8 @@ master_doc = 'index' # General information about the project. -project = u'asyncio_redis' -copyright = u'2013, Jonathan Slenders' +project = 'asyncio_redis' +copyright = '2013, Jonathan Slenders' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -203,8 +203,8 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'asyncio_redis.tex', u'asyncio\\_redis Documentation', - u'Jonathan Slenders', 'manual'), + ('index', 'asyncio_redis.tex', 'asyncio\\_redis Documentation', + 'Jonathan Slenders', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -233,8 +233,8 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'asyncio_redis', u'asyncio_redis Documentation', - [u'Jonathan Slenders'], 1) + ('index', 'asyncio_redis', 'asyncio_redis Documentation', + ['Jonathan Slenders'], 1) ] # If true, show URL addresses after external links. @@ -247,8 +247,8 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'asyncio_redis', u'asyncio_redis Documentation', - u'Jonathan Slenders', 'asyncio_redis', 'One line description of project.', + ('index', 'asyncio_redis', 'asyncio_redis Documentation', + 'Jonathan Slenders', 'asyncio_redis', 'One line description of project.', 'Miscellaneous'), ] diff --git a/tests.py b/tests.py index ef489d1..1b099ad 100755 --- a/tests.py +++ b/tests.py @@ -108,443 +108,443 @@ def setUp(self): async def test_ping(self, transport, protocol): result = await protocol.ping() self.assertEqual(result, StatusReply('PONG')) - self.assertEqual(repr(result), u"StatusReply(status='PONG')") + self.assertEqual(repr(result), "StatusReply(status='PONG')") @redis_test async def test_echo(self, transport, protocol): - result = await protocol.echo(u'my string') - self.assertEqual(result, u'my string') + result = await protocol.echo('my string') + self.assertEqual(result, 'my string') @redis_test async def test_set_and_get(self, transport, protocol): # Set - value = await protocol.set(u'my_key', u'my_value') + value = await protocol.set('my_key', 'my_value') self.assertEqual(value, StatusReply('OK')) # Get - value = await protocol.get(u'my_key') - self.assertEqual(value, u'my_value') + value = await protocol.get('my_key') + self.assertEqual(value, 'my_value') # Getset - value = await protocol.getset(u'my_key', u'new_value') - self.assertEqual(value, u'my_value') + value = await protocol.getset('my_key', 'new_value') + self.assertEqual(value, 'my_value') - value = await protocol.get(u'my_key') - self.assertEqual(value, u'new_value') + value = await protocol.get('my_key') + self.assertEqual(value, 'new_value') @redis_test async def test_extended_set(self, transport, protocol): - await protocol.delete([u'my_key', u'other_key']) + await protocol.delete(['my_key', 'other_key']) # set with expire only if not exists - value = await protocol.set(u'my_key', u'my_value', + value = await protocol.set('my_key', 'my_value', expire=10, only_if_not_exists=True) self.assertEqual(value, StatusReply('OK')) - value = await protocol.ttl(u'my_key') + value = await protocol.ttl('my_key') self.assertIn(value, (10, 9)) # check NX flag for SET command - value = await protocol.set(u'my_key', u'my_value', + value = await protocol.set('my_key', 'my_value', expire=10, only_if_not_exists=True) self.assertIsNone(value) # check XX flag for SET command - value = await protocol.set(u'other_key', 'some_value', only_if_exists=True) + value = await protocol.set('other_key', 'some_value', only_if_exists=True) self.assertIsNone(value) # set with pexpire only if key exists - value = await protocol.set(u'my_key', u'other_value', + value = await protocol.set('my_key', 'other_value', pexpire=20000, only_if_exists=True) self.assertEqual(value, StatusReply('OK')) - value = await protocol.get(u'my_key') + value = await protocol.get('my_key') - self.assertEqual(value, u'other_value') + self.assertEqual(value, 'other_value') - value = await protocol.ttl(u'my_key') + value = await protocol.ttl('my_key') self.assertIn(value, (20, 19)) @redis_test async def test_setex(self, transport, protocol): # Set - value = await protocol.setex(u'my_key', 10, u'my_value') + value = await protocol.setex('my_key', 10, 'my_value') self.assertEqual(value, StatusReply('OK')) # TTL - value = await protocol.ttl(u'my_key') + value = await protocol.ttl('my_key') self.assertIn(value, (10, 9)) # may be some delay # Get - value = await protocol.get(u'my_key') - self.assertEqual(value, u'my_value') + value = await protocol.get('my_key') + self.assertEqual(value, 'my_value') @redis_test async def test_setnx(self, transport, protocol): - await protocol.delete([u'my_key']) + await protocol.delete(['my_key']) # Setnx while key does not exists - value = await protocol.setnx(u'my_key', u'my_value') + value = await protocol.setnx('my_key', 'my_value') self.assertEqual(value, True) # Get - value = await protocol.get(u'my_key') - self.assertEqual(value, u'my_value') + value = await protocol.get('my_key') + self.assertEqual(value, 'my_value') # Setnx if key exists - value = await protocol.setnx(u'my_key', u'other_value') + value = await protocol.setnx('my_key', 'other_value') self.assertEqual(value, False) # Get old value - value = await protocol.get(u'my_key') - self.assertEqual(value, u'my_value') + value = await protocol.get('my_key') + self.assertEqual(value, 'my_value') @redis_test async def test_special_characters(self, transport, protocol): # Test some special unicode values and spaces. - value = u'my value with special chars " # éçåø´¨åø´h ' + value = 'my value with special chars " # éçåø´¨åø´h ' - result = await protocol.set(u'my key with spaces', value) - result = await protocol.get(u'my key with spaces') + result = await protocol.set('my key with spaces', value) + result = await protocol.get('my key with spaces') self.assertEqual(result, value) # Test newlines - value = u'ab\ncd\ref\r\ngh' - result = await protocol.set(u'my-key', value) - result = await protocol.get(u'my-key') + value = 'ab\ncd\ref\r\ngh' + result = await protocol.set('my-key', value) + result = await protocol.get('my-key') self.assertEqual(result, value) @redis_test async def test_mget(self, transport, protocol): # mget - await protocol.set(u'my_key', u'a') - await protocol.set(u'my_key2', u'b') - result = await protocol.mget([ u'my_key', u'my_key2', u'not_exists']) + await protocol.set('my_key', 'a') + await protocol.set('my_key2', 'b') + result = await protocol.mget([ 'my_key', 'my_key2', 'not_exists']) self.assertIsInstance(result, ListReply) result = await result.aslist() - self.assertEqual(result, [u'a', u'b', None]) + self.assertEqual(result, ['a', 'b', None]) @redis_test async def test_strlen(self, transport, protocol): - await protocol.delete([ u'my_key' ]) - await protocol.delete([ u'my_key2' ]) - await protocol.delete([ u'my_key3' ]) - await protocol.set(u'my_key', u'my_value') - await protocol.hset(u'my_key3', u'a', u'b') + await protocol.delete([ 'my_key' ]) + await protocol.delete([ 'my_key2' ]) + await protocol.delete([ 'my_key3' ]) + await protocol.set('my_key', 'my_value') + await protocol.hset('my_key3', 'a', 'b') # strlen - value = await protocol.strlen(u'my_key') - self.assertEqual(value, len(u'my_value')) + value = await protocol.strlen('my_key') + self.assertEqual(value, len('my_value')) - value = await protocol.strlen(u'my_key2') + value = await protocol.strlen('my_key2') self.assertEqual(value, 0) with self.assertRaises(ErrorReply): - await protocol.strlen(u'my_key3') + await protocol.strlen('my_key3') # Redis exception: b'ERR Operation against a key holding the wrong kind of value') @redis_test async def test_exists_and_delete(self, transport, protocol): # Set - await protocol.set(u'my_key', u'aaa') - value = await protocol.append(u'my_key', u'bbb') + await protocol.set('my_key', 'aaa') + value = await protocol.append('my_key', 'bbb') self.assertEqual(value, 6) # Total length - value = await protocol.get(u'my_key') - self.assertEqual(value, u'aaabbb') + value = await protocol.get('my_key') + self.assertEqual(value, 'aaabbb') @redis_test async def test_exists_and_delete2(self, transport, protocol): # Exists - value = await protocol.exists(u'unknown_key') + value = await protocol.exists('unknown_key') self.assertEqual(value, False) # Set - value = await protocol.set(u'known_key', u'value') - value = await protocol.exists(u'known_key') + value = await protocol.set('known_key', 'value') + value = await protocol.exists('known_key') self.assertEqual(value, True) # Delete - value = await protocol.set(u'known_key2', u'value') - value = await protocol.delete([ u'known_key', u'known_key2' ]) + value = await protocol.set('known_key2', 'value') + value = await protocol.delete([ 'known_key', 'known_key2' ]) self.assertEqual(value, 2) - value = await protocol.delete([ u'known_key' ]) + value = await protocol.delete([ 'known_key' ]) self.assertEqual(value, 0) - value = await protocol.exists(u'known_key') + value = await protocol.exists('known_key') self.assertEqual(value, False) @redis_test async def test_rename(self, transport, protocol): # Set - value = await protocol.set(u'old_key', u'value') - value = await protocol.exists(u'old_key') + value = await protocol.set('old_key', 'value') + value = await protocol.exists('old_key') self.assertEqual(value, True) # Rename - value = await protocol.rename(u'old_key', u'new_key') + value = await protocol.rename('old_key', 'new_key') self.assertEqual(value, StatusReply('OK')) - value = await protocol.exists(u'old_key') + value = await protocol.exists('old_key') self.assertEqual(value, False) - value = await protocol.exists(u'new_key') + value = await protocol.exists('new_key') self.assertEqual(value, True) - value = await protocol.get(u'old_key') + value = await protocol.get('old_key') self.assertEqual(value, None) - value = await protocol.get(u'new_key') + value = await protocol.get('new_key') self.assertEqual(value, 'value') # RenameNX - await protocol.delete([ u'key3' ]) - value = await protocol.renamenx(u'new_key', u'key3') + await protocol.delete([ 'key3' ]) + value = await protocol.renamenx('new_key', 'key3') self.assertEqual(value, 1) - await protocol.set(u'key4', u'existing-value') - value = await protocol.renamenx(u'key3', u'key4') + await protocol.set('key4', 'existing-value') + value = await protocol.renamenx('key3', 'key4') self.assertEqual(value, 0) @redis_test async def test_expire(self, transport, protocol): # Set - value = await protocol.set(u'key', u'value') + value = await protocol.set('key', 'value') # Expire (10s) - value = await protocol.expire(u'key', 10) + value = await protocol.expire('key', 10) self.assertEqual(value, 1) - value = await protocol.exists(u'key') + value = await protocol.exists('key') self.assertEqual(value, True) # TTL - value = await protocol.ttl(u'key') + value = await protocol.ttl('key') self.assertIsInstance(value, int) self.assertLessEqual(value, 10) # PTTL - value = await protocol.pttl(u'key') + value = await protocol.pttl('key') self.assertIsInstance(value, int) self.assertLessEqual(value, 10 * 1000) # Pexpire - value = await protocol.pexpire(u'key', 10*1000) + value = await protocol.pexpire('key', 10*1000) self.assertEqual(value, 1) # XXX: check this - value = await protocol.pttl(u'key') + value = await protocol.pttl('key') self.assertLessEqual(value, 10 * 1000) # Expire (1s) and wait - value = await protocol.expire(u'key', 1) - value = await protocol.exists(u'key') + value = await protocol.expire('key', 1) + value = await protocol.exists('key') self.assertEqual(value, True) await asyncio.sleep(2) - value = await protocol.exists(u'key') + value = await protocol.exists('key') self.assertEqual(value, False) # Test persist - await protocol.set(u'key', u'value') - await protocol.expire(u'key', 1) - value = await protocol.persist(u'key') + await protocol.set('key', 'value') + await protocol.expire('key', 1) + value = await protocol.persist('key') self.assertEqual(value, 1) - value = await protocol.persist(u'key') + value = await protocol.persist('key') self.assertEqual(value, 0) await asyncio.sleep(2) - value = await protocol.exists(u'key') + value = await protocol.exists('key') self.assertEqual(value, True) # Test expireat - value = await protocol.expireat(u'key', 1293840000) + value = await protocol.expireat('key', 1293840000) self.assertIsInstance(value, int) # Test pexpireat - value = await protocol.pexpireat(u'key', 1555555555005) + value = await protocol.pexpireat('key', 1555555555005) self.assertIsInstance(value, int) @redis_test async def test_set(self, transport, protocol): # Create set - value = await protocol.delete([ u'our_set' ]) - value = await protocol.sadd(u'our_set', [u'a', u'b']) - value = await protocol.sadd(u'our_set', [u'c']) + value = await protocol.delete([ 'our_set' ]) + value = await protocol.sadd('our_set', ['a', 'b']) + value = await protocol.sadd('our_set', ['c']) self.assertEqual(value, 1) # scard - value = await protocol.scard(u'our_set') + value = await protocol.scard('our_set') self.assertEqual(value, 3) # Smembers - value = await protocol.smembers(u'our_set') + value = await protocol.smembers('our_set') self.assertIsInstance(value, SetReply) - self.assertEqual(repr(value), u"SetReply(length=3)") + self.assertEqual(repr(value), "SetReply(length=3)") value = await value.asset() - self.assertEqual(value, { u'a', u'b', u'c' }) + self.assertEqual(value, { 'a', 'b', 'c' }) # sismember - value = await protocol.sismember(u'our_set', 'a') + value = await protocol.sismember('our_set', 'a') self.assertEqual(value, True) - value = await protocol.sismember(u'our_set', 'd') + value = await protocol.sismember('our_set', 'd') self.assertEqual(value, False) # Intersection, union and diff - await protocol.delete([ u'set2' ]) - await protocol.sadd(u'set2', [u'b', u'c', u'd', u'e']) + await protocol.delete([ 'set2' ]) + await protocol.sadd('set2', ['b', 'c', 'd', 'e']) - value = await protocol.sunion([ u'our_set', 'set2' ]) + value = await protocol.sunion([ 'our_set', 'set2' ]) self.assertIsInstance(value, SetReply) value = await value.asset() - self.assertEqual(value, set([u'a', u'b', u'c', u'd', u'e'])) + self.assertEqual(value, {'a', 'b', 'c', 'd', 'e'}) - value = await protocol.sinter([ u'our_set', 'set2' ]) + value = await protocol.sinter([ 'our_set', 'set2' ]) value = await value.asset() - self.assertEqual(value, set([u'b', u'c'])) + self.assertEqual(value, {'b', 'c'}) - value = await protocol.sdiff([ u'our_set', 'set2' ]) + value = await protocol.sdiff([ 'our_set', 'set2' ]) self.assertIsInstance(value, SetReply) value = await value.asset() - self.assertEqual(value, set([u'a'])) - value = await protocol.sdiff([ u'set2', u'our_set' ]) + self.assertEqual(value, {'a'}) + value = await protocol.sdiff([ 'set2', 'our_set' ]) value = await value.asset() - self.assertEqual(value, set([u'd', u'e'])) + self.assertEqual(value, {'d', 'e'}) # Interstore - value = await protocol.sinterstore(u'result', [u'our_set', 'set2']) + value = await protocol.sinterstore('result', ['our_set', 'set2']) self.assertEqual(value, 2) - value = await protocol.smembers(u'result') + value = await protocol.smembers('result') self.assertIsInstance(value, SetReply) value = await value.asset() - self.assertEqual(value, set([u'b', u'c'])) + self.assertEqual(value, {'b', 'c'}) # Unionstore - value = await protocol.sunionstore(u'result', [u'our_set', 'set2']) + value = await protocol.sunionstore('result', ['our_set', 'set2']) self.assertEqual(value, 5) - value = await protocol.smembers(u'result') + value = await protocol.smembers('result') self.assertIsInstance(value, SetReply) value = await value.asset() - self.assertEqual(value, set([u'a', u'b', u'c', u'd', u'e'])) + self.assertEqual(value, {'a', 'b', 'c', 'd', 'e'}) # Sdiffstore - value = await protocol.sdiffstore(u'result', [u'set2', 'our_set']) + value = await protocol.sdiffstore('result', ['set2', 'our_set']) self.assertEqual(value, 2) - value = await protocol.smembers(u'result') + value = await protocol.smembers('result') self.assertIsInstance(value, SetReply) value = await value.asset() - self.assertEqual(value, set([u'd', u'e'])) + self.assertEqual(value, {'d', 'e'}) @redis_test async def test_srem(self, transport, protocol): - await protocol.delete([ u'our_set' ]) - await protocol.sadd(u'our_set', [u'a', u'b', u'c', u'd']) + await protocol.delete([ 'our_set' ]) + await protocol.sadd('our_set', ['a', 'b', 'c', 'd']) # Call srem - result = await protocol.srem(u'our_set', [u'b', u'c']) + result = await protocol.srem('our_set', ['b', 'c']) self.assertEqual(result, 2) - result = await protocol.smembers(u'our_set') + result = await protocol.smembers('our_set') self.assertIsInstance(result, SetReply) result = await result.asset() - self.assertEqual(result, set([u'a', u'd'])) + self.assertEqual(result, {'a', 'd'}) @redis_test async def test_spop(self, transport, protocol): async def setup(): - await protocol.delete([ u'my_set' ]) - await protocol.sadd(u'my_set', [u'value1']) - await protocol.sadd(u'my_set', [u'value2']) + await protocol.delete([ 'my_set' ]) + await protocol.sadd('my_set', ['value1']) + await protocol.sadd('my_set', ['value2']) # Test spop await setup() - result = await protocol.spop(u'my_set') - self.assertIn(result, [u'value1', u'value2']) - result = await protocol.smembers(u'my_set') + result = await protocol.spop('my_set') + self.assertIn(result, ['value1', 'value2']) + result = await protocol.smembers('my_set') self.assertIsInstance(result, SetReply) result = await result.asset() self.assertEqual(len(result), 1) # Test srandmember await setup() - result = await protocol.srandmember(u'my_set') + result = await protocol.srandmember('my_set') self.assertIsInstance(result, SetReply) result = await result.asset() - self.assertIn(list(result)[0], [u'value1', u'value2']) - result = await protocol.smembers(u'my_set') + self.assertIn(list(result)[0], ['value1', 'value2']) + result = await protocol.smembers('my_set') self.assertIsInstance(result, SetReply) result = await result.asset() self.assertEqual(len(result), 2) # Popping from non-existing key should return None. - await protocol.delete([ u'my_set' ]) - result = await protocol.spop(u'my_set') + await protocol.delete([ 'my_set' ]) + result = await protocol.spop('my_set') self.assertEqual(result, None) @redis_test async def test_type(self, transport, protocol): # Setup - await protocol.delete([ u'key1' ]) - await protocol.delete([ u'key2' ]) - await protocol.delete([ u'key3' ]) + await protocol.delete([ 'key1' ]) + await protocol.delete([ 'key2' ]) + await protocol.delete([ 'key3' ]) - await protocol.set(u'key1', u'value') - await protocol.lpush(u'key2', [u'value']) - await protocol.sadd(u'key3', [u'value']) + await protocol.set('key1', 'value') + await protocol.lpush('key2', ['value']) + await protocol.sadd('key3', ['value']) # Test types - value = await protocol.type(u'key1') + value = await protocol.type('key1') self.assertEqual(value, StatusReply('string')) - value = await protocol.type(u'key2') + value = await protocol.type('key2') self.assertEqual(value, StatusReply('list')) - value = await protocol.type(u'key3') + value = await protocol.type('key3') self.assertEqual(value, StatusReply('set')) @redis_test async def test_list(self, transport, protocol): # Create list - await protocol.delete([ u'my_list' ]) - value = await protocol.lpush(u'my_list', [u'v1', u'v2']) - value = await protocol.rpush(u'my_list', [u'v3', u'v4']) + await protocol.delete([ 'my_list' ]) + value = await protocol.lpush('my_list', ['v1', 'v2']) + value = await protocol.rpush('my_list', ['v3', 'v4']) self.assertEqual(value, 4) # lrange - value = await protocol.lrange(u'my_list') + value = await protocol.lrange('my_list') self.assertIsInstance(value, ListReply) - self.assertEqual(repr(value), u"ListReply(length=4)") + self.assertEqual(repr(value), "ListReply(length=4)") value = await value.aslist() - self.assertEqual(value, [ u'v2', 'v1', 'v3', 'v4']) + self.assertEqual(value, [ 'v2', 'v1', 'v3', 'v4']) # lset - value = await protocol.lset(u'my_list', 3, 'new-value') + value = await protocol.lset('my_list', 3, 'new-value') self.assertEqual(value, StatusReply('OK')) - value = await protocol.lrange(u'my_list') + value = await protocol.lrange('my_list') self.assertIsInstance(value, ListReply) value = await value.aslist() - self.assertEqual(value, [ u'v2', 'v1', 'v3', 'new-value']) + self.assertEqual(value, [ 'v2', 'v1', 'v3', 'new-value']) # lindex - value = await protocol.lindex(u'my_list', 1) + value = await protocol.lindex('my_list', 1) self.assertEqual(value, 'v1') - value = await protocol.lindex(u'my_list', 10) # Unknown index + value = await protocol.lindex('my_list', 10) # Unknown index self.assertEqual(value, None) # Length - value = await protocol.llen(u'my_list') + value = await protocol.llen('my_list') self.assertEqual(value, 4) # Remove element from list. - value = await protocol.lrem(u'my_list', value=u'new-value') + value = await protocol.lrem('my_list', value='new-value') self.assertEqual(value, 1) # Pop - value = await protocol.rpop(u'my_list') - self.assertEqual(value, u'v3') - value = await protocol.lpop(u'my_list') - self.assertEqual(value, u'v2') - value = await protocol.lpop(u'my_list') - self.assertEqual(value, u'v1') - value = await protocol.lpop(u'my_list') + value = await protocol.rpop('my_list') + self.assertEqual(value, 'v3') + value = await protocol.lpop('my_list') + self.assertEqual(value, 'v2') + value = await protocol.lpop('my_list') + self.assertEqual(value, 'v1') + value = await protocol.lpop('my_list') self.assertEqual(value, None) # Blocking lpop @@ -552,196 +552,196 @@ async def test_list(self, transport, protocol): async def blpop(): test_order.append('#1') - value = await protocol.blpop([u'my_list']) + value = await protocol.blpop(['my_list']) self.assertIsInstance(value, BlockingPopReply) - self.assertEqual(value.list_name, u'my_list') - self.assertEqual(value.value, u'value') + self.assertEqual(value.list_name, 'my_list') + self.assertEqual(value.value, 'value') test_order.append('#3') f = asyncio.ensure_future(blpop()) transport2, protocol2 = await connect() test_order.append('#2') - await protocol2.rpush(u'my_list', [u'value']) + await protocol2.rpush('my_list', ['value']) await f self.assertEqual(test_order, ['#1', '#2', '#3']) # Blocking rpop async def brpop(): - value = await protocol.brpop([u'my_list']) + value = await protocol.brpop(['my_list']) self.assertIsInstance(value, BlockingPopReply) - self.assertEqual(value.list_name, u'my_list') - self.assertEqual(value.value, u'value2') + self.assertEqual(value.list_name, 'my_list') + self.assertEqual(value.value, 'value2') f = asyncio.ensure_future(brpop()) - await protocol2.rpush(u'my_list', [u'value2']) + await protocol2.rpush('my_list', ['value2']) await f transport2.close() @redis_test async def test_brpoplpush(self, transport, protocol): - await protocol.delete([ u'from' ]) - await protocol.delete([ u'to' ]) - await protocol.lpush(u'to', [u'1']) + await protocol.delete([ 'from' ]) + await protocol.delete([ 'to' ]) + await protocol.lpush('to', ['1']) async def brpoplpush(): - result = await protocol.brpoplpush(u'from', u'to') - self.assertEqual(result, u'my_value') + result = await protocol.brpoplpush('from', 'to') + self.assertEqual(result, 'my_value') f = asyncio.ensure_future(brpoplpush()) transport2, protocol2 = await connect() - await protocol2.rpush(u'from', [u'my_value']) + await protocol2.rpush('from', ['my_value']) await f transport2.close() @redis_test async def test_blocking_timeout(self, transport, protocol): - await protocol.delete([u'from']) - await protocol.delete([u'to']) + await protocol.delete(['from']) + await protocol.delete(['to']) # brpoplpush with self.assertRaises(TimeoutError) as e: - await protocol.brpoplpush(u'from', u'to', 1) + await protocol.brpoplpush('from', 'to', 1) self.assertIn('Timeout in brpoplpush', e.exception.args[0]) # brpop with self.assertRaises(TimeoutError) as e: - await protocol.brpop([u'from'], 1) + await protocol.brpop(['from'], 1) self.assertIn('Timeout in blocking pop', e.exception.args[0]) # blpop with self.assertRaises(TimeoutError) as e: - await protocol.blpop([u'from'], 1) + await protocol.blpop(['from'], 1) self.assertIn('Timeout in blocking pop', e.exception.args[0]) @redis_test async def test_linsert(self, transport, protocol): # Prepare - await protocol.delete([ u'my_list' ]) - await protocol.rpush(u'my_list', [u'1']) - await protocol.rpush(u'my_list', [u'2']) - await protocol.rpush(u'my_list', [u'3']) + await protocol.delete([ 'my_list' ]) + await protocol.rpush('my_list', ['1']) + await protocol.rpush('my_list', ['2']) + await protocol.rpush('my_list', ['3']) # Insert after - result = await protocol.linsert(u'my_list', u'1', u'A') + result = await protocol.linsert('my_list', '1', 'A') self.assertEqual(result, 4) - result = await protocol.lrange(u'my_list') + result = await protocol.lrange('my_list') self.assertIsInstance(result, ListReply) result = await result.aslist() - self.assertEqual(result, [u'1', u'A', u'2', u'3']) + self.assertEqual(result, ['1', 'A', '2', '3']) # Insert before - result = await protocol.linsert(u'my_list', u'3', u'B', before=True) + result = await protocol.linsert('my_list', '3', 'B', before=True) self.assertEqual(result, 5) - result = await protocol.lrange(u'my_list') + result = await protocol.lrange('my_list') self.assertIsInstance(result, ListReply) result = await result.aslist() - self.assertEqual(result, [u'1', u'A', u'2', u'B', u'3']) + self.assertEqual(result, ['1', 'A', '2', 'B', '3']) @redis_test async def test_rpoplpush(self, transport, protocol): # Prepare - await protocol.delete([ u'my_list' ]) - await protocol.delete([ u'my_list2' ]) - await protocol.lpush(u'my_list', [u'value']) - await protocol.lpush(u'my_list2', [u'value2']) + await protocol.delete([ 'my_list' ]) + await protocol.delete([ 'my_list2' ]) + await protocol.lpush('my_list', ['value']) + await protocol.lpush('my_list2', ['value2']) - value = await protocol.llen(u'my_list') - value2 = await protocol.llen(u'my_list2') + value = await protocol.llen('my_list') + value2 = await protocol.llen('my_list2') self.assertEqual(value, 1) self.assertEqual(value2, 1) # rpoplpush - result = await protocol.rpoplpush(u'my_list', u'my_list2') - self.assertEqual(result, u'value') - result = await protocol.rpoplpush(u'my_list', u'my_list2') + result = await protocol.rpoplpush('my_list', 'my_list2') + self.assertEqual(result, 'value') + result = await protocol.rpoplpush('my_list', 'my_list2') self.assertEqual(result, None) @redis_test async def test_pushx(self, transport, protocol): - await protocol.delete([ u'my_list' ]) + await protocol.delete([ 'my_list' ]) # rpushx - result = await protocol.rpushx(u'my_list', u'a') + result = await protocol.rpushx('my_list', 'a') self.assertEqual(result, 0) - await protocol.rpush(u'my_list', [u'a']) - result = await protocol.rpushx(u'my_list', u'a') + await protocol.rpush('my_list', ['a']) + result = await protocol.rpushx('my_list', 'a') self.assertEqual(result, 2) # lpushx - await protocol.delete([ u'my_list' ]) - result = await protocol.lpushx(u'my_list', u'a') + await protocol.delete([ 'my_list' ]) + result = await protocol.lpushx('my_list', 'a') self.assertEqual(result, 0) - await protocol.rpush(u'my_list', [u'a']) - result = await protocol.lpushx(u'my_list', u'a') + await protocol.rpush('my_list', ['a']) + result = await protocol.lpushx('my_list', 'a') self.assertEqual(result, 2) @redis_test async def test_ltrim(self, transport, protocol): - await protocol.delete([ u'my_list' ]) - await protocol.lpush(u'my_list', [u'a']) - await protocol.lpush(u'my_list', [u'b']) - result = await protocol.ltrim(u'my_list') + await protocol.delete([ 'my_list' ]) + await protocol.lpush('my_list', ['a']) + await protocol.lpush('my_list', ['b']) + result = await protocol.ltrim('my_list') self.assertEqual(result, StatusReply('OK')) @redis_test async def test_hashes(self, transport, protocol): - await protocol.delete([ u'my_hash' ]) + await protocol.delete([ 'my_hash' ]) # Set in hash - result = await protocol.hset(u'my_hash', u'key', u'value') + result = await protocol.hset('my_hash', 'key', 'value') self.assertEqual(result, 1) - result = await protocol.hset(u'my_hash', u'key2', u'value2') + result = await protocol.hset('my_hash', 'key2', 'value2') self.assertEqual(result, 1) # hlen - result = await protocol.hlen(u'my_hash') + result = await protocol.hlen('my_hash') self.assertEqual(result, 2) # hexists - result = await protocol.hexists(u'my_hash', u'key') + result = await protocol.hexists('my_hash', 'key') self.assertEqual(result, True) - result = await protocol.hexists(u'my_hash', u'unknown_key') + result = await protocol.hexists('my_hash', 'unknown_key') self.assertEqual(result, False) # Get from hash - result = await protocol.hget(u'my_hash', u'key2') - self.assertEqual(result, u'value2') - result = await protocol.hget(u'my_hash', u'unknown-key') + result = await protocol.hget('my_hash', 'key2') + self.assertEqual(result, 'value2') + result = await protocol.hget('my_hash', 'unknown-key') self.assertEqual(result, None) - result = await protocol.hgetall(u'my_hash') + result = await protocol.hgetall('my_hash') self.assertIsInstance(result, DictReply) - self.assertEqual(repr(result), u"DictReply(length=2)") + self.assertEqual(repr(result), "DictReply(length=2)") result = await result.asdict() - self.assertEqual(result, {u'key': u'value', u'key2': u'value2' }) + self.assertEqual(result, {'key': 'value', 'key2': 'value2' }) - result = await protocol.hkeys(u'my_hash') + result = await protocol.hkeys('my_hash') self.assertIsInstance(result, SetReply) result = await result.asset() self.assertIsInstance(result, set) - self.assertEqual(result, {u'key', u'key2' }) + self.assertEqual(result, {'key', 'key2' }) - result = await protocol.hvals(u'my_hash') + result = await protocol.hvals('my_hash') self.assertIsInstance(result, ListReply) result = await result.aslist() self.assertIsInstance(result, list) - self.assertEqual(set(result), {u'value', u'value2' }) + self.assertEqual(set(result), {'value', 'value2' }) # HDel - result = await protocol.hdel(u'my_hash', [u'key2']) + result = await protocol.hdel('my_hash', ['key2']) self.assertEqual(result, 1) - result = await protocol.hdel(u'my_hash', [u'key2']) + result = await protocol.hdel('my_hash', ['key2']) self.assertEqual(result, 0) - result = await protocol.hkeys(u'my_hash') + result = await protocol.hkeys('my_hash') self.assertIsInstance(result, SetReply) result = await result.asset() - self.assertEqual(result, { u'key' }) + self.assertEqual(result, { 'key' }) @redis_test async def test_keys(self, transport, protocol): @@ -751,7 +751,7 @@ async def test_keys(self, transport, protocol): await protocol.set('our-keytest-key3', 'a') # Test 'keys' - multibulk = await protocol.keys(u'our-keytest-key*') + multibulk = await protocol.keys('our-keytest-key*') all_keys = await asyncio.gather(*multibulk) self.assertEqual(set(all_keys), { 'our-keytest-key1', @@ -760,46 +760,46 @@ async def test_keys(self, transport, protocol): @redis_test async def test_hmset_get(self, transport, protocol): - await protocol.delete([ u'my_hash' ]) - await protocol.hset(u'my_hash', u'a', u'1') + await protocol.delete([ 'my_hash' ]) + await protocol.hset('my_hash', 'a', '1') # HMSet - result = await protocol.hmset(u'my_hash', { 'b':'2', 'c': '3'}) + result = await protocol.hmset('my_hash', { 'b':'2', 'c': '3'}) self.assertEqual(result, StatusReply('OK')) # HMGet - result = await protocol.hmget(u'my_hash', [u'a', u'b', u'c']) + result = await protocol.hmget('my_hash', ['a', 'b', 'c']) self.assertIsInstance(result, ListReply) result = await result.aslist() - self.assertEqual(result, [ u'1', u'2', u'3']) + self.assertEqual(result, [ '1', '2', '3']) - result = await protocol.hmget(u'my_hash', [u'c', u'b']) + result = await protocol.hmget('my_hash', ['c', 'b']) self.assertIsInstance(result, ListReply) result = await result.aslist() - self.assertEqual(result, [ u'3', u'2' ]) + self.assertEqual(result, [ '3', '2' ]) # Hsetnx - result = await protocol.hsetnx(u'my_hash', u'b', '4') + result = await protocol.hsetnx('my_hash', 'b', '4') self.assertEqual(result, 0) # Existing key. Not set - result = await protocol.hget(u'my_hash', u'b') - self.assertEqual(result, u'2') + result = await protocol.hget('my_hash', 'b') + self.assertEqual(result, '2') - result = await protocol.hsetnx(u'my_hash', u'd', '5') + result = await protocol.hsetnx('my_hash', 'd', '5') self.assertEqual(result, 1) # New key, set - result = await protocol.hget(u'my_hash', u'd') - self.assertEqual(result, u'5') + result = await protocol.hget('my_hash', 'd') + self.assertEqual(result, '5') @redis_test async def test_hincr(self, transport, protocol): - await protocol.delete([ u'my_hash' ]) - await protocol.hset(u'my_hash', u'a', u'10') + await protocol.delete([ 'my_hash' ]) + await protocol.hset('my_hash', 'a', '10') # hincrby - result = await protocol.hincrby(u'my_hash', u'a', 2) + result = await protocol.hincrby('my_hash', 'a', 2) self.assertEqual(result, 12) # hincrbyfloat - result = await protocol.hincrbyfloat(u'my_hash', u'a', 3.7) + result = await protocol.hincrbyfloat('my_hash', 'a', 3.7) self.assertEqual(result, 15.7) @redis_test @@ -812,43 +812,43 @@ async def listener(): subscription = await protocol2.start_subscribe() self.assertIsInstance(subscription, Subscription) self.assertEqual(protocol2.in_pubsub, True) - await subscription.subscribe([u'our_channel']) + await subscription.subscribe(['our_channel']) value = await subscription.next_published() self.assertIsInstance(value, PubSubReply) - self.assertEqual(value.channel, u'our_channel') - self.assertEqual(value.value, u'message1') + self.assertEqual(value.channel, 'our_channel') + self.assertEqual(value.value, 'message1') value = await subscription.next_published() self.assertIsInstance(value, PubSubReply) - self.assertEqual(value.channel, u'our_channel') - self.assertEqual(value.value, u'message2') - self.assertEqual(repr(value), u"PubSubReply(channel='our_channel', value='message2')") + self.assertEqual(value.channel, 'our_channel') + self.assertEqual(value.value, 'message2') + self.assertEqual(repr(value), "PubSubReply(channel='our_channel', value='message2')") return transport2 f = asyncio.ensure_future(listener()) async def sender(): - value = await protocol.publish(u'our_channel', 'message1') + value = await protocol.publish('our_channel', 'message1') self.assertGreaterEqual(value, 1) # Nr of clients that received the message - value = await protocol.publish(u'our_channel', 'message2') + value = await protocol.publish('our_channel', 'message2') self.assertGreaterEqual(value, 1) # Test pubsub_channels result = await protocol.pubsub_channels() self.assertIsInstance(result, ListReply) result = await result.aslist() - self.assertIn(u'our_channel', result) + self.assertIn('our_channel', result) - result = await protocol.pubsub_channels_aslist(u'our_c*') - self.assertIn(u'our_channel', result) + result = await protocol.pubsub_channels_aslist('our_c*') + self.assertIn('our_channel', result) - result = await protocol.pubsub_channels_aslist(u'unknown-channel-prefix*') + result = await protocol.pubsub_channels_aslist('unknown-channel-prefix*') self.assertEqual(result, []) # Test pubsub numsub. - result = await protocol.pubsub_numsub([ u'our_channel', u'some_unknown_channel' ]) + result = await protocol.pubsub_numsub([ 'our_channel', 'some_unknown_channel' ]) self.assertIsInstance(result, DictReply) result = await result.asdict() self.assertEqual(len(result), 2) @@ -951,24 +951,24 @@ async def sender(): @redis_test async def test_incr(self, transport, protocol): - await protocol.set(u'key1', u'3') + await protocol.set('key1', '3') # Incr - result = await protocol.incr(u'key1') + result = await protocol.incr('key1') self.assertEqual(result, 4) - result = await protocol.incr(u'key1') + result = await protocol.incr('key1') self.assertEqual(result, 5) # Incrby - result = await protocol.incrby(u'key1', 10) + result = await protocol.incrby('key1', 10) self.assertEqual(result, 15) # Decr - result = await protocol.decr(u'key1') + result = await protocol.decr('key1') self.assertEqual(result, 14) # Decrby - result = await protocol.decrby(u'key1', 4) + result = await protocol.decrby('key1', 4) self.assertEqual(result, 10) @redis_test @@ -1083,7 +1083,7 @@ async def test_zset(self, transport, protocol): # Test zrange result = await protocol.zrange('myzset') self.assertIsInstance(result, ZRangeReply) - self.assertEqual(repr(result), u"ZRangeReply(length=3)") + self.assertEqual(repr(result), "ZRangeReply(length=3)") self.assertEqual((await result.asdict()), { 'key': 4.0, 'key2': 5.0, 'key3': 5.5 }) @@ -1177,7 +1177,7 @@ async def test_zrevrange(self, transport, protocol): # Test zrevrange result = await protocol.zrevrange('myzset') self.assertIsInstance(result, ZRangeReply) - self.assertEqual(repr(result), u"ZRangeReply(length=3)") + self.assertEqual(repr(result), "ZRangeReply(length=3)") self.assertEqual((await result.asdict()), { 'key': 4.0, 'key2': 5.0, 'key3': 5.5 }) @@ -1310,7 +1310,7 @@ async def test_zinterstore(self, transport, protocol): @redis_test async def test_randomkey(self, transport, protocol): - await protocol.set(u'key1', u'value') + await protocol.set('key1', 'value') result = await protocol.randomkey() self.assertIsInstance(result, str) @@ -1322,12 +1322,12 @@ async def test_dbsize(self, transport, protocol): @redis_test async def test_client_names(self, transport, protocol): # client_setname - result = await protocol.client_setname(u'my-connection-name') + result = await protocol.client_setname('my-connection-name') self.assertEqual(result, StatusReply('OK')) # client_getname result = await protocol.client_getname() - self.assertEqual(result, u'my-connection-name') + self.assertEqual(result, 'my-connection-name') # client list result = await protocol.client_list() @@ -1378,7 +1378,7 @@ async def test_lua_script(self, transport, protocol): self.assertIsInstance(result, EvalScriptReply) result = await result.return_value() self.assertIsInstance(result, str) - self.assertEqual(result, u'text') + self.assertEqual(result, 'text') @redis_test async def test_script_return_types(self, transport, protocol): @@ -1437,11 +1437,11 @@ async def run_while_true(): @redis_test async def test_transaction(self, transport, protocol): # Prepare - await protocol.set(u'my_key', u'a') - await protocol.set(u'my_key2', u'b') - await protocol.set(u'my_key3', u'c') - await protocol.delete([ u'my_hash' ]) - await protocol.hmset(u'my_hash', {'a':'1', 'b':'2', 'c':'3'}) + await protocol.set('my_key', 'a') + await protocol.set('my_key2', 'b') + await protocol.set('my_key3', 'c') + await protocol.delete([ 'my_hash' ]) + await protocol.hmset('my_hash', {'a':'1', 'b':'2', 'c':'3'}) # Start transaction self.assertEqual(protocol.in_transaction, False) @@ -1480,10 +1480,10 @@ async def test_transaction(self, transport, protocol): r4 = await r4.aslist() r5 = await r5.asdict() - self.assertEqual(r1, u'a') - self.assertEqual(r2, [u'a', u'b']) - self.assertEqual(r3, u'c') - self.assertEqual(r4, [u'b', u'c']) + self.assertEqual(r1, 'a') + self.assertEqual(r2, ['a', 'b']) + self.assertEqual(r3, 'c') + self.assertEqual(r4, ['b', 'c']) self.assertEqual(r5, { 'a': '1', 'b': '2', 'c': '3' }) @redis_test @@ -1503,21 +1503,21 @@ async def test_run_command_outside_transaction(self, transport, protocol): @redis_test async def test_discard_transaction(self, transport, protocol): - await protocol.set(u'my_key', u'a') + await protocol.set('my_key', 'a') transaction = await protocol.multi() - await transaction.set(u'my_key', 'b') + await transaction.set('my_key', 'b') # Discard result = await transaction.discard() self.assertEqual(result, None) - result = await protocol.get(u'my_key') - self.assertEqual(result, u'a') + result = await protocol.get('my_key') + self.assertEqual(result, 'a') # Calling anything on the transaction after discard should fail. with self.assertRaises(Error) as e: - result = await transaction.get(u'my_key') + result = await transaction.get('my_key') self.assertEqual(e.exception.args[0], 'Transaction already finished or invalid.') @redis_test @@ -1711,19 +1711,19 @@ async def test_alternate_gets(self, transport, protocol): Test _asdict/_asset/_aslist suffixes. """ # Prepare - await protocol.set(u'my_key', u'a') - await protocol.set(u'my_key2', u'b') + await protocol.set('my_key', 'a') + await protocol.set('my_key2', 'b') - await protocol.delete([ u'my_set' ]) - await protocol.sadd(u'my_set', [u'value1']) - await protocol.sadd(u'my_set', [u'value2']) + await protocol.delete([ 'my_set' ]) + await protocol.sadd('my_set', ['value1']) + await protocol.sadd('my_set', ['value2']) - await protocol.delete([ u'my_hash' ]) - await protocol.hmset(u'my_hash', {'a':'1', 'b':'2', 'c':'3'}) + await protocol.delete([ 'my_hash' ]) + await protocol.hmset('my_hash', {'a':'1', 'b':'2', 'c':'3'}) # Test mget_aslist result = await protocol.mget_aslist(['my_key', 'my_key2']) - self.assertEqual(result, [u'a', u'b']) + self.assertEqual(result, ['a', 'b']) self.assertIsInstance(result, list) # Test keys_aslist @@ -1731,8 +1731,8 @@ async def test_alternate_gets(self, transport, protocol): self.assertIsInstance(result, list) # Test smembers - result = await protocol.smembers_asset(u'my_set') - self.assertEqual(result, { u'value1', u'value2' }) + result = await protocol.smembers_asset('my_set') + self.assertEqual(result, { 'value1', 'value2' }) self.assertIsInstance(result, set) # Test hgetall_asdict @@ -1743,7 +1743,7 @@ async def test_alternate_gets(self, transport, protocol): # test all inside a transaction. transaction = await protocol.multi() f1 = await transaction.mget_aslist(['my_key', 'my_key2']) - f2 = await transaction.smembers_asset(u'my_set') + f2 = await transaction.smembers_asset('my_set') f3 = await transaction.hgetall_asdict('my_hash') await transaction.exec() @@ -1751,10 +1751,10 @@ async def test_alternate_gets(self, transport, protocol): result2 = await f2 result3 = await f3 - self.assertEqual(result1, [u'a', u'b']) + self.assertEqual(result1, ['a', 'b']) self.assertIsInstance(result1, list) - self.assertEqual(result2, { u'value1', u'value2' }) + self.assertEqual(result2, { 'value1', 'value2' }) self.assertIsInstance(result2, set) self.assertEqual(result3, {'a':'1', 'b':'2', 'c':'3'}) @@ -1783,21 +1783,21 @@ async def test_watch_1(self, transport, protocol): Test a transaction using watch. (Retrieve the watched value then use it inside the transaction.) """ - await protocol.set(u'key', u'val') + await protocol.set('key', 'val') # Test - await protocol.watch([u'key']) - value = await protocol.get(u'key') + await protocol.watch(['key']) + value = await protocol.get('key') t = await protocol.multi() - await t.set(u'key', value + u'ue') + await t.set('key', value + 'ue') await t.exec() # Check - result = await protocol.get(u'key') - self.assertEqual(result, u'value') + result = await protocol.get('key') + self.assertEqual(result, 'value') @redis_test async def test_multi_watch_1(self, transport, protocol): @@ -1805,25 +1805,25 @@ async def test_multi_watch_1(self, transport, protocol): Test a transaction, using watch (Test using the watched key inside the transaction.) """ - await protocol.set(u'key', u'0') - await protocol.set(u'other_key', u'0') + await protocol.set('key', '0') + await protocol.set('other_key', '0') # Test self.assertEqual(protocol.in_transaction, False) t = await protocol.multi(watch=['other_key']) self.assertEqual(protocol.in_transaction, True) - await t.set(u'key', u'value') - await t.set(u'other_key', u'my_value') + await t.set('key', 'value') + await t.set('other_key', 'my_value') await t.exec() # Check self.assertEqual(protocol.in_transaction, False) - result = await protocol.get(u'key') - self.assertEqual(result, u'value') - result = await protocol.get(u'other_key') - self.assertEqual(result, u'my_value') + result = await protocol.get('key') + self.assertEqual(result, 'value') + result = await protocol.get('other_key') + self.assertEqual(result, 'my_value') @redis_test async def test_multi_watch_2(self, transport, protocol): @@ -1834,21 +1834,21 @@ async def test_multi_watch_2(self, transport, protocol): # Setup transport2, protocol2 = await connect() - await protocol.set(u'key', u'0') - await protocol.set(u'other_key', u'0') + await protocol.set('key', '0') + await protocol.set('other_key', '0') # Test t = await protocol.multi(watch=['other_key']) await protocol2.set('other_key', 'other_value') - await t.set(u'other_key', u'value') + await t.set('other_key', 'value') with self.assertRaises(TransactionError): await t.exec() # Check self.assertEqual(protocol.in_transaction, False) - result = await protocol.get(u'other_key') - self.assertEqual(result, u'other_value') + result = await protocol.get('other_key') + self.assertEqual(result, 'other_value') transport2.close() @@ -2011,7 +2011,7 @@ async def sink(): self.assertIsInstance(reply, BlockingPopReply) self.assertIsInstance(reply.value, str) results.append(reply.value) - self.assertIn(u"BlockingPopReply(list_name='my-list', value='", repr(reply)) + self.assertIn("BlockingPopReply(list_name='my-list', value='", repr(reply)) # Source: Push items on the queue async def source(): @@ -2116,19 +2116,19 @@ async def test_transactions(self): self.assertIn('No available connections in the pool', e.exception.args[0]) # Run commands in transaction - await t1.set(u'key', u'value') - await t2.set(u'key2', u'value2') + await t1.set('key', 'value') + await t2.set('key2', 'value2') # Commit. await t1.exec() await t2.exec() # Check - result1 = await connection.get(u'key') - result2 = await connection.get(u'key2') + result1 = await connection.get('key') + result2 = await connection.get('key2') - self.assertEqual(result1, u'value') - self.assertEqual(result2, u'value2') + self.assertEqual(result1, 'value') + self.assertEqual(result2, 'value2') connection.close() From 44544cfd8cbcc8913de5cb4ef6cc2e49f94c49a6 Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 15 Jan 2020 16:19:28 +0000 Subject: [PATCH 14/21] examples --- examples/benchmarks/hiredis_test.py | 61 ++++++++++++------------- examples/benchmarks/speed_test.py | 58 +++++++++++++----------- examples/protocol/example.py | 27 +++++------ examples/pubsub/receiver.py | 32 ++++++------- examples/pubsub/sender.py | 38 ++++++++-------- examples/reconnect/test.py | 31 ++++++------- examples/streaming-multi-bulk/test.py | 65 ++++++++++++++------------- 7 files changed, 158 insertions(+), 154 deletions(-) diff --git a/examples/benchmarks/hiredis_test.py b/examples/benchmarks/hiredis_test.py index cf16da3..ff1ca91 100644 --- a/examples/benchmarks/hiredis_test.py +++ b/examples/benchmarks/hiredis_test.py @@ -15,69 +15,64 @@ from asyncio_redis.protocol import HiRedisProtocol -@asyncio.coroutine -def test1(connection): +async def test1(connection): """ Del/get/set of keys """ - yield from connection.delete(['key']) - yield from connection.set('key', 'value') - result = yield from connection.get('key') + await connection.delete(['key']) + await connection.set('key', 'value') + result = await connection.get('key') assert result == 'value' -@asyncio.coroutine -def test2(connection): +async def test2(connection): """ Get/set of a hash of 100 items (with _asdict) """ d = { str(i):str(i) for i in range(100) } - yield from connection.delete(['key']) - yield from connection.hmset('key', d) - result = yield from connection.hgetall_asdict('key') + await connection.delete(['key']) + await connection.hmset('key', d) + result = await connection.hgetall_asdict('key') assert result == d -@asyncio.coroutine -def test3(connection): +async def test3(connection): """ Get/set of a hash of 100 items (without _asdict) """ d = { str(i):str(i) for i in range(100) } - yield from connection.delete(['key']) - yield from connection.hmset('key', d) + await connection.delete(['key']) + await connection.hmset('key', d) - result = yield from connection.hgetall('key') + result = await connection.hgetall('key') d2 = {} for f in result: - k,v = yield from f + k,v = await f d2[k] = v assert d2 == d -@asyncio.coroutine -def test4(connection): +async def test4(connection): """ sadd/smembers of a set of 100 items. (with _asset) """ s = { str(i) for i in range(100) } - yield from connection.delete(['key']) - yield from connection.sadd('key', list(s)) + await connection.delete(['key']) + await connection.sadd('key', list(s)) - s2 = yield from connection.smembers_asset('key') + s2 = await connection.smembers_asset('key') assert s2 == s -@asyncio.coroutine -def test5(connection): +async def test5(connection): """ sadd/smembers of a set of 100 items. (without _asset) """ s = { str(i) for i in range(100) } - yield from connection.delete(['key']) - yield from connection.sadd('key', list(s)) + await connection.delete(['key']) + await connection.sadd('key', list(s)) - result = yield from connection.smembers('key') + result = await connection.smembers('key') s2 = set() for f in result: - i = yield from f + i = await f s2.add(i) assert s2 == s @@ -92,10 +87,10 @@ def test5(connection): ] -def run(): - connection = yield from asyncio_redis.Connection.create(host='localhost', port=6379) +async def main(): + connection = await asyncio_redis.Connection.create(host='localhost', port=6379) if hiredis: - hiredis_connection = yield from asyncio_redis.Connection.create(host='localhost', port=6379, protocol_class=HiRedisProtocol) + hiredis_connection = await asyncio_redis.Connection.create(host='localhost', port=6379, protocol_class=HiRedisProtocol) try: for count, f in benchmarks: @@ -104,14 +99,14 @@ def run(): # Benchmark without hredis start = time.time() for i in range(count): - yield from f(connection) + await f(connection) print(' Pure Python: ', time.time() - start) # Benchmark with hredis if hiredis: start = time.time() for i in range(count): - yield from f(hiredis_connection) + await f(hiredis_connection) print(' hiredis: ', time.time() - start) print() else: @@ -124,4 +119,4 @@ def run(): if __name__ == '__main__': loop = asyncio.get_event_loop() - loop.run_until_complete(run()) + loop.run_until_complete(main()) diff --git a/examples/benchmarks/speed_test.py b/examples/benchmarks/speed_test.py index 808d778..16a37d4 100755 --- a/examples/benchmarks/speed_test.py +++ b/examples/benchmarks/speed_test.py @@ -7,43 +7,49 @@ import asyncio_redis import time -if __name__ == '__main__': - loop = asyncio.get_event_loop() +async def main(): # Enable logging logging.getLogger().addHandler(logging.StreamHandler()) logging.getLogger().setLevel(logging.INFO) - def run(): - #connection = yield from asyncio_redis.Connection.create(host='localhost', port=6379) - connection = yield from asyncio_redis.Pool.create(host='localhost', port=6379, poolsize=50) + # connection = await asyncio_redis.Connection.create(host='localhost', port=6379) + connection = await asyncio_redis.Pool.create(host='localhost', port=6379, + poolsize=50) + + try: + # === Benchmark 1 == + print( + '1. How much time does it take to set 10,000 values in Redis? (without pipelining)') + print('Starting...') + start = time.time() - try: - # === Benchmark 1 == - print('1. How much time does it take to set 10,000 values in Redis? (without pipelining)') - print('Starting...') - start = time.time() + # Do 10,000 set requests + for i in range(10 * 1000): + await connection.set('key', + 'value') # By using await here, we wait for the answer. - # Do 10,000 set requests - for i in range(10 * 1000): - yield from connection.set('key', 'value') # By using yield from here, we wait for the answer. + print('Done. Duration=', time.time() - start) + print() - print('Done. Duration=', time.time() - start) - print() + # === Benchmark 2 (should be at least 3x as fast) == - # === Benchmark 2 (should be at least 3x as fast) == + print( + '2. How much time does it take if we use asyncio.gather, and pipeline requests?') + print('Starting...') + start = time.time() - print('2. How much time does it take if we use asyncio.gather, and pipeline requests?') - print('Starting...') - start = time.time() + # Do 10,000 set requests + futures = [asyncio.Task(connection.set('key', 'value')) for x in + range(10 * 1000)] + await asyncio.gather(*futures) - # Do 10,000 set requests - futures = [ asyncio.Task(connection.set('key', 'value')) for x in range(10 * 1000) ] - yield from asyncio.gather(*futures) + print('Done. Duration=', time.time() - start) - print('Done. Duration=', time.time() - start) + finally: + connection.close() - finally: - connection.close() - loop.run_until_complete(run()) +if __name__ == '__main__': + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) diff --git a/examples/protocol/example.py b/examples/protocol/example.py index 3faa477..0ce999b 100755 --- a/examples/protocol/example.py +++ b/examples/protocol/example.py @@ -5,22 +5,23 @@ import asyncio from asyncio_redis import RedisProtocol -if __name__ == '__main__': - loop = asyncio.get_event_loop() - def run(): - # Create connection - transport, protocol = yield from loop.create_connection(RedisProtocol, 'localhost', 6379) +async def main(): + # Create connection + transport, protocol = await loop.create_connection(RedisProtocol, 'localhost', 6379) + + # Set a key + await protocol.set('key', 'value') - # Set a key - yield from protocol.set('key', 'value') + # Retrieve a key + result = await protocol.get('key') - # Retrieve a key - result = yield from protocol.get('key') + # Print result + print('Succeeded', result == 'value') - # Print result - print ('Succeeded', result == 'value') + transport.close() - transport.close() - loop.run_until_complete(run()) +if __name__ == '__main__': + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) diff --git a/examples/pubsub/receiver.py b/examples/pubsub/receiver.py index 7dd9532..9c2d0ca 100755 --- a/examples/pubsub/receiver.py +++ b/examples/pubsub/receiver.py @@ -3,28 +3,28 @@ import logging import asyncio_redis -if __name__ == '__main__': - loop = asyncio.get_event_loop() +async def main(): # Enable logging logging.getLogger().addHandler(logging.StreamHandler()) logging.getLogger().setLevel(logging.INFO) - def run(): - # Create a new redis connection (this will also auto reconnect) - connection = yield from asyncio_redis.Connection.create('localhost', 6379) + # Create a new redis connection (this will also auto reconnect) + connection = await asyncio_redis.Connection.create('localhost', 6379) + + try: + # Subscribe to a channel. + subscriber = await connection.start_subscribe() + await subscriber.subscribe(['our-channel']) - try: - # Subscribe to a channel. - subscriber = yield from connection.start_subscribe() - yield from subscriber.subscribe([ 'our-channel' ]) + # Print published values in a while/true loop. + while True: + reply = await subscriber.next_published() + print('Received: ', repr(reply.value), 'on channel', reply.channel) - # Print published values in a while/true loop. - while True: - reply = yield from subscriber.next_published() - print('Received: ', repr(reply.value), 'on channel', reply.channel) + finally: + connection.close() - finally: - connection.close() - loop.run_until_complete(run()) +if __name__ == '__main__': + asyncio.get_event_loop().run_until_complete(main()) diff --git a/examples/pubsub/sender.py b/examples/pubsub/sender.py index f5affbb..5293b0e 100755 --- a/examples/pubsub/sender.py +++ b/examples/pubsub/sender.py @@ -4,30 +4,30 @@ import logging -if __name__ == '__main__': - loop = asyncio.get_event_loop() - +async def main(): # Enable logging logging.getLogger().addHandler(logging.StreamHandler()) logging.getLogger().setLevel(logging.INFO) - def run(): - # Create a new redis connection (this will also auto reconnect) - connection = yield from asyncio_redis.Connection.create('localhost', 6379) + # Create a new redis connection (this will also auto reconnect) + connection = await asyncio_redis.Connection.create('localhost', 6379) - try: - while True: - # Get input (always use executor for blocking calls) - text = yield from loop.run_in_executor(None, input, 'Enter message: ') + loop = asyncio.get_event_loop() + try: + while True: + # Get input (always use executor for blocking calls) + text = await loop.run_in_executor(None, input, 'Enter message: ') - # Publish value - try: - yield from connection.publish('our-channel', text) - print('Published.') - except asyncio_redis.Error as e: - print('Published failed', repr(e)) + # Publish value + try: + await connection.publish('our-channel', text) + print('Published.') + except asyncio_redis.Error as e: + print('Published failed', repr(e)) - finally: - connection.close() + finally: + connection.close() - loop.run_until_complete(run()) +if __name__ == '__main__': + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) diff --git a/examples/reconnect/test.py b/examples/reconnect/test.py index c4caaa7..cc1be7e 100755 --- a/examples/reconnect/test.py +++ b/examples/reconnect/test.py @@ -7,26 +7,27 @@ import logging import asyncio_redis -if __name__ == '__main__': - loop = asyncio.get_event_loop() +async def main(): # Enable logging logging.getLogger().addHandler(logging.StreamHandler()) logging.getLogger().setLevel(logging.INFO) - def run(): - connection = yield from asyncio_redis.Connection.create(host='localhost', port=6379) + connection = await asyncio_redis.Connection.create(host='localhost', port=6379) + + try: + while True: + await asyncio.sleep(.5) - try: - while True: - yield from asyncio.sleep(.5) + try: + # Try to send message + await connection.publish('our-channel', 'message') + except Exception as e: + print ('errero', repr(e)) + finally: + connection.close() - try: - # Try to send message - yield from connection.publish('our-channel', 'message') - except Exception as e: - print ('errero', repr(e)) - finally: - connection.close() - loop.run_until_complete(run()) +if __name__ == '__main__': + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) diff --git a/examples/streaming-multi-bulk/test.py b/examples/streaming-multi-bulk/test.py index edec3c1..46462ac 100755 --- a/examples/streaming-multi-bulk/test.py +++ b/examples/streaming-multi-bulk/test.py @@ -7,48 +7,49 @@ import logging import asyncio_redis -if __name__ == '__main__': - loop = asyncio.get_event_loop() +async def main(): # Enable logging logging.getLogger().addHandler(logging.StreamHandler()) logging.getLogger().setLevel(logging.INFO) - def run(): - connection = yield from asyncio_redis.Connection.create(host='localhost', port=6379) + connection = await asyncio_redis.Connection.create(host='localhost', port=6379) + + # Create a set that contains a million items + print('Creating big set contains a million items (Can take about half a minute)') - # Create a set that contains a million items - print('Creating big set contains a million items (Can take about half a minute)') + await connection.delete(['my-big-set']) - yield from connection.delete(['my-big-set']) + # We will suffix all the items with a very long key, just to be sure + # that this needs many IP packets, in order to send or receive this. + long_string = 'abcdefghij' * 1000 # len=10k - # We will suffix all the items with a very long key, just to be sure - # that this needs many IP packets, in order to send or receive this. - long_string = 'abcdefghij' * 1000 # len=10k + for prefix in range(10): + print('Callidng redis sadd:', prefix, '/10') + await connection.sadd('my-big-set', ('%s-%s-%s' % (prefix, i, long_string) for i in range(10 * 1000) )) + print('Done\n') - for prefix in range(10): - print('Callidng redis sadd:', prefix, '/10') - yield from connection.sadd('my-big-set', ('%s-%s-%s' % (prefix, i, long_string) for i in range(10 * 1000) )) - print('Done\n') + # Now stream the values from the database: + print('Streaming values, calling smembers') - # Now stream the values from the database: - print('Streaming values, calling smembers') + # The following smembers call will block until the first IP packet + # containing the head of the multi bulk reply comes in. This will + # contain the size of the multi bulk reply and that's enough + # information to create a SetReply instance. Probably the first packet + # will also contain the first X members, so we don't have to wait for + # these anymore. + set_reply = await connection.smembers('my-big-set') + print('Got: ', set_reply) - # The following smembers call will block until the first IP packet - # containing the head of the multi bulk reply comes in. This will - # contain the size of the multi bulk reply and that's enough - # information to create a SetReply instance. Probably the first packet - # will also contain the first X members, so we don't have to wait for - # these anymore. - set_reply = yield from connection.smembers('my-big-set') - print('Got: ', set_reply) + # Stream the items, this will probably wait for the next IP packets to come in. + count = 0 + for f in set_reply: + await f + count += 1 + if count % 1000 == 0: + print('Received %i items' % count) - # Stream the items, this will probably wait for the next IP packets to come in. - count = 0 - for f in set_reply: - yield from f - count += 1 - if count % 1000 == 0: - print('Received %i items' % count) - loop.run_until_complete(run()) +if __name__ == '__main__': + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) From 01766bab1fc2edb2d128e38b07474e7e1dcff976 Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 15 Jan 2020 16:23:09 +0000 Subject: [PATCH 15/21] unused import --- asyncio_redis/connection.py | 1 - 1 file changed, 1 deletion(-) diff --git a/asyncio_redis/connection.py b/asyncio_redis/connection.py index 40182e3..6d86336 100644 --- a/asyncio_redis/connection.py +++ b/asyncio_redis/connection.py @@ -2,7 +2,6 @@ from .protocol import RedisProtocol, _all_commands import asyncio import logging -import warnings class Connection: From 813c2fdccc59f77d7f215f6107aedcdcf41a119d Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 15 Jan 2020 16:34:41 +0000 Subject: [PATCH 16/21] Remove loop from object state --- CHANGELOG | 1 + asyncio_redis/connection.py | 17 ++++++++++------- asyncio_redis/pool.py | 7 ++++--- asyncio_redis/protocol.py | 20 ++++++++++---------- 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 0e756aa..e732d83 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -5,6 +5,7 @@ CHANGELOG ------------------ - Added support for Python 3.8 - Dropped support for Python <3.6 +- Deprecated 'loop' parameter in Connection.create and Pool.create - Added than support for zpopmin and zrangebylex methods - Fix DeprecationWarning for inspect.formatargspec() - Fix missing object asyncio.streams.IncompleteReadError diff --git a/asyncio_redis/connection.py b/asyncio_redis/connection.py index 6d86336..6de1d04 100644 --- a/asyncio_redis/connection.py +++ b/asyncio_redis/connection.py @@ -2,6 +2,7 @@ from .protocol import RedisProtocol, _all_commands import asyncio import logging +import warnings class Connection: @@ -32,19 +33,19 @@ async def create(cls, host='localhost', port=6379, *, password=None, db=0, :class:`~asyncio_redis.encoders.BaseEncoder` :param bool auto_reconnect: Enable auto reconnect - :param loop: - (optional) asyncio event loop. :param protocol_class: (optional) redis protocol implementation :type protocol_class: :class:`~asyncio_redis.RedisProtocol` """ assert port >= 0, "Unexpected port value: %r" % (port, ) + if loop: + warnings.warn("Deprecated parameter: loop", DeprecationWarning) + connection = cls() connection.host = host connection.port = port - connection._loop = loop or asyncio.get_event_loop() connection._retry_interval = .5 connection._closed = False connection._closing = False @@ -54,11 +55,12 @@ async def create(cls, host='localhost', port=6379, *, password=None, db=0, # Create protocol instance def connection_lost(): if connection._auto_reconnect and not connection._closing: - connection._loop.create_task(connection._reconnect()) + loop = asyncio.get_event_loop() + loop.create_task(connection._reconnect()) # Create protocol instance connection.protocol = protocol_class(password=password, db=db, encoder=encoder, - connection_lost_callback=connection_lost, loop=connection._loop) + connection_lost_callback=connection_lost) # Connect if connection._auto_reconnect: @@ -89,11 +91,12 @@ async def _connect(self): """ Set up Redis connection. """ + loop = asyncio.get_event_loop() logger.log(logging.INFO, 'Connecting to redis') if self.port: - await self._loop.create_connection(lambda: self.protocol, self.host, self.port) + await loop.create_connection(lambda: self.protocol, self.host, self.port) else: - await self._loop.create_unix_connection(lambda: self.protocol, self.host) + await loop.create_unix_connection(lambda: self.protocol, self.host) async def _reconnect(self): """ diff --git a/asyncio_redis/pool.py b/asyncio_redis/pool.py index b13d670..23b4dda 100644 --- a/asyncio_redis/pool.py +++ b/asyncio_redis/pool.py @@ -4,6 +4,7 @@ from functools import wraps import asyncio +import warnings class Pool: @@ -42,8 +43,6 @@ async def create(cls, host='localhost', port=6379, *, password=None, db=0, The number of parallel connections. :param bool auto_reconnect: Enable auto reconnect - :param loop: - (optional) asyncio event loop :param protocol_class: (optional) redis protocol implementation :type protocol_class: @@ -54,6 +53,9 @@ async def create(cls, host='localhost', port=6379, *, password=None, db=0, self._port = port self._poolsize = poolsize + if loop: + warnings.warn("Deprecated parameter: loop", DeprecationWarning) + # Create connections conn_coros = [ Connection.create( @@ -63,7 +65,6 @@ async def create(cls, host='localhost', port=6379, *, password=None, db=0, db=db, encoder=encoder, auto_reconnect=auto_reconnect, - loop=loop, protocol_class=protocol_class, ) for _ in range(poolsize) diff --git a/asyncio_redis/protocol.py b/asyncio_redis/protocol.py index 12d5d02..b38e5be 100644 --- a/asyncio_redis/protocol.py +++ b/asyncio_redis/protocol.py @@ -606,7 +606,8 @@ async def done(result): typecheck_return(protocol_self, result) future2.set_result(result) - future.add_done_callback(lambda f: protocol_self._loop.create_task(done(f.result()))) + loop = asyncio.get_event_loop() + future.add_done_callback(lambda f: loop.create_task(done(f.result()))) return future2 @@ -724,7 +725,7 @@ class RedisProtocol(asyncio.Protocol, metaclass=_RedisProtocolMeta): enabled. :type enable_typechecking: bool """ - def __init__(self, *, password=None, db=0, encoder=None, connection_lost_callback=None, enable_typechecking=True, loop=None): + def __init__(self, *, password=None, db=0, encoder=None, connection_lost_callback=None, enable_typechecking=True): if encoder is None: encoder = UTF8Encoder() @@ -736,7 +737,6 @@ def __init__(self, *, password=None, db=0, encoder=None, connection_lost_callbac self.password = password self.db = db self._connection_lost_callback = connection_lost_callback - self._loop = loop or asyncio.get_event_loop() # Take encode / decode settings from encoder self.encode_from_native = encoder.encode_from_native @@ -779,7 +779,7 @@ def connection_made(self, transport): # Start parsing reader stream. self._reader = asyncio.StreamReader() self._reader.set_transport(transport) - self._reader_f = self._loop.create_task(self._reader_coroutine()) + self._reader_f = asyncio.get_event_loop().create_task(self._reader_coroutine()) async def initialize(): # If a password or database was been given, first connect to that one. @@ -797,7 +797,8 @@ async def initialize(): if self._pubsub_patterns: await self._psubscribe(self._subscription, list(self._pubsub_patterns)) - self._loop.create_task(initialize()) + loop = asyncio.get_event_loop() + loop.create_task(initialize()) def data_received(self, data): """ Process data received from Redis server. """ @@ -945,7 +946,8 @@ async def _handle_multi_bulk_reply(self, cb): # Return the empty queue immediately as an answer. if self._in_pubsub: - self._loop.create_task(self._handle_pubsub_multibulk_reply(reply)) + loop = asyncio.get_event_loop() + loop.create_task(self._handle_pubsub_multibulk_reply(reply)) else: cb(reply) @@ -2432,14 +2434,12 @@ class HiRedisProtocol(RedisProtocol, metaclass=_RedisProtocolMeta): response has been parsed. """ def __init__(self, *, password=None, db=0, encoder=None, - connection_lost_callback=None, enable_typechecking=True, - loop=None): + connection_lost_callback=None, enable_typechecking=True): super().__init__(password=password, db=db, encoder=encoder, connection_lost_callback=connection_lost_callback, - enable_typechecking=enable_typechecking, - loop=loop) + enable_typechecking=enable_typechecking) self._hiredis = None assert hiredis, "`hiredis` libary not available. Please don't use HiRedisProtocol." From 837c9492f9fb1d1647b531c75aaa919a35a83f21 Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 15 Jan 2020 17:09:15 +0000 Subject: [PATCH 17/21] Fix Sphinx errors --- CHANGELOG | 4 ++ asyncio_redis/protocol.py | 16 +++---- docs/conf.py | 2 +- docs/pages/examples.rst | 91 ++++++++++++++++++--------------------- docs/pages/reference.rst | 22 +++++++--- tests.py | 2 +- 6 files changed, 72 insertions(+), 65 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index e732d83..0088c03 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -9,6 +9,10 @@ CHANGELOG - Added than support for zpopmin and zrangebylex methods - Fix DeprecationWarning for inspect.formatargspec() - Fix missing object asyncio.streams.IncompleteReadError +- Intersphinx: all classes that can be imported from the top-level module must now be + referenced from the top-level module in the docstrings too. + E.g. :class:`asyncio_redis.exceptions.Error` has changed to + :class:`asyncio_redis.Error`. 0.15.1: 2018-07-30 diff --git a/asyncio_redis/protocol.py b/asyncio_redis/protocol.py index b38e5be..6ce52d9 100644 --- a/asyncio_redis/protocol.py +++ b/asyncio_redis/protocol.py @@ -524,7 +524,7 @@ def get_name(type_): SetReply: ":class:`SetReply `", StatusReply: ":class:`StatusReply `", ZRangeReply: ":class:`ZRangeReply `", - ZScoreBoundary: ":class:`ZScoreBoundary `", + ZScoreBoundary: ":class:`ZScoreBoundary `", EvalScriptReply: ":class:`EvalScriptReply `", Cursor: ":class:`Cursor `", SetCursor: ":class:`SetCursor `", @@ -710,7 +710,7 @@ class RedisProtocol(asyncio.Protocol, metaclass=_RedisProtocolMeta): :: - self.loop = asyncio.get_event_loop() + loop = asyncio.get_event_loop() transport, protocol = await loop.create_connection(RedisProtocol, 'localhost', 6379) :param password: Redis database password @@ -1458,14 +1458,14 @@ def lindex(self, tr, key:NativeType, index:int) -> (NativeType, NoneType): @_query_command def blpop(self, tr, keys:ListOf(NativeType), timeout:int=0) -> BlockingPopReply: """ Remove and get the first element in a list, or block until one is available. - This will raise :class:`~asyncio_redis.exceptions.TimeoutError` when + This will raise :class:`~asyncio_redis.TimeoutError` when the timeout was exceeded and Redis returns `None`. """ return self._blocking_pop(tr, b'blpop', keys, timeout=timeout) @_query_command def brpop(self, tr, keys:ListOf(NativeType), timeout:int=0) -> BlockingPopReply: """ Remove and get the last element in a list, or block until one is available. - This will raise :class:`~asyncio_redis.exceptions.TimeoutError` when + This will raise :class:`~asyncio_redis.TimeoutError` when the timeout was exceeded and Redis returns `None`. """ return self._blocking_pop(tr, b'brpop', keys, timeout=timeout) @@ -2036,7 +2036,7 @@ def script_flush(self, tr) -> StatusReply: async def script_kill(self, tr) -> StatusReply: """ Kill the script currently in execution. This raises - :class:`~asyncio_redis.exceptions.NoRunningScriptError` when there are no + :class:`~asyncio_redis.NoRunningScriptError` when there are no scrips running. """ try: @@ -2057,7 +2057,7 @@ async def evalsha(self, tr, sha:str, The return type/value depends on the script. - This will raise a :class:`~asyncio_redis.exceptions.ScriptKilledError` + This will raise a :class:`~asyncio_redis.ScriptKilledError` exception if the script was killed. """ if not keys: keys = [] @@ -2330,7 +2330,7 @@ def run(self, keys=[], args=[]): # If the LUA script returns something, retrieve the return value result = await script_reply.return_value() - This will raise a :class:`~asyncio_redis.exceptions.ScriptKilledError` + This will raise a :class:`~asyncio_redis.ScriptKilledError` exception if the script was killed. """ return self.get_evalsha_func()(self.sha, keys, args) @@ -2377,7 +2377,7 @@ def exec(self): """ Execute transaction. - This can raise a :class:`~asyncio_redis.exceptions.TransactionError` + This can raise a :class:`~asyncio_redis.TransactionError` when the transaction fails. """ return self._protocol._exec(self) diff --git a/docs/conf.py b/docs/conf.py index 99ef769..8da515b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -140,7 +140,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +# html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. diff --git a/docs/pages/examples.rst b/docs/pages/examples.rst index 330873d..c931060 100644 --- a/docs/pages/examples.rst +++ b/docs/pages/examples.rst @@ -17,17 +17,18 @@ command of the protocol can be called directly at the connection. import asyncio import asyncio_redis - @asyncio.coroutine - def example(): + + async def example(): # Create Redis connection - connection = yield from asyncio_redis.Connection.create(host='localhost', port=6379) + connection = await asyncio_redis.Connection.create(host='localhost', port=6379) # Set a key - yield from connection.set('my_key', 'my_value') + await connection.set('my_key', 'my_value') # When finished, close the connection. connection.close() + if __name__ == '__main__': loop = asyncio.get_event_loop() loop.run_until_complete(example()) @@ -51,13 +52,13 @@ commands. import asyncio import asyncio_redis - @asyncio.coroutine - def example(): + + async def example(): # Create Redis connection - connection = yield from asyncio_redis.Pool.create(host='localhost', port=6379, poolsize=10) + connection = await asyncio_redis.Pool.create(host='localhost', port=6379, poolsize=10) # Set a key - yield from connection.set('my_key', 'my_value') + await connection.set('my_key', 'my_value') # When finished, close the connection pool. connection.close() @@ -79,24 +80,24 @@ with :func:`exec `. import asyncio import asyncio_redis - @asyncio.coroutine - def example(loop): + + async def example(loop): # Create Redis connection - connection = yield from asyncio_redis.Pool.create(host='localhost', port=6379, poolsize=10) + connection = await asyncio_redis.Pool.create(host='localhost', port=6379, poolsize=10) # Create transaction - transaction = yield from connection.multi() + transaction = await connection.multi() # Run commands in transaction (they return future objects) - f1 = yield from transaction.set('key', 'value') - f2 = yield from transaction.set('another_key', 'another_value') + f1 = await transaction.set('key', 'value') + f2 = await transaction.set('another_key', 'another_value') # Commit transaction - yield from transaction.exec() + await transaction.exec() # Retrieve results - result1 = yield from f1 - result2 = yield from f2 + result1 = await f1 + result2 = await f2 # When finished, close the connection pool. connection.close() @@ -119,20 +120,19 @@ the :class:`Connection ` class or through the :class:` import asyncio import asyncio_redis - @asyncio.coroutine - def example(): + async def example(): # Create connection - connection = yield from asyncio_redis.Connection.create(host='localhost', port=6379) + connection = await asyncio_redis.Connection.create(host='localhost', port=6379) # Create subscriber. - subscriber = yield from connection.start_subscribe() + subscriber = await connection.start_subscribe() # Subscribe to channel. - yield from subscriber.subscribe([ 'our-channel' ]) + await subscriber.subscribe([ 'our-channel' ]) # Inside a while loop, wait for incoming events. while True: - reply = yield from subscriber.next_published() + reply = await subscriber.next_published() print('Received: ', repr(reply.value), 'on channel', reply.channel) # When finished, close the connection. @@ -160,19 +160,19 @@ function -- which can be used to register a LUA script -- returns a return value * ARGV[1] """ - @asyncio.coroutine - def example(): - connection = yield from asyncio_redis.Connection.create(host='localhost', port=6379) + + async def example(): + connection = await asyncio_redis.Connection.create(host='localhost', port=6379) # Set a key - yield from connection.set('my_key', '2') + await connection.set('my_key', '2') # Register script - multiply = yield from connection.register_script(code) + multiply = await connection.register_script(code) # Run script - script_reply = yield from multiply.run(keys=['my_key'], args=['5']) - result = yield from script_reply.return_value() + script_reply = await multiply.run(keys=['my_key'], args=['5']) + result = await script_reply.return_value() print(result) # prints 2 * 5 # When finished, close the connection. @@ -193,16 +193,15 @@ connection, pool or protocol. import asyncio import asyncio_redis - from asyncio_redis.encoders import BytesEncoder - @asyncio.coroutine - def example(): + + async def example(): # Create Redis connection - connection = yield from asyncio_redis.Connection.create(host='localhost', port=6379, encoder=BytesEncoder()) + connection = await asyncio_redis.Connection.create(host='localhost', port=6379, encoder=BytesEncoder()) # Set a key - yield from connection.set(b'my_key', b'my_value') + await connection.set(b'my_key', b'my_value') # When finished, close the connection. connection.close() @@ -224,14 +223,13 @@ The following example will print all the keys in the database: import asyncio import asyncio_redis - from asyncio_redis.encoders import BytesEncoder - @asyncio.coroutine - def example(): - cursor = yield from protocol.scan(match='*') + + async def example(): + cursor = await protocol.scan(match='*') while True: - item = yield from cursor.fetchone() + item = await cursor.fetchone() if item is None: break else: @@ -256,24 +254,21 @@ it as follows: import asyncio import asyncio_redis - @asyncio.coroutine - def example(): + + async def example(): loop = asyncio.get_event_loop() # Create Redis connection - transport, protocol = yield from loop.create_connection( + transport, protocol = await loop.create_connection( asyncio_redis.RedisProtocol, 'localhost', 6379) # Set a key - yield from protocol.set('my_key', 'my_value') + await protocol.set('my_key', 'my_value') # Get a key - result = yield from protocol.get('my_key') + result = await protocol.get('my_key') print(result) - if __name__ == '__main__': - asyncio.get_event_loop().run_until_complete(example()) - .. note:: It is not recommended to use the Protocol class directly, because the low-level Redis implementation could change. Prefer the diff --git a/docs/pages/reference.rst b/docs/pages/reference.rst index dda6d75..7174f69 100644 --- a/docs/pages/reference.rst +++ b/docs/pages/reference.rst @@ -41,12 +41,14 @@ Connection :members: :exclude-members: register_script + Connection pool --------------- .. autoclass:: asyncio_redis.Pool :members: + Command replies --------------- @@ -77,6 +79,12 @@ Command replies .. autoclass:: asyncio_redis.replies.ClientListReply :members: +.. autoclass:: asyncio_redis.replies.ConfigPairReply + :members: + +.. autoclass:: asyncio_redis.replies.EvalScriptReply + :members: + Cursors ------- @@ -116,23 +124,23 @@ Utils Exceptions ---------- -.. autoclass:: asyncio_redis.exceptions.TransactionError +.. autoclass:: asyncio_redis.TransactionError :members: -.. autoclass:: asyncio_redis.exceptions.NotConnectedError +.. autoclass:: asyncio_redis.NotConnectedError :members: -.. autoclass:: asyncio_redis.exceptions.TimeoutError +.. autoclass:: asyncio_redis.TimeoutError :members: -.. autoclass:: asyncio_redis.exceptions.ConnectionLostError +.. autoclass:: asyncio_redis.ConnectionLostError :members: -.. autoclass:: asyncio_redis.exceptions.NoAvailableConnectionsInPoolError +.. autoclass:: asyncio_redis.NoAvailableConnectionsInPoolError :members: -.. autoclass:: asyncio_redis.exceptions.ScriptKilledError +.. autoclass:: asyncio_redis.ScriptKilledError :members: -.. autoclass:: asyncio_redis.exceptions.NoRunningScriptError +.. autoclass:: asyncio_redis.NoRunningScriptError :members: diff --git a/tests.py b/tests.py index 1b099ad..e6b94b0 100755 --- a/tests.py +++ b/tests.py @@ -2211,7 +2211,7 @@ def test_no_global_loop(self): # ** Run code on the new loop. ** # Create connection - connection = new_loop.run_until_complete(Connection.create(host=HOST, port=PORT, loop=new_loop)) + connection = new_loop.run_until_complete(Connection.create(host=HOST, port=PORT)) self.assertIsInstance(connection, Connection) try: From dee306f7622cf85945f142c782d3d0f454502b09 Mon Sep 17 00:00:00 2001 From: crusaderky Date: Mon, 20 Jan 2020 23:32:31 +0000 Subject: [PATCH 18/21] Update CHANGELOG --- CHANGELOG | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 0088c03..18afc11 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -6,7 +6,7 @@ CHANGELOG - Added support for Python 3.8 - Dropped support for Python <3.6 - Deprecated 'loop' parameter in Connection.create and Pool.create -- Added than support for zpopmin and zrangebylex methods +- Added support for zpopmin and zrangebylex methods - Fix DeprecationWarning for inspect.formatargspec() - Fix missing object asyncio.streams.IncompleteReadError - Intersphinx: all classes that can be imported from the top-level module must now be From e8b029e71aa4fc9b02f1a863b6cd88a00cf34ebd Mon Sep 17 00:00:00 2001 From: crusaderky Date: Mon, 20 Jan 2020 23:33:53 +0000 Subject: [PATCH 19/21] Apply suggestions from code review --- CHANGELOG | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 18afc11..4b090be 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -11,8 +11,8 @@ CHANGELOG - Fix missing object asyncio.streams.IncompleteReadError - Intersphinx: all classes that can be imported from the top-level module must now be referenced from the top-level module in the docstrings too. - E.g. :class:`asyncio_redis.exceptions.Error` has changed to - :class:`asyncio_redis.Error`. + E.g. asyncio_redis.exceptions.Error has changed to + asyncio_redis.Error. 0.15.1: 2018-07-30 From 7926aab74be6d31f27ead3450dc1102daa4938f4 Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Mon, 20 Jan 2020 23:59:23 +0000 Subject: [PATCH 20/21] Clean up replies.py --- asyncio_redis/replies.py | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/asyncio_redis/replies.py b/asyncio_redis/replies.py index 02bad11..e09fe07 100644 --- a/asyncio_redis/replies.py +++ b/asyncio_redis/replies.py @@ -13,12 +13,6 @@ ) -try: - ensure_future = asyncio.ensure_future -except AttributeError: - ensure_future = getattr(asyncio, "async") - - class StatusReply: """ Wrapper for Redis status replies. @@ -28,7 +22,7 @@ def __init__(self, status): self.status = status def __repr__(self): - return 'StatusReply(status=%r)' % self.status + return f'StatusReply(status={self.status!r})' def __eq__(self, other): return self.status == other.status @@ -66,7 +60,7 @@ async def getter(f): for _ in range(self._result.count // 2): read_future = self._result._read(count=2) - yield ensure_future(getter(read_future)) + yield asyncio.ensure_future(getter(read_future)) async def asdict(self): """ @@ -76,7 +70,7 @@ async def asdict(self): return dict(self._parse(k, v) for k, v in zip(data[::2], data[1::2])) def __repr__(self): - return '%s(length=%r)' % (self.__class__.__name__, int(self._result.count / 2)) + return f'{self.__class__.__name__}(length={self._result.count // 2})' class ZRangeReply(DictReply): @@ -113,7 +107,7 @@ async def asset(self): return set(data) def __repr__(self): - return 'SetReply(length=%r)' % (self._result.count) + return f'SetReply(length={self._result.count})' class ListReply: @@ -141,7 +135,7 @@ def aslist(self): return self._result._read(count=self._result.count) def __repr__(self): - return 'ListReply(length=%r)' % (self._result.count, ) + return f'ListReply(length={self._result.count})' class BlockingPopReply: @@ -164,7 +158,7 @@ def value(self): return self._value def __repr__(self): - return 'BlockingPopReply(list_name=%r, value=%r)' % (self.list_name, self.value) + return f'BlockingPopReply(list_name={self.list_name!r}, value={self.value!r})' class ConfigPairReply: @@ -184,7 +178,7 @@ def value(self): return self._value def __repr__(self): - return 'ConfigPairReply(parameter=%r, value=%r)' % (self.parameter, self.value) + return f'ConfigPairReply(parameter={self.parameter!r}, value={self.value!r})' class InfoReply: @@ -222,7 +216,7 @@ def pattern(self): return self._pattern def __repr__(self): - return 'PubSubReply(channel=%r, value=%r)' % (self.channel, self.value) + return f'PubSubReply(channel={self.channel!r}, value={self.value!r})' def __eq__(self, other): return (self._channel == other._channel and @@ -257,11 +251,7 @@ async def decode(obj): elif isinstance(obj, MultiBulkReply): # Unpack MultiBulkReply recursively as Python list. - result = [] - for f in obj: - item = await f - result.append((await decode(item))) - return result + return [(await decode(await f)) for f in obj] else: # Nonetype, or decoded bytes. From 4043a7fd33281c03e77f18b0aad3b3a2a6cc0fcd Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Tue, 21 Jan 2020 00:06:13 +0000 Subject: [PATCH 21/21] CHANGELOG --- CHANGELOG | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 4b090be..18afc11 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -11,8 +11,8 @@ CHANGELOG - Fix missing object asyncio.streams.IncompleteReadError - Intersphinx: all classes that can be imported from the top-level module must now be referenced from the top-level module in the docstrings too. - E.g. asyncio_redis.exceptions.Error has changed to - asyncio_redis.Error. + E.g. :class:`asyncio_redis.exceptions.Error` has changed to + :class:`asyncio_redis.Error`. 0.15.1: 2018-07-30