diff --git a/.flake8 b/.flake8 index b7573aa..3a4abf6 100644 --- a/.flake8 +++ b/.flake8 @@ -1,3 +1,3 @@ [flake8] exclude = .get,__pycache__,old,build,dist -max-line-length = 88 +max-line-length = 90 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e565da1..62a88d1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,6 +16,10 @@ jobs: image: redis ports: - 6379:6379 + memcached: + image: memcached:latest + ports: + - 11211:11211 strategy: matrix: diff --git a/cache_tower/adapters/memcached_adapter.py b/cache_tower/adapters/memcached_adapter.py new file mode 100644 index 0000000..0c7ac95 --- /dev/null +++ b/cache_tower/adapters/memcached_adapter.py @@ -0,0 +1,64 @@ +from .base_adapter import BaseAdapter +from typing import Any, Dict, List, Optional +from pymemcache.client.base import Client, PooledClient +from pymemcache.client.hash import HashClient + + +class MemcachedAdapter(BaseAdapter): + def __init__(self, params: dict, namespace: str = "", ttl: int = 0) -> None: + super().__init__(params, namespace, ttl) + memcached_params = params.copy() + self._client_type = memcached_params.pop("client_type") + if self._client_type == "client": + self._memcached_client = Client(**memcached_params) + elif self._client_type == "pooled_client": + self._memcached_client = PooledClient(**memcached_params) + elif self._client_type == "hash_client": + self._memcached_client = HashClient(**memcached_params) + else: + raise RuntimeError("Invalid client type for the memcached adapter") + + def _encode(self, value: str) -> bytes: + return str(value).encode("utf-8") + + def _decode(self, value: bytes) -> str: + if value is None: + return None + return bytes(value).decode("utf-8") + + def _encode_items( + self, items: Dict[str, Optional[Any]] + ) -> Dict[str, Optional[Any]]: + return {key: self._encode(value) for key, value in items.items()} # type: ignore + + def get(self, key: str) -> Any: + return self._decode(self._memcached_client.get(self._create_key(key))) + + def mget(self, keys: List[str]) -> Dict[str, Optional[Any]]: + namespaced_keys = self._create_keys(keys) + items = self._memcached_client.get_many(namespaced_keys) + + namespace_len = len(self._namespace) + result: Dict[str, Optional[Any]] = {} + for key in namespaced_keys: + result[key[namespace_len:]] = self._decode(items.get(key, None)) + return result + + def set(self, key: str, value: Any) -> None: + self._memcached_client.set( + self._create_key(key), self._encode(value), expire=self._ttl + ) + + def mset(self, items: Dict[str, Any]) -> None: + self._memcached_client.set_many( + self._encode_items(self._create_items(items)), expire=self._ttl + ) + + def delete(self, key: str) -> None: + self._memcached_client.delete(self._create_key(key)) + + def exists(self, key: str) -> bool: + return bool(self._memcached_client.get(self._create_key(key))) + + def flush(self) -> None: + self._memcached_client.flush_all() diff --git a/cache_tower/adapters/memory_adapter.py b/cache_tower/adapters/memory_adapter.py index 88fa6ec..ecce5ba 100644 --- a/cache_tower/adapters/memory_adapter.py +++ b/cache_tower/adapters/memory_adapter.py @@ -28,9 +28,7 @@ def get(self, key: str) -> Any: def mget(self, keys: List[str]) -> Dict[str, Optional[Any]]: values = {} for key in keys: - value = self.get(key) - if value: - values[key] = value + values[key] = self.get(key) return values def set(self, key: str, value: Any) -> None: diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..2f37a3e --- /dev/null +++ b/mypy.ini @@ -0,0 +1,5 @@ +[mypy] +warn_unused_configs = True + +[mypy-pymemcache.*] +ignore_missing_imports = True \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 31f126a..1fbca95 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,9 +9,11 @@ packages = [{include = "cache_tower"}] [tool.poetry.dependencies] python = "^3.9" redis = {version = "^4.6.0", optional = true} +pymemcache = {version = "^4.0.0", optional = true} [tool.poetry.extras] redis = ["redis"] +memcached = ["pymemcache"] [tool.poetry.group.dev.dependencies] flake8 = "^6.1.0" @@ -25,3 +27,7 @@ ruff = "^0.0.282" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + +[tool.ruff] +# Allow lines to be as long as 120 characters. +line-length = 90 \ No newline at end of file diff --git a/tests/test_adapters/test_memcached_adapter_client.py b/tests/test_adapters/test_memcached_adapter_client.py new file mode 100644 index 0000000..815f6c8 --- /dev/null +++ b/tests/test_adapters/test_memcached_adapter_client.py @@ -0,0 +1,125 @@ +from typing import Iterator +from cache_tower.adapters.memcached_adapter import MemcachedAdapter +from pymemcache.client.base import Client +import pytest + + +@pytest.fixture +def memcached_client() -> Iterator[Client]: + memcached_client = Client(("localhost", 11211)) + memcached_client.flush_all() + yield memcached_client + memcached_client.close() + + +def test_invalid_client(): + with pytest.raises(RuntimeError): + MemcachedAdapter( + {"client_type": "invalid_client", "server": ("localhost", 11211)}, + "namespace:", + 60, + ) + + +def test_get(memcached_client: Client): + adapter = MemcachedAdapter( + {"client_type": "client", "server": ("localhost", 11211)}, "namespace:", 60 + ) + + memcached_client.set("namespace:key", "value") + + assert memcached_client.get("namespace:key") == b"value" + assert adapter.get("key") == "value" + + +def test_set(memcached_client: Client): + adapter = MemcachedAdapter( + {"client_type": "client", "server": ("localhost", 11211)}, "namespace:", 60 + ) + + assert bool(memcached_client.get("namespace:key")) is False + + adapter.set("key", "value") + + memcached_client.touch("namespace:key") # Avoids flaky tests + assert bool(memcached_client.get("namespace:key")) is True + assert memcached_client.get("namespace:key") == b"value" + assert adapter.exists("key") + assert adapter.get("key") == "value" + + +def test_delete(memcached_client: Client): + adapter = MemcachedAdapter( + {"client_type": "client", "server": ("localhost", 11211)}, "namespace:", 60 + ) + + memcached_client.set("namespace:key", "value") + assert bool(memcached_client.get("namespace:key")) is True + + adapter.delete("key") + + adapter.get("key") + assert bool(memcached_client.get("namespace:key")) is False + + +def test_mget(memcached_client: Client): + adapter = MemcachedAdapter( + {"client_type": "client", "server": ("localhost", 11211)}, "namespace:", 60 + ) + + memcached_client.set("namespace:key1", "value1") + memcached_client.set("namespace:key2", "value2") + memcached_client.set("namespace:key3", "value3") + assert bool(memcached_client.get("namespace:key1")) is True + assert bool(memcached_client.get("namespace:key2")) is True + assert bool(memcached_client.get("namespace:key3")) is True + assert memcached_client.get("namespace:key1") == b"value1" + assert memcached_client.get("namespace:key2") == b"value2" + assert memcached_client.get("namespace:key3") == b"value3" + + result = adapter.mget(["key1", "key4", "key2"]) + + assert "key1" in result + assert result["key1"] == "value1" + assert "key4" in result + assert result["key4"] is None + assert "key2" in result + assert result["key2"] == "value2" + assert len(result) == 3 + + +def test_mset(memcached_client: Client): + adapter = MemcachedAdapter( + {"client_type": "client", "server": ("localhost", 11211)}, "namespace:", 60 + ) + + assert bool(memcached_client.get("namespace:key1")) is False + assert bool(memcached_client.get("namespace:key2")) is False + + adapter.mset({"key1": "value1", "key2": "value2"}) + + memcached_client.touch("namespace:ke1") # avoiding flaky tests + assert bool(memcached_client.get("namespace:key1")) is True + assert bool(memcached_client.get("namespace:key2")) is True + assert memcached_client.get("namespace:key1") == b"value1" + assert memcached_client.get("namespace:key2") == b"value2" + + +def test_flush(): + adapter = MemcachedAdapter( + {"client_type": "client", "server": ("localhost", 11211)}, "namespace:", 60 + ) + + adapter.set("key1", "value1") + adapter.mset({"key2": "value2", "key3": "value3"}) + assert adapter.get("key1") == "value1" + assert adapter.get("key2") == "value2" + assert adapter.get("key3") == "value3" + + adapter.flush() + assert not adapter.exists("key1") + assert not adapter.exists("key2") + assert not adapter.exists("key3") + assert adapter.get("key1") is None + assert adapter.get("key2") is None + assert adapter.get("key3") is None diff --git a/tests/test_adapters/test_memcached_adapter_hashclient.py b/tests/test_adapters/test_memcached_adapter_hashclient.py new file mode 100644 index 0000000..3f2a412 --- /dev/null +++ b/tests/test_adapters/test_memcached_adapter_hashclient.py @@ -0,0 +1,128 @@ +from typing import Iterator +from cache_tower.adapters.memcached_adapter import MemcachedAdapter +from pymemcache.client.hash import Client +import pytest + + +@pytest.fixture +def memcached_client() -> Iterator[Client]: + memcached_client = Client(("localhost", 11211)) + memcached_client.flush_all() + yield memcached_client + memcached_client.close() + + +def test_get(memcached_client: Client): + adapter = MemcachedAdapter( + {"client_type": "hash_client", "servers": [("localhost", 11211)]}, + "namespace:", + 60, + ) + + memcached_client.set("namespace:key", "value") + + assert memcached_client.get("namespace:key") == b"value" + assert adapter.get("key") == "value" + + +def test_set(memcached_client: Client): + adapter = MemcachedAdapter( + {"client_type": "hash_client", "servers": [("localhost", 11211)]}, + "namespace:", + 60, + ) + + assert bool(memcached_client.get("namespace:key")) is False + + adapter.set("key", "value") + + memcached_client.touch("namespace:key") # Avoids flaky tests + assert bool(memcached_client.get("namespace:key")) is True + assert memcached_client.get("namespace:key") == b"value" + assert adapter.exists("key") + assert adapter.get("key") == "value" + + +def test_delete(memcached_client: Client): + adapter = MemcachedAdapter( + {"client_type": "hash_client", "servers": [("localhost", 11211)]}, + "namespace:", + 60, + ) + + memcached_client.set("namespace:key", "value") + assert bool(memcached_client.get("namespace:key")) is True + + adapter.delete("key") + + adapter.get("key") + assert bool(memcached_client.get("namespace:key")) is False + + +def test_mget(memcached_client: Client): + adapter = MemcachedAdapter( + {"client_type": "hash_client", "servers": [("localhost", 11211)]}, + "namespace:", + 60, + ) + + memcached_client.set("namespace:key1", "value1") + memcached_client.set("namespace:key2", "value2") + memcached_client.set("namespace:key3", "value3") + assert bool(memcached_client.get("namespace:key1")) is True + assert bool(memcached_client.get("namespace:key2")) is True + assert bool(memcached_client.get("namespace:key3")) is True + assert memcached_client.get("namespace:key1") == b"value1" + assert memcached_client.get("namespace:key2") == b"value2" + assert memcached_client.get("namespace:key3") == b"value3" + + result = adapter.mget(["key1", "key4", "key2"]) + + assert "key1" in result + assert result["key1"] == "value1" + assert "key4" in result + assert result["key4"] is None + assert "key2" in result + assert result["key2"] == "value2" + assert len(result) == 3 + + +def test_mset(memcached_client: Client): + adapter = MemcachedAdapter( + {"client_type": "hash_client", "servers": [("localhost", 11211)]}, + "namespace:", + 60, + ) + + assert bool(memcached_client.get("namespace:key1")) is False + assert bool(memcached_client.get("namespace:key2")) is False + + adapter.mset({"key1": "value1", "key2": "value2"}) + + memcached_client.touch("namespace:ke1") # avoiding flaky tests + assert bool(memcached_client.get("namespace:key1")) is True + assert bool(memcached_client.get("namespace:key2")) is True + assert memcached_client.get("namespace:key1") == b"value1" + assert memcached_client.get("namespace:key2") == b"value2" + + +def test_flush(): + adapter = MemcachedAdapter( + {"client_type": "hash_client", "servers": [("localhost", 11211)]}, + "namespace:", + 60, + ) + + adapter.set("key1", "value1") + adapter.mset({"key2": "value2", "key3": "value3"}) + assert adapter.get("key1") == "value1" + assert adapter.get("key2") == "value2" + assert adapter.get("key3") == "value3" + + adapter.flush() + assert not adapter.exists("key1") + assert not adapter.exists("key2") + assert not adapter.exists("key3") + assert adapter.get("key1") is None + assert adapter.get("key2") is None + assert adapter.get("key3") is None diff --git a/tests/test_adapters/test_memcached_adapter_pooled_client.py b/tests/test_adapters/test_memcached_adapter_pooled_client.py new file mode 100644 index 0000000..1f786f4 --- /dev/null +++ b/tests/test_adapters/test_memcached_adapter_pooled_client.py @@ -0,0 +1,128 @@ +from typing import Iterator +from cache_tower.adapters.memcached_adapter import MemcachedAdapter +from pymemcache.client.base import Client +import pytest + + +@pytest.fixture +def memcached_client() -> Iterator[Client]: + memcached_client = Client(("localhost", 11211)) + memcached_client.flush_all() + yield memcached_client + memcached_client.close() + + +def test_get(memcached_client: Client): + adapter = MemcachedAdapter( + {"client_type": "pooled_client", "server": ("localhost", 11211)}, + "namespace:", + 60, + ) + + memcached_client.set("namespace:key", "value") + + assert memcached_client.get("namespace:key") == b"value" + assert adapter.get("key") == "value" + + +def test_set(memcached_client: Client): + adapter = MemcachedAdapter( + {"client_type": "pooled_client", "server": ("localhost", 11211)}, + "namespace:", + 60, + ) + + assert bool(memcached_client.get("namespace:key")) is False + + adapter.set("key", "value") + + memcached_client.touch("namespace:key") # Avoids flaky tests + assert bool(memcached_client.get("namespace:key")) is True + assert memcached_client.get("namespace:key") == b"value" + assert adapter.exists("key") + assert adapter.get("key") == "value" + + +def test_delete(memcached_client: Client): + adapter = MemcachedAdapter( + {"client_type": "pooled_client", "server": ("localhost", 11211)}, + "namespace:", + 60, + ) + + memcached_client.set("namespace:key", "value") + assert bool(memcached_client.get("namespace:key")) is True + + adapter.delete("key") + + adapter.get("key") + assert bool(memcached_client.get("namespace:key")) is False + + +def test_mget(memcached_client: Client): + adapter = MemcachedAdapter( + {"client_type": "pooled_client", "server": ("localhost", 11211)}, + "namespace:", + 60, + ) + + memcached_client.set("namespace:key1", "value1") + memcached_client.set("namespace:key2", "value2") + memcached_client.set("namespace:key3", "value3") + assert bool(memcached_client.get("namespace:key1")) is True + assert bool(memcached_client.get("namespace:key2")) is True + assert bool(memcached_client.get("namespace:key3")) is True + assert memcached_client.get("namespace:key1") == b"value1" + assert memcached_client.get("namespace:key2") == b"value2" + assert memcached_client.get("namespace:key3") == b"value3" + + result = adapter.mget(["key1", "key4", "key2"]) + + assert "key1" in result + assert result["key1"] == "value1" + assert "key4" in result + assert result["key4"] is None + assert "key2" in result + assert result["key2"] == "value2" + assert len(result) == 3 + + +def test_mset(memcached_client: Client): + adapter = MemcachedAdapter( + {"client_type": "pooled_client", "server": ("localhost", 11211)}, + "namespace:", + 60, + ) + + assert bool(memcached_client.get("namespace:key1")) is False + assert bool(memcached_client.get("namespace:key2")) is False + + adapter.mset({"key1": "value1", "key2": "value2"}) + + memcached_client.touch("namespace:ke1") # avoiding flaky tests + assert bool(memcached_client.get("namespace:key1")) is True + assert bool(memcached_client.get("namespace:key2")) is True + assert memcached_client.get("namespace:key1") == b"value1" + assert memcached_client.get("namespace:key2") == b"value2" + + +def test_flush(): + adapter = MemcachedAdapter( + {"client_type": "pooled_client", "server": ("localhost", 11211)}, + "namespace:", + 60, + ) + + adapter.set("key1", "value1") + adapter.mset({"key2": "value2", "key3": "value3"}) + assert adapter.get("key1") == "value1" + assert adapter.get("key2") == "value2" + assert adapter.get("key3") == "value3" + + adapter.flush() + assert not adapter.exists("key1") + assert not adapter.exists("key2") + assert not adapter.exists("key3") + assert adapter.get("key1") is None + assert adapter.get("key2") is None + assert adapter.get("key3") is None diff --git a/tests/test_adapters/test_memory_adapter.py b/tests/test_adapters/test_memory_adapter.py index 1cc98ec..aad994f 100644 --- a/tests/test_adapters/test_memory_adapter.py +++ b/tests/test_adapters/test_memory_adapter.py @@ -97,13 +97,15 @@ def test_mget(): adapter.set("key2", "value2") adapter.set("key3", "value3") - values = adapter.mget(["key1", "key2"]) + values = adapter.mget(["key1", "key4", "key2"]) assert "key1" in values assert "key2" in values + assert "key4" in values assert values["key1"] == "value1" assert values["key2"] == "value2" - assert len(values) == 2 + assert values["key4"] is None + assert len(values) == 3 def test_mset(): diff --git a/tests/test_adapters/test_redis_adapter.py b/tests/test_adapters/test_redis_adapter.py index 4e8e14b..671e636 100644 --- a/tests/test_adapters/test_redis_adapter.py +++ b/tests/test_adapters/test_redis_adapter.py @@ -75,13 +75,15 @@ def test_mget(redis_client: redis.Redis): assert redis_client.get("namespace:key2") == "value2" assert redis_client.get("namespace:key3") == "value3" - result = adapter.mget(["key1", "key2"]) + result = adapter.mget(["key1", "key4", "key2"]) assert "key1" in result assert result["key1"] == "value1" assert "key2" in result assert result["key2"] == "value2" - assert len(result) == 2 + assert "key4" in result + assert result["key4"] is None + assert len(result) == 3 def test_mset(redis_client: redis.Redis):