From 63b3934bdb43e393f3948d18e4b28a6069e6af8e Mon Sep 17 00:00:00 2001 From: Daan Debie Date: Thu, 6 Sep 2018 22:24:36 +0200 Subject: [PATCH] Add HBase storage --- docs/user/usage.rst | 13 ++++++- machine/__about__.py | 2 +- machine/storage/backends/hbase.py | 61 +++++++++++++++++++++++++++++++ requirements-dev.txt | 4 +- setup.py | 3 +- tests/test_hbase_storage.py | 46 +++++++++++++++++++++++ 6 files changed, 124 insertions(+), 5 deletions(-) create mode 100644 machine/storage/backends/hbase.py create mode 100644 tests/test_hbase_storage.py diff --git a/docs/user/usage.rst b/docs/user/usage.rst index 621b62ed..83b9af81 100644 --- a/docs/user/usage.rst +++ b/docs/user/usage.rst @@ -114,14 +114,23 @@ Out of the box, Slack Machine provides 2 options for storage backend: *Class*: ``machine.storage.backends.redis.RedisStorage`` -So if, for example, you want to configure Slack Machine to use Redis as a storage backend, with your Redis +- **HBase**: this backend stores data in `HBase`_. HBase is a columnar store. This backend is for +advanced users only. You should only use it if you already have a HBase cluster running and cannot +use Redis for some reason. This backend requires 2 variables to be set in your ``local_settings.py``: +``HBASE_URL`` and ``HBASE_TABLE``. + + *Class*: ``machine.storage.backends.hbase.HBaseStorage`` + +So if, for example, you want to configure Slack Machine to use Redis as a storage backend, with your Redis instance running on *localhost* on the default port, you would add this to your ``local_settings.py``: .. code-block:: python - + STORAGE_BACKEND = 'machine.storage.backends.redis.RedisStorage' REDIS_URL = redis://localhost:6379' .. _Redis: https://redis.io/ +.. _HBase: https://hbase.apache.org/ + That's all there is to it! diff --git a/machine/__about__.py b/machine/__about__.py index cb476971..53fe39a1 100644 --- a/machine/__about__.py +++ b/machine/__about__.py @@ -7,7 +7,7 @@ __description__ = "A sexy, simple, yet powerful and extendable Slack bot" __uri__ = "https://github.com/DandyDev/slack-machine" -__version_info__ = (0, 15, 0) +__version_info__ = (0, 16, 0) __version__ = '.'.join(map(str, __version_info__)) __author__ = "Daan Debie" diff --git a/machine/storage/backends/hbase.py b/machine/storage/backends/hbase.py new file mode 100644 index 00000000..a22f44eb --- /dev/null +++ b/machine/storage/backends/hbase.py @@ -0,0 +1,61 @@ +from datetime import datetime, timedelta + +from happybase import Connection +from machine.storage.backends.base import MachineBaseStorage + + +def bytes_to_float(byte_arr): + s = byte_arr.decode('utf-8') + return float(s) + + +def float_to_bytes(i): + return bytes(str(i), 'utf-8') + + +class HBaseStorage(MachineBaseStorage): + + _VAL = b'values:value' + _EXP = b'values:expires_at' + _COLS = [_VAL, _EXP] + + def __init__(self, settings): + super().__init__(settings) + hbase_host = settings['HBASE_HOST'] + hbase_table = settings['HBASE_TABLE'] + self._connection = Connection(hbase_host) + self._table = self._connection.table(hbase_table) + + def _get_value(self, key): + row = self._table.row(key, self._COLS) + val = row.get(self._VAL) + if val: + exp = row.get(self._EXP) + if not exp: + return val + elif datetime.fromtimestamp(bytes_to_float(exp)) > datetime.utcnow(): + return val + else: + self.delete(key) + return None + return None + + def has(self, key): + val = self._get_value(key) + return bool(val) + + def get(self, key): + return self._get_value(key) + + def set(self, key, value, expires=None): + data = {self._VAL: value} + if expires: + expires_at = datetime.utcnow() + timedelta(seconds=expires) + data[self._EXP] = float_to_bytes(expires_at.timestamp()) + self._table.put(key, data) + + def delete(self, key): + self._table.delete(key) + + def size(self): + return 0 diff --git a/requirements-dev.txt b/requirements-dev.txt index 769b98ac..9acb2546 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,4 +10,6 @@ coverage==4.5.1 pytest-cov==2.6.0 Sphinx==1.7.9 sphinx-autobuild==0.7.1 -redis==2.10.6 \ No newline at end of file +redis==2.10.6 +Cython==0.28.5 +happybase==1.1.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 7df9eb9a..65477eeb 100644 --- a/setup.py +++ b/setup.py @@ -79,7 +79,8 @@ def run(self): install_requires=dependencies, python_requires='~=3.3', extras_require={ - 'redis': ['redis', 'hiredis'] + 'redis': ['redis', 'hiredis'], + 'hbase': ['Cython==0.28.5', 'happybase'] }, classifiers=[ "Development Status :: 3 - Alpha", diff --git a/tests/test_hbase_storage.py b/tests/test_hbase_storage.py new file mode 100644 index 00000000..7d064276 --- /dev/null +++ b/tests/test_hbase_storage.py @@ -0,0 +1,46 @@ +import pytest +from happybase import Table + +from machine.storage.backends.hbase import bytes_to_float, float_to_bytes, HBaseStorage + +_VAL = b'values:value' +_EXP = b'values:expires_at' +_COLS = [_VAL, _EXP] + + +@pytest.fixture +def table(mocker): + table = mocker.MagicMock(spec=Table) + ConnectionCls = mocker.patch('machine.storage.backends.hbase.Connection', autospec=True) + instance = ConnectionCls.return_value + instance.table.return_value = table + return table + + +@pytest.fixture +def hbase_storage(table): + return HBaseStorage({'HBASE_HOST': 'foo', 'HBASE_TABLE': 'bar'}) + + +def test_float_conversion(): + assert bytes_to_float(float_to_bytes(3.14159265359)) == 3.14159265359 + + +def test_get(table, hbase_storage): + hbase_storage.get('key1') + table.row.assert_called_with('key1', _COLS) + + +def test_has(table, hbase_storage): + hbase_storage.has('key1') + table.row.assert_called_with('key1', _COLS) + + +def test_delete(table, hbase_storage): + hbase_storage.delete('key1') + table.delete.assert_called_with('key1') + + +def test_set(table, hbase_storage): + hbase_storage.set('key1', 'val1') + table.put.assert_called_with('key1', {b'values:value': 'val1'})