WIP feat(redis): redis client and caching logic

indexd/app.py

@@ -1,4 +1,6 @@
 import flask
+import redis
+from urllib.parse import urlparse
 from .bulk.blueprint import blueprint as indexd_bulk_blueprint
 from .index.blueprint import blueprint as indexd_index_blueprint
 from .alias.blueprint import blueprint as indexd_alias_blueprint
@@ -48,5 +50,20 @@ def get_app(settings=None):
             pass
 
     app_init(app, settings)
+    _setup_redis_client(app)
 
     return app
+
+
+def _setup_redis_client(app):
+    """
+    Sets up the redis client based on config
+    """
+    redis_url_parts = urlparse(app.config["REDIS_HOST"])
+    ssl = redis_url_parts.scheme == "https"
+    app.redis_client = redis.Redis(
+        host=redis_url_parts.netloc,
+        port=app.config["REDIS_PORT"],
+        db=app.config["REDIS_DB"],
+        ssl=ssl,
+    )

indexd/default_settings.py

@@ -57,4 +57,8 @@
 
 AUTH = SQLAlchemyAuthDriver("sqlite:///auth.sq3")
 
+CONFIG["REDIS_HOST"] = "http://redis-service"
+CONFIG["REDIS_PORT"] = "6379"
+CONFIG["REDIS_DB"] = 0
+
 settings = {"config": CONFIG, "auth": AUTH}

indexd/index/blueprint.py

@@ -220,9 +220,19 @@ def get_index_record(record):
     Returns a record.
     """
 
-    ret = blueprint.index_driver.get(record)
+    # check redis
+    json_record = flask.current_app.redis_client.get(record)
+    if json_record:
+        json_record = json.loads(json_record)
+    else:
+        # get from db
+        json_record = blueprint.index_driver.get(record)
 
-    return flask.jsonify(ret), 200
+        # update redis
+        logger.info(f"setting redis record {record}")
+        flask.current_app.redis_client.set(record, json.dumps(json_record))
+
+    return flask.jsonify(json_record), 200
 
 
 @blueprint.route("/index/", methods=["POST"])
@@ -311,6 +321,10 @@ def put_index_blank_record(record):
     )
     ret = {"did": did, "rev": rev, "baseid": baseid}
 
+    # invalidate redis for this did
+    logger.info(f"removing redis record {record}")
+    flask.current_app.redis_client.delete(did)
+
     return flask.jsonify(ret), 200
 
 
@@ -330,6 +344,10 @@ def put_index_record(record):
 
     ret = {"did": did, "baseid": baseid, "rev": rev}
 
+    # invalidate redis for this did
+    logger.info(f"removing redis record {record}")
+    flask.current_app.redis_client.delete(did)
+
     return flask.jsonify(ret), 200
 
 
@@ -345,6 +363,10 @@ def delete_index_record(record):
     # authorize done in delete
     blueprint.index_driver.delete(record, rev)
 
+    # invalidate redis for this did
+    logger.info(f"removing redis record {record}")
+    flask.current_app.redis_client.delete(record)
+
     return "", 200
 
 
@@ -388,6 +410,10 @@ def add_index_record_version(record):
 
     ret = {"did": did, "baseid": baseid, "rev": rev}
 
+    # invalidate redis for this did
+    logger.info(f"removing redis record {record}")
+    flask.current_app.redis_client.delete(did)
+
     return flask.jsonify(ret), 200
 
 
@@ -425,6 +451,11 @@ def append_aliases(record):
 
     aliases = blueprint.index_driver.get_aliases_for_did(record)
     aliases_payload = {"aliases": [{"value": alias} for alias in aliases]}
+
+    # invalidate redis for this did
+    logger.info(f"removing redis record {record}")
+    flask.current_app.redis_client.delete(record)
+
     return flask.jsonify(aliases_payload), 200
 
 
@@ -448,6 +479,11 @@ def replace_aliases(record):
     blueprint.index_driver.replace_aliases_for_did(aliases, record)
 
     aliases_payload = {"aliases": [{"value": alias} for alias in aliases]}
+
+    # invalidate redis for this did
+    logger.info(f"removing redis record {record}")
+    flask.current_app.redis_client.delete(record)
+
     return flask.jsonify(aliases_payload), 200
 
 
@@ -456,6 +492,10 @@ def delete_all_aliases(record):
     # authorization and error handling done in driver
     blueprint.index_driver.delete_all_aliases_for_did(record)
 
+    # invalidate redis for this did
+    logger.info(f"removing redis record {record}")
+    flask.current_app.redis_client.delete(record)
+
     return flask.jsonify("Aliases deleted successfully"), 200
 
 
@@ -464,6 +504,10 @@ def delete_one_alias(record, alias):
     # authorization and error handling done in driver
     blueprint.index_driver.delete_one_alias_for_did(alias, record)
 
+    # invalidate redis for this did
+    logger.info(f"removing redis record {record}")
+    flask.current_app.redis_client.delete(record)
+
     return flask.jsonify("Aliases deleted successfully"), 200
 
 

requirements.txt

@@ -10,3 +10,4 @@ git+https://github.com/uc-cdis/[email protected]#egg=dosclient
 git+https://github.com/uc-cdis/[email protected]#egg=hsclient
 authutils==4.0.0
 gen3rbac==0.1.2
+redis==3.4.1

setup.py

@@ -18,6 +18,7 @@
         "hsclient",
         "authutils",
         "gen3rbac",
+        "redis~=3.4.1",
     ],
     dependency_links=[
         "git+https://github.com/uc-cdis/[email protected]#egg=cdislogging",

tests/default_test_settings.py

@@ -20,6 +20,10 @@
 
 os.environ["PRESIGNED_FENCE_URL"] = "https://fictitious-commons.io/"
 os.environ["HOSTNAME"] = "fictitious-commons.io"
+CONFIG["REDIS_HOST"] = "http://redis-service"
+CONFIG["REDIS_PORT"] = "6379"
+CONFIG["REDIS_DB"] = 0
+
 settings = {"config": CONFIG, "auth": AUTH}
 
 settings["config"]["TEST_DB"] = "postgres://postgres@localhost/test_migration_db"