[Garbage collector] Additional tests

New tests:
* Tests based on the API, checking that files added with
  the /storage/add_* endpoints are scheduled for deletion.
* A test to check that a STORE message will dequeue the deletion
  of the underlying file.

Author: odesenfans

src/aleph/model/scheduled_deletions.py

@@ -22,6 +22,9 @@ def from_db(cls, db_value: Dict) -> "ScheduledDeletionInfo":
             object_id=db_value["_id"],
         )
 
+    def to_dict(self):
+        return {"filename": self.filename, "delete_by": self.delete_by}
+
 
 class ScheduledDeletion(BaseClass):
     COLLECTION = "scheduled_deletions"
@@ -31,7 +34,7 @@ class ScheduledDeletion(BaseClass):
 
     @classmethod
     async def insert(cls, scheduled_deletion: ScheduledDeletionInfo):
-        await cls.collection.insert_one(asdict(scheduled_deletion))
+        await cls.collection.insert_one(scheduled_deletion.to_dict())
 
     @classmethod
     async def files_to_delete(

tests/api/fixtures/fixture_messages.json

@@ -6,6 +6,7 @@
     "type": "POST",
     "time": 1652126646.5008686,
     "item_content": "{\"address\":\"0x696879aE4F6d8DaDD5b8F1cbb1e663B89b08f106\",\"time\":1652126646.5007327,\"content\":{\"title\":\"My first blog post using Aleph.im\",\"body\":\"Using Aleph.im, we can make a decentralized blog.\"},\"type\":\"test\"}",
+    "item_type": "inline",
     "item_hash": "4c33dd1ebf61bbb4342d8258b591fcd52cca73fd7c425542f78311d8f45ba274",
     "signature": "0x999ab556b92351e6edf894b4a67f01f0344c7023883eb5bafdf4cd0b98ca91781692ac6b95246c1bf940eedcedfd6dc04751accfbc417ee1b1ae13893634e7eb1c",
     "confirmed": false,
@@ -27,7 +28,8 @@
     "time": 1652126721.497669,
     "item_content": "{\"address\":\"0x696879aE4F6d8DaDD5b8F1cbb1e663B89b08f106\",\"time\":1652126721.4974446,\"item_type\":\"storage\",\"item_hash\":\"5ccdd7bccfbc5955e2e40166dd0cdea0b093154fd87bc2bea57e7c768cde2f21\",\"mime_type\":\"text/plain\"}",
     "item_hash": "2953f0b52beb79fc0ed1bc455346fdcb530611605e16c636778a0d673d7184af",
-    "signature": "0xa10129dd561c1bc93e8655daf09520e9f1694989263e25f330b403ad33563f4b64c9ae18f6cbfb33e8a47a095be7a181b140a369e6205fd04eef55397624a7121b",
+    "item_type": "inline",
+   "signature": "0xa10129dd561c1bc93e8655daf09520e9f1694989263e25f330b403ad33563f4b64c9ae18f6cbfb33e8a47a095be7a181b140a369e6205fd04eef55397624a7121b",
     "content": {
       "address": "0x696879aE4F6d8DaDD5b8F1cbb1e663B89b08f106",
       "time": 1652126721.4974446,

tests/conftest.py

@@ -1,30 +1,44 @@
-import pytest
-from aleph.model import init_db
-from aleph.config import get_defaults
-from configmanager import Config
 import pymongo
+import pytest
 import pytest_asyncio
+from configmanager import Config
 
+import aleph.config
+from aleph.config import get_defaults
+from aleph.model import init_db
 
 TEST_DB = "ccn_automated_tests"
 
 
+@pytest.fixture
+def mock_config():
+    config = Config(aleph.config.get_defaults())
+    # To test handle_new_storage
+    config.storage.store_files.value = True
+
+    # We set the global variable directly instead of patching it because of an issue
+    # with mocker.patch. mocker.patch uses hasattr to determine the properties of
+    # the mock, which does not work well with configmanager Config objects.
+    aleph.config.app_config = config
+    return config
+
+
 def drop_db(db_name: str, config: Config):
     client = pymongo.MongoClient(config.mongodb.uri.value)
     client.drop_database(db_name)
 
 
 @pytest_asyncio.fixture
-async def test_db():
+async def test_db(mock_config):
     """
     Initializes and cleans a MongoDB database dedicated to automated tests.
     """
 
-    config = Config(schema=get_defaults())
-    config.mongodb.database.value = TEST_DB
+    mock_config.mongodb.database.value = TEST_DB
 
-    drop_db(TEST_DB, config)
-    init_db(config, ensure_indexes=True)
+    drop_db(TEST_DB, mock_config)
+    init_db(mock_config, ensure_indexes=True)
 
     from aleph.model import db
+
     yield db

tests/garbage_collector/test_api_storage_and_gc.py

@@ -0,0 +1,92 @@
+"""
+Tests for the storage API to check that temporary files are properly marked for deletion.
+"""
+
+
+import datetime as dt
+import json
+
+import pytest
+import pytz
+from aiohttp import FormData
+from configmanager import Config
+
+from aleph.model import ScheduledDeletion
+from aleph.model.hashes import get_value as read_gridfs_file
+from aleph.web import create_app
+
+
+async def check_scheduled_deletion(
+    config: Config, file_hash: str, post_datetime: dt.datetime
+):
+    scheduled_deletion = await ScheduledDeletion.collection.find_one(
+        {"filename": file_hash}
+    )
+
+    assert scheduled_deletion is not None
+
+    # Check that the file is scheduled for deletion at least after
+    # the expected interval.
+    delete_interval = config.storage.delete_interval.value
+    delete_by = scheduled_deletion["delete_by"]
+    assert delete_by >= post_datetime + dt.timedelta(seconds=delete_interval)
+
+
+@pytest.mark.asyncio
+async def test_store_temporary_file(mock_config, test_db, aiohttp_client):
+    """
+    Checks that the garbage collector schedules temporary files uploaded
+    with /storage/add_file for deletion.
+    """
+
+    app = create_app()
+    app["config"] = mock_config
+    client = await aiohttp_client(app)
+
+    file_content = b"Some file I'd like to upload"
+
+    data = FormData()
+    data.add_field("file", file_content)
+
+    post_datetime = pytz.utc.localize(dt.datetime.utcnow())
+    response = await client.post(f"/api/v0/storage/add_file", data=data)
+    assert response.status == 200, await response.text()
+
+    data = await response.json()
+    assert data["status"] == "success"
+    file_hash = data["hash"]
+
+    db_content = await read_gridfs_file(file_hash)
+    assert db_content == file_content
+
+    await check_scheduled_deletion(mock_config, file_hash, post_datetime)
+
+
+@pytest.mark.asyncio
+async def test_store_temporary_json(mock_config, test_db, aiohttp_client):
+    """
+    Checks that the garbage collector schedules temporary JSON files uploaded
+    with /storage/add_json for deletion.
+    """
+
+    app = create_app()
+    app["config"] = mock_config
+    client = await aiohttp_client(app)
+
+    json_content = {
+        "title": "A garbage collector for CCNs",
+        "body": "Discover the new GC for Aleph CCNs. Deletes all the files, even useful ones!",
+    }
+
+    post_datetime = pytz.utc.localize(dt.datetime.utcnow())
+    response = await client.post(f"/api/v0/storage/add_json", json=json_content)
+    assert response.status == 200, await response.text()
+
+    data = await response.json()
+    assert data["status"] == "success"
+    file_hash = data["hash"]
+
+    db_content = await read_gridfs_file(file_hash)
+    assert json.loads(db_content) == json_content
+
+    await check_scheduled_deletion(mock_config, file_hash, post_datetime)

tests/garbage_collector/test_cancel_scheduled_deletion.py

@@ -0,0 +1,56 @@
+import datetime as dt
+import json
+
+import pytest
+
+from aleph.chains.common import process_one_message
+from aleph.model.hashes import (
+    get_value as read_gridfs_file,
+    set_value as store_gridfs_file,
+)
+from aleph.model.scheduled_deletions import ScheduledDeletionInfo, ScheduledDeletion
+
+
+@pytest.mark.asyncio
+async def test_cancel_scheduled_deletion(test_db):
+    """
+    Test that a file marked for deletion will be preserved once a message
+    stores that content.
+    """
+
+    store_message = {
+        "chain": "ETH",
+        "channel": "unit-tests",
+        "sender": "0x696879aE4F6d8DaDD5b8F1cbb1e663B89b08f106",
+        "type": "STORE",
+        "time": 1652126721.497669,
+        "item_content": '{"address":"0x696879aE4F6d8DaDD5b8F1cbb1e663B89b08f106","time":1652126721.4974446,"item_type":"storage","item_hash":"5ccdd7bccfbc5955e2e40166dd0cdea0b093154fd87bc2bea57e7c768cde2f21","mime_type":"text/plain"}',
+        "item_type": "inline",
+        "item_hash": "2953f0b52beb79fc0ed1bc455346fdcb530611605e16c636778a0d673d7184af",
+        "signature": "0xa10129dd561c1bc93e8655daf09520e9f1694989263e25f330b403ad33563f4b64c9ae18f6cbfb33e8a47a095be7a181b140a369e6205fd04eef55397624a7121b",
+    }
+
+    content = json.loads(store_message["item_content"])
+    file_hash = content["item_hash"]
+    file_content = b"Hello, Aleph.im!\n"
+
+    # Store the file
+    await store_gridfs_file(file_hash, file_content)
+    await ScheduledDeletion.insert(
+        ScheduledDeletionInfo(
+            filename=file_hash,
+            delete_by=dt.datetime.utcnow() + dt.timedelta(seconds=3600),
+        )
+    )
+
+    await process_one_message(store_message)
+
+    # Check that the file is no longer marked for deletion
+    scheduled_deletion = await ScheduledDeletion.collection.find_one(
+        {"filename": file_hash}
+    )
+    assert scheduled_deletion is None
+
+    # Check that the file is unmodified
+    db_file_content = await read_gridfs_file(file_hash)
+    assert db_file_content == file_content