Merge pull request #258 from msqd/janitor

Janitor

docs/reference/apps/harp_apps.janitor.rst

@@ -0,0 +1,16 @@
+harp_apps.janitor
+=================
+
+.. automodule:: harp_apps.janitor
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Submodules
+----------
+
+.. toctree::
+    :maxdepth: 1
+
+    harp_apps.janitor.settings
+    harp_apps.janitor.worker

docs/reference/apps/harp_apps.janitor.settings.rst

@@ -0,0 +1,7 @@
+harp_apps.janitor.settings
+==========================
+
+.. automodule:: harp_apps.janitor.settings
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/reference/apps/harp_apps.janitor.worker.rst

@@ -0,0 +1,7 @@
+harp_apps.janitor.worker
+========================
+
+.. automodule:: harp_apps.janitor.worker
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/reference/apps/harp_apps.rst

@@ -15,6 +15,7 @@ Submodules
     harp_apps.contrib
     harp_apps.dashboard
     harp_apps.http_client
+    harp_apps.janitor
     harp_apps.proxy
     harp_apps.sqlalchemy_storage
     harp_apps.telemetry

docs/reference/apps/harp_apps.sqlalchemy_storage.models.metrics.rst

@@ -0,0 +1,7 @@
+harp_apps.sqlalchemy_storage.models.metrics
+===========================================
+
+.. automodule:: harp_apps.sqlalchemy_storage.models.metrics
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/reference/apps/harp_apps.sqlalchemy_storage.models.rst

@@ -16,6 +16,7 @@ Submodules
     harp_apps.sqlalchemy_storage.models.blobs
     harp_apps.sqlalchemy_storage.models.flags
     harp_apps.sqlalchemy_storage.models.messages
+    harp_apps.sqlalchemy_storage.models.metrics
     harp_apps.sqlalchemy_storage.models.tags
     harp_apps.sqlalchemy_storage.models.transactions
     harp_apps.sqlalchemy_storage.models.users

frontend/src/Domain/System/index.ts

@@ -1,3 +1,4 @@
 export { useSystemSettingsQuery } from "./useSystemSettingsQuery.ts"
 export { useSystemQuery } from "./useSystemQuery.ts"
 export { useSystemDependenciesQuery } from "./useSystemDependenciesQuery.ts"
+export { useSystemStorageQuery } from "./useSystemStorageQuery.ts"

frontend/src/Domain/System/useSystemStorageQuery.ts

@@ -0,0 +1,8 @@
+import { useQuery } from "react-query"
+
+import { useApi } from "Domain/Api"
+
+export function useSystemStorageQuery() {
+  const api = useApi()
+  return useQuery(["system", "storage"], () => api.fetch("/system/storage").then((r) => r.json()))
+}

frontend/src/Pages/System/SystemPage.tsx

@@ -3,6 +3,7 @@ import { Tab } from "mkui/Components/Tabs"
 
 import { SystemDependenciesTabPanel } from "./SystemDependenciesTabPanel.tsx"
 import { SystemSettingsTabPanel } from "./SystemSettingsTabPanel.tsx"
+import { SystemStorageTabPanel } from "./SystemStorageTabPanel.tsx"
 import { SystemTopologyTabPanel } from "./SystemTopologyTabPanel.tsx"
 
 export const SystemPage = () => {
@@ -12,11 +13,13 @@ export const SystemPage = () => {
         <Tab.List as="nav" aria-label="Tabs">
           <Tab>Topology</Tab>
           <Tab>Settings</Tab>
+          <Tab>Storage</Tab>
           <Tab>Dependencies</Tab>
         </Tab.List>
         <Tab.Panels>
           <SystemTopologyTabPanel />
           <SystemSettingsTabPanel />
+          <SystemStorageTabPanel />
           <SystemDependenciesTabPanel />
         </Tab.Panels>
       </Tab.Group>

frontend/src/Pages/System/SystemStorageTabPanel.tsx

@@ -0,0 +1,23 @@
+import { OnQuerySuccess } from "Components/Utilities/OnQuerySuccess.tsx"
+import { useSystemStorageQuery } from "Domain/System"
+import { KeyValueSettings, Setting } from "Domain/System/useSystemSettingsQuery.ts"
+import { Pane } from "mkui/Components/Pane"
+import { Tab } from "mkui/Components/Tabs"
+
+import { SettingsTable } from "./Components"
+
+export function SystemStorageTabPanel() {
+  const query = useSystemStorageQuery()
+
+  return (
+    <Tab.Panel>
+      <OnQuerySuccess query={query}>
+        {(query) => (
+          <Pane hasDefaultPadding={false}>
+            <SettingsTable settings={query.data as KeyValueSettings | Array<Setting>} />
+          </Pane>
+        )}
+      </OnQuerySuccess>
+    </Tab.Panel>
+  )
+}

frontend/src/Pages/System/__snapshots__/SystemPage.test.tsx.snap

@@ -33,7 +33,7 @@ exports[`SystemPage > renders the title and data when the query is successful 1`
             role="tablist"
           >
             <button
-              aria-controls="headlessui-tabs-panel-:r3:"
+              aria-controls="headlessui-tabs-panel-:r4:"
               aria-selected="true"
               class="focus:ring-0 focus:ring-offset-0 focus:outline-none"
               data-headlessui-state="selected"
@@ -49,7 +49,7 @@ exports[`SystemPage > renders the title and data when the query is successful 1`
               </span>
             </button>
             <button
-              aria-controls="headlessui-tabs-panel-:r4:"
+              aria-controls="headlessui-tabs-panel-:r5:"
               aria-selected="false"
               class="focus:ring-0 focus:ring-offset-0 focus:outline-none"
               data-headlessui-state=""
@@ -65,14 +65,30 @@ exports[`SystemPage > renders the title and data when the query is successful 1`
               </span>
             </button>
             <button
-              aria-controls="headlessui-tabs-panel-:r5:"
+              aria-controls="headlessui-tabs-panel-:r6:"
               aria-selected="false"
               class="focus:ring-0 focus:ring-offset-0 focus:outline-none"
               data-headlessui-state=""
               id="headlessui-tabs-tab-:r2:"
               role="tab"
               tabindex="-1"
               type="button"
+            >
+              <span
+                class="css-gebh7q"
+              >
+                Storage
+              </span>
+            </button>
+            <button
+              aria-controls="headlessui-tabs-panel-:r7:"
+              aria-selected="false"
+              class="focus:ring-0 focus:ring-offset-0 focus:outline-none"
+              data-headlessui-state=""
+              id="headlessui-tabs-tab-:r3:"
+              role="tab"
+              tabindex="-1"
+              type="button"
             >
               <span
                 class="css-gebh7q"
@@ -88,7 +104,7 @@ exports[`SystemPage > renders the title and data when the query is successful 1`
               aria-labelledby="headlessui-tabs-tab-:r0:"
               class="css-hu3rmt"
               data-headlessui-state="selected"
-              id="headlessui-tabs-panel-:r3:"
+              id="headlessui-tabs-panel-:r4:"
               role="tabpanel"
               tabindex="0"
             >
@@ -215,14 +231,21 @@ exports[`SystemPage > renders the title and data when the query is successful 1`
             </div>
             <span
               aria-labelledby="headlessui-tabs-tab-:r1:"
-              id="headlessui-tabs-panel-:r4:"
+              id="headlessui-tabs-panel-:r5:"
               role="tabpanel"
               style="position: fixed; top: 1px; left: 1px; width: 1px; height: 0px; padding: 0px; margin: -1px; overflow: hidden; clip: rect(0px, 0px, 0px, 0px); white-space: nowrap; border-width: 0px;"
               tabindex="-1"
             />
             <span
               aria-labelledby="headlessui-tabs-tab-:r2:"
-              id="headlessui-tabs-panel-:r5:"
+              id="headlessui-tabs-panel-:r6:"
+              role="tabpanel"
+              style="position: fixed; top: 1px; left: 1px; width: 1px; height: 0px; padding: 0px; margin: -1px; overflow: hidden; clip: rect(0px, 0px, 0px, 0px); white-space: nowrap; border-width: 0px;"
+              tabindex="-1"
+            />
+            <span
+              aria-labelledby="headlessui-tabs-tab-:r3:"
+              id="headlessui-tabs-panel-:r7:"
               role="tabpanel"
               style="position: fixed; top: 1px; left: 1px; width: 1px; height: 0px; padding: 0px; margin: -1px; overflow: hidden; clip: rect(0px, 0px, 0px, 0px); white-space: nowrap; border-width: 0px;"
               tabindex="-1"

frontend/src/Pages/Transactions/TransactionListOnQuerySuccess.tsx

@@ -29,7 +29,7 @@ export function TransactionListOnQuerySuccess({
   return (
     <div className="flex w-full items-start gap-x-8 relative">
       {isFiltersOpen ? (
-        <aside className="sticky top-8 hidden w-1/5 min-w-56 max-w-96 shrink-0 lg:block">
+        <aside className="sticky top-8 hidden w-1/5 min-w-40 max-w-60 2xl:min-w-52 2xl:max-w-72 shrink-0 lg:block">
           <div className="text-right">
             <FiltersHideButton onClick={() => setIsFiltersOpen(false)} />
           </div>

frontend/src/Pages/Transactions/__snapshots__/TransactionListPage.test.tsx.snap

@@ -33,7 +33,7 @@ exports[`renders well when the query is successful 1`] = `
             class="flex w-full items-start gap-x-8 relative"
           >
             <aside
-              class="sticky top-8 hidden w-1/5 min-w-56 max-w-96 shrink-0 lg:block"
+              class="sticky top-8 hidden w-1/5 min-w-40 max-w-60 2xl:min-w-52 2xl:max-w-72 shrink-0 lg:block"
             >
               <div
                 class="text-right"

harp/config/config.py

@@ -34,6 +34,7 @@ class Config:
         "harp_apps.dashboard",
         "harp_apps.sqlalchemy_storage",
         "harp_apps.telemetry",
+        "harp_apps.janitor",
         "harp_apps.contrib.sentry",  # todo: allow to extend application list in config file without overriding all
     ]
 

harp/config/factories/kernel_factory.py

@@ -49,7 +49,7 @@ async def build(self):
         self.configuration.validate()
 
         for application in self.configuration.applications:
-            logger.debug(f"📦 {application}")
+            logger.info(f"📦 {application}")
 
         dispatcher = self.build_event_dispatcher()
         container = self.build_container(dispatcher)

harp_apps/dashboard/controllers/__init__.py

@@ -96,7 +96,7 @@ def _create_internal_api_controller(self):
 
         self.children = [
             BlobsController(storage=self.storage, router=root.router),
-            SystemController(settings=self.global_settings, router=root.router),
+            SystemController(storage=self.storage, settings=self.global_settings, router=root.router),
             TransactionsController(storage=self.storage, router=root.router),
             OverviewController(storage=self.storage, router=root.router),
         ]

harp_apps/dashboard/controllers/system.py

@@ -1,24 +1,34 @@
 import re
 from copy import deepcopy
+from typing import cast
 
-from harp import __revision__, __version__
+from sqlalchemy import func, select
+from sqlalchemy.orm import aliased, joinedload
+
+from harp import __revision__, __version__, get_logger
 from harp.controllers import GetHandler, RouterPrefix, RoutingController
 from harp.http import HttpRequest
-from harp.typing.global_settings import GlobalSettings
+from harp.typing import GlobalSettings, Storage
 from harp.views.json import json
+from harp_apps.sqlalchemy_storage.models import MetricValue
+from harp_apps.sqlalchemy_storage.storage import SqlAlchemyStorage
 
 from ..utils.dependencies import get_python_dependencies, parse_dependencies
 
+logger = get_logger(__name__)
+
 
 @RouterPrefix("/api/system")
 class SystemController(RoutingController):
-    def __init__(self, *, settings: GlobalSettings, handle_errors=True, router=None):
+    def __init__(self, *, storage: Storage, settings: GlobalSettings, handle_errors=True, router=None):
         # a bit of scrambling for passwords etc.
         if "storage" in settings:
             if "url" in settings["storage"]:
                 settings["storage"]["url"] = re.sub(r"//[^@]*@", "//***@", settings["storage"]["url"])
 
         self.settings = deepcopy(dict(settings))
+        self.storage: SqlAlchemyStorage = cast(SqlAlchemyStorage, storage)
+
         self._dependencies = None
 
         super().__init__(handle_errors=handle_errors, router=router)
@@ -43,6 +53,25 @@ async def get_settings(self):
     async def get_dependencies(self):
         return json({"python": await self.__get_cached_python_dependencies()})
 
+    @GetHandler("/storage")
+    async def get_storage(self):
+        subquery = select(
+            func.rank().over(order_by=MetricValue.created_at.desc(), partition_by=MetricValue.metric_id).label("rank"),
+            MetricValue,
+        ).subquery()
+        v = aliased(MetricValue, subquery)
+        query = select(v).where(subquery.c.rank == 1).options(joinedload(v.metric))
+
+        async with self.storage.session() as session:
+            result = (await session.execute(query)).scalars().all()
+
+        return json(
+            {
+                "settings": self.settings.get("storage", {}),
+                "counts": {value.metric.name.split(".", 1)[-1]: value.value for value in result},
+            }
+        )
+
     async def __get_cached_python_dependencies(self):
         if self._dependencies is None:
             self._dependencies = parse_dependencies(await get_python_dependencies())

harp_apps/janitor/__app__.py

@@ -0,0 +1,18 @@
+import asyncio
+
+from harp.config import Application
+from harp.config.events import FactoryBindEvent, FactoryBoundEvent
+from harp.typing import Storage
+from harp_apps.janitor.worker import JanitorWorker
+
+
+class JanitorApplication(Application):
+    async def on_bind(self, event: FactoryBindEvent):
+        pass
+
+    async def on_bound(self, event: FactoryBoundEvent):
+        self.worker = JanitorWorker(event.provider.get(Storage))
+        self.worker_task = asyncio.create_task(self.worker.run())
+
+    async def unmount(self):
+        self.worker.stop()

harp_apps/janitor/settings.py

@@ -0,0 +1,7 @@
+from datetime import timedelta
+
+#: How many seconds to wait between loop executions.
+PERIOD = 600
+
+#: Number of days after we consider something as "old".
+OLD_AFTER = timedelta(days=60)

harp_apps/janitor/tests/test_worker.py

@@ -0,0 +1,56 @@
+from datetime import UTC, datetime, timedelta
+
+from harp_apps.janitor.settings import OLD_AFTER
+from harp_apps.janitor.worker import JanitorWorker
+from harp_apps.sqlalchemy_storage.storage import SqlAlchemyStorage
+from harp_apps.sqlalchemy_storage.utils.testing.mixins import SqlalchemyStorageTestFixtureMixin
+
+
+class TestJanitorWorker(SqlalchemyStorageTestFixtureMixin):
+    async def test_delete_old_transactions(self, storage: SqlAlchemyStorage):
+        worker = JanitorWorker(storage)
+
+        await self.create_transaction(
+            storage, started_at=datetime.now(UTC).replace(tzinfo=None) - OLD_AFTER - timedelta(hours=1)
+        )
+        await self.create_transaction(
+            storage, started_at=datetime.now(UTC).replace(tzinfo=None) - OLD_AFTER - timedelta(minutes=1)
+        )
+        await self.create_transaction(
+            storage, started_at=datetime.now(UTC).replace(tzinfo=None) - OLD_AFTER + timedelta(minutes=1)
+        )
+
+        async with storage.session() as session:
+            assert (await worker.compute_metrics(session))["storage.transactions"] == 3
+
+        await worker.delete_old_transactions()
+
+        async with storage.session() as session:
+            assert (await worker.compute_metrics(session))["storage.transactions"] == 1
+
+    async def test_delete_orphan_blobs(self, storage: SqlAlchemyStorage):
+        worker = JanitorWorker(storage)
+
+        b1 = await self.create_blob(storage, "foo")
+        b2 = await self.create_blob(storage, "bar")
+        await self.create_blob(storage, "baz")
+
+        async with storage.session() as session:
+            metrics = await worker.compute_metrics(session)
+            assert metrics["storage.blobs"] == 3
+            assert metrics["storage.blobs.orphans"] == 3
+
+        t = await self.create_transaction(storage)
+        await self.create_message(storage, transaction_id=t.id, kind="misc", summary="foo", headers=b1.id, body=b2.id)
+
+        async with storage.session() as session:
+            metrics = await worker.compute_metrics(session)
+            assert metrics["storage.blobs"] == 3
+            assert metrics["storage.blobs.orphans"] == 1
+
+        await worker.delete_orphan_blobs()
+
+        async with storage.session() as session:
+            metrics = await worker.compute_metrics(session)
+            assert metrics["storage.blobs"] == 2
+            assert metrics["storage.blobs.orphans"] == 0