Merge pull request #76 from CentreForDigitalHumanities/redux

Version 3.0

.github/workflows/build-images.yaml

@@ -0,0 +1,87 @@
+name: Build and Push Docker Images
+
+on:
+  push:
+    tags:
+      - '*'
+  workflow_dispatch:
+
+env:
+  AGENT_IMAGE_NAME: humitifier
+  AGENT_DOCKERFILE_PATH: ./agent/Dockerfile
+  AGENT_CONTEXT_PATH: ./agent
+  SERVER_IMAGE_NAME: humitifier-server
+  SERVER_DOCKERFILE_PATH: ./humitifier-server/Dockerfile
+  SERVER_CONTEXT_PATH: ./humitifier-server
+
+jobs:
+  build-and-push-agent:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push main image
+        uses: docker/build-push-action@v6
+        with:
+          context: ${{ env.AGENT_CONTEXT_PATH }}
+          file: ${{ env.AGENT_DOCKERFILE_PATH }}
+          push: true
+          tags: |
+            ghcr.io/centrefordigitalhumanities/humitifier/${{ env.AGENT_IMAGE_NAME }}:${{ github.ref_name }}
+
+#            ghcr.io/centrefordigitalhumanities/humitifier/${{ env.AGENT_IMAGE_NAME }}:latest
+
+      - name: Grype Scan
+        id: scan
+        uses: anchore/scan-action@v3
+        with:
+          image: ghcr.io/centrefordigitalhumanities/humitifier/${{ env.AGENT_IMAGE_NAME }}:${{ github.ref_name }}
+          fail-build: false
+
+      - name: upload Grype SARIF report
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: ${{ steps.scan.outputs.sarif }}
+
+  build-and-push-server:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push main image
+        uses: docker/build-push-action@v6
+        with:
+          context: ${{ env.SERVER_CONTEXT_PATH }}
+          file: ${{ env.SERVER_DOCKERFILE_PATH }}
+          push: true
+          tags: |
+            ghcr.io/centrefordigitalhumanities/humitifier/${{ env.SERVER_IMAGE_NAME }}:${{ github.ref_name }}
+#            ghcr.io/centrefordigitalhumanities/humitifier/${{ env.SERVER_IMAGE_NAME }}:latest
+
+      - name: Grype Scan
+        id: scan
+        uses: anchore/scan-action@v3
+        with:
+          image: ghcr.io/centrefordigitalhumanities/humitifier/${{ env.SERVER_IMAGE_NAME }}:${{ github.ref_name }}
+          fail-build: false
+
+      - name: upload Grype SARIF report
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: ${{ steps.scan.outputs.sarif }}

.gitignore

@@ -10,6 +10,8 @@ data/
 docker-compose.yaml
 .local/
 .envrc
+.idea
+node_modules/
 
 # Supabase
 .branches

Dockerfile → agent/Dockerfile

@@ -17,7 +17,6 @@ FROM python:3.11.9-slim
 WORKDIR /app
 
 COPY --from=builder /app .
-COPY static/ static
 COPY supabase/migrations /migrations
 COPY entrypoint/main.py entrypoint.py
 

docker-compose.example.yaml → agent/docker-compose.example.yaml

@@ -1,9 +1,8 @@
-version: '3'
 services:
   web:
     image: humitifier:dev
-    ports:
-      - "8000:8000"
+#    ports:
+#      - "8000:8000"
     environment:
       - SSH_AUTH_SOCK=/ssh-agent
       - HUMITIFIER_CONFIG=/code/app_config.toml

agent/entrypoint/main.py

@@ -0,0 +1,17 @@
+import asyncio
+from humitifier.tasks import app as task_app
+from humitifier.logging import logging
+
+
+
+async def main():
+    "Run scheduler"
+    logging.info("Starting scheduler")
+
+    sched = asyncio.create_task(task_app.serve())
+
+    await asyncio.wait([sched])
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

humitifier/config.py → agent/humitifier/config.py

@@ -15,6 +15,7 @@ class Config:
     inventory: list[str]
     pssh: dict
     tasks: dict[str, str]
+    upload_endpoint: str
 
     @classmethod
     def load(cls) -> "Config":

humitifier/tasks.py → agent/humitifier/tasks.py

@@ -1,7 +1,9 @@
+import asyncio
 from sys import stdout
 
 import asyncpg
 import json
+import requests
 import time
 from datetime import datetime
 from rocketry import Rocketry
@@ -10,10 +12,8 @@
 from pssh.output import HostOutput
 from humitifier import facts
 from humitifier.config import CONFIG, Config
-from humitifier.dashboard import template_env, host_filters
 from humitifier.logging import logging
 from humitifier.utils import FactError
-from humitifier.models import get_hosts
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -68,7 +68,6 @@ async def sync_hosts():
 
 @app.task(hourly)
 async def scan_hosts():
-    app.session
     logger.info("Initiating scan of hosts")
     ts = datetime.now()
     conn = await asyncpg.connect(CONFIG.db)
@@ -114,6 +113,29 @@ async def parse_facts():
     conn = await asyncpg.connect(CONFIG.db)
     rows = await conn.fetch("""SELECT name, host, scan, stdout, stderr, exception, exit_code FROM host_outputs""")
     parsed_rows = [(row["name"], row["host"], row["scan"], parse_row_data(row)) for row in rows]
+
+    hosts = {row["host"] for row in rows}
+    host_data = []
+
+    for host in hosts:
+        host_data.append(
+            {
+                "host": host,
+                "data": {
+                    name: parsed.to_sql()
+                    for name, this_host, scan, parsed in parsed_rows
+                    if this_host == host
+                },
+            }
+        )
+
+    await asyncio.to_thread(
+        requests.post,
+        CONFIG.upload_endpoint,
+        data=json.dumps(host_data),
+        headers={'Content-Type': 'application/json'}
+    )
+
     await conn.executemany(
         """INSERT INTO facts(name, host, scan, data) VALUES($1, $2, $3, $4)""",
         [(name, host, scan, json.dumps(parsed.to_sql())) for name, host, scan, parsed in parsed_rows],
@@ -151,20 +173,3 @@ async def cleanup_db():
     await conn.execute("VACUUM;")
 
     await conn.close()
-
-
-@app.task(after_success(cleanup_db))
-async def pre_render_index():
-    logger.info("Pre-rendering index")
-    template = template_env.get_template("page_index.jinja2")
-    hosts = await get_hosts()
-    filters = host_filters(None, hosts)
-    html = template.render(
-        current_hosts=hosts,
-        critical_count=len([h for h in hosts if h.severity == "critical"]),
-        warning_count=len([h for h in hosts if h.severity == "warning"]),
-        info_count=len([h for h in hosts if h.severity == "info"]),
-        filters=filters,
-    )
-    with open("static/index_prerender.html", "w") as out:
-        out.write(html)