Add Webhook-Ingest service using NATS JetStream

.github/labeler.yml

@@ -14,6 +14,10 @@ intelligence-service:
   - changed-files:
     - any-glob-to-any-file: "server/intelligence-service/**"
 
+webhook-ingest:
+  - changed-files:
+    - any-glob-to-any-file: "server/webhook-ingest/**"
+
 feature:
   - head-branch: ['^feature', 'feature', '^feat', 'feat']
 

.vscode/settings.json

@@ -6,6 +6,14 @@
 		"cn\\(([^)]*)\\)", "[\"'`]([^\"'`]*).*?[\"'`]"
 	],
 	"editor.codeActionsOnSave": {
-		"source.fixAll": true
-	}
+		"source.fixAll": "explicit"
+	},
+	"[python]": {
+    "editor.defaultFormatter": "ms-python.autopep8"
+  },
+  "python.testing.pytestArgs": ["."],
+  "python.testing.pytestEnabled": true,
+	"python.terminal.activateEnvironment": true,
+	"python.terminal.activateEnvInCurrentTerminal": true,
+	"python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python"
 }

project.code-workspace

@@ -0,0 +1,30 @@
+{
+  "folders": [
+    {
+      "name": "Hephaestus",
+      "path": "./"
+    },
+    {
+      "name": "webapp",
+      "path": "./webapp"
+    },
+    {
+      "name": "server/application-server",
+      "path": "./server/application-server"
+    },
+    {
+      "name": "server/intelligence-service",
+      "path": "./server/intelligence-service"
+    },
+    {
+      "name": "server/webhook-ingest",
+      "path": "./server/webhook-ingest"
+    },
+  ],
+  "settings": {
+    "java.compile.nullAnalysis.mode": "automatic",
+    "python.terminal.activateEnvironment": true,
+    "python.terminal.activateEnvInCurrentTerminal": true,
+    "python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python"
+  }
+}

server/webhook-ingest/Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.9 as requirements-stage
+
+WORKDIR /tmp
+
+RUN pip install poetry
+COPY ./pyproject.toml ./poetry.lock* /tmp/
+RUN poetry export -f requirements.txt --output requirements.txt --without-hashes
+
+FROM python:3.9
+
+WORKDIR /code
+
+COPY --from=requirements-stage /tmp/requirements.txt /code/requirements.txt
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+COPY ./app /code/app
+
+CMD ["fastapi", "run", "app/main.py", "--port", "4200"]

server/webhook-ingest/README.md

@@ -0,0 +1,69 @@
+# WebHook Ingest
+
+## Overview
+
+A service to ingest GitHub webhooks and publish the data to NATS JetStream.
+
+## Setup
+
+### Prerequisites
+
+- **Python 3.9+**
+- **Poetry** for dependency management
+- **Docker** for containerization
+
+### Installation
+
+Install dependencies using Poetry:
+
+```bash
+pip install poetry
+poetry install
+```
+
+## Running the Service
+
+### Development
+
+```bash
+fastapi dev
+```
+
+### Production
+
+```bash
+fastapi run
+```
+
+## Docker Deployment
+
+Build and run with Docker Compose:
+
+```bash
+docker-compose up --build
+```
+
+Service ports:
+- **Webhook Service**: `4200`
+- **NATS Server**: `4222`
+
+## Environment Variables
+
+- `NATS_URL`: NATS server URL
+- `SECRET`: HMAC secret for verifying GitHub webhooks
+
+## Usage
+
+Configure your GitHub webhooks to POST to:
+
+```
+http://<server>:4200/github
+```
+
+### Event Handling
+
+Events are published to NATS with the subject:
+
+```
+github.<owner>.<repo>.<event_type>
+```

server/webhook-ingest/app/config.py

@@ -0,0 +1,10 @@
+from pydantic_settings import BaseSettings
+
+class Settings(BaseSettings):
+    NATS_URL: str = "localhost"
+    SECRET: str = ""
+
+    class Config:
+        env_file = ".env"
+
+settings = Settings()

server/webhook-ingest/app/main.py

@@ -0,0 +1,60 @@
+import hmac
+import hashlib
+from contextlib import asynccontextmanager
+from fastapi import Body, FastAPI, HTTPException, Header, Request
+from nats.js.api import StreamConfig
+from app.config import settings
+from app.nats_client import nats_client
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    await nats_client.connect()
+    await nats_client.js.add_stream(name="github", subjects=["github.>"], config=StreamConfig(storage="file"))
+    yield
+    await nats_client.close()
+
+
+app = FastAPI(lifespan=lifespan)
+
+
+def verify_github_signature(signature, secret, body):
+    mac = hmac.new(secret.encode(), body, hashlib.sha1)
+    expected_signature = "sha1=" + mac.hexdigest()
+    return hmac.compare_digest(signature, expected_signature)
+
+
+@app.post("/github")
+async def github_webhook(
+    request: Request, 
+    signature: str = Header(
+        None, 
+        alias="X-Hub-Signature", 
+        description="GitHub's HMAC hex digest of the payload, used for verifying the webhook's authenticity"
+    ), 
+    event_type: str = Header(
+        None, 
+        alias="X-Github-Event",
+        description="The type of event that triggered the webhook, such as 'push', 'pull_request', etc.",
+    ),
+    body = Body(...),
+):    
+    body = await request.body()
+
+    if not verify_github_signature(signature, settings.SECRET, body):
+        raise HTTPException(status_code=401, detail="Invalid signature")
+
+    # Ignore ping events
+    if event_type == "ping":
+        return { "status": "pong" }
+
+    # Extract subject from the payload
+    payload = await request.json()
+    owner = payload["repository"]["owner"]["login"]
+    repo = payload["repository"]["name"]
+    subject = f"github.{owner}.{repo}.{event_type}"
+
+    # Publish the payload to NATS JetStream
+    await nats_client.js.publish(subject, body)
+
+    return { "status": "ok" }

server/webhook-ingest/app/nats_client.py

@@ -0,0 +1,21 @@
+import nats
+from .config import settings
+
+
+class NATSClient:
+    def __init__(self, nats_url: str):
+        self.nats_url = nats_url
+
+    async def connect(self):
+        self.nc = await nats.connect(self.nats_url)
+        self.js = self.nc.jetstream()
+
+    async def publish(self, subject: str, message: bytes):
+        ack = await self.js.publish(subject, message)
+        print(ack)
+
+    async def close(self):
+        await self.nc.close()
+
+
+nats_client = NATSClient(settings.NATS_URL)

server/webhook-ingest/compose.yaml

@@ -0,0 +1,28 @@
+services:
+  webhook-ingest:
+    build: .
+    ports:
+      - "4200"
+    environment:
+      - NATS_URL=nats://nats-server:4222
+      - SECRET=${SECRET}
+    depends_on:
+      - nats-server
+    networks:
+      - common-network
+
+  nats-server:
+    image: nats:latest
+    ports:
+      - "4222"
+    command: "-js"
+    volumes:
+      - nats_data:/data
+    networks:
+      - common-network
+
+networks:
+  common-network:
+
+volumes:
+  nats_data: