Add Webhook-Ingest service using NATS JetStream (#61)

.github/labeler.yml

@@ -14,6 +14,10 @@ intelligence-service:
   - changed-files:
     - any-glob-to-any-file: "server/intelligence-service/**"
 
+webhook-ingest:
+  - changed-files:
+    - any-glob-to-any-file: "server/webhook-ingest/**"
+
 feature:
   - head-branch: ['^feature', 'feature', '^feat', 'feat']
 

.vscode/settings.json

@@ -6,6 +6,14 @@
 		"cn\\(([^)]*)\\)", "[\"'`]([^\"'`]*).*?[\"'`]"
 	],
 	"editor.codeActionsOnSave": {
-		"source.fixAll": true
-	}
+		"source.fixAll": "explicit"
+	},
+	"[python]": {
+    "editor.defaultFormatter": "ms-python.autopep8"
+  },
+  "python.testing.pytestArgs": ["."],
+  "python.testing.pytestEnabled": true,
+	"python.terminal.activateEnvironment": true,
+	"python.terminal.activateEnvInCurrentTerminal": true,
+	"python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python"
 }

project.code-workspace

@@ -0,0 +1,30 @@
+{
+  "folders": [
+    {
+      "name": "Hephaestus",
+      "path": "./"
+    },
+    {
+      "name": "webapp",
+      "path": "./webapp"
+    },
+    {
+      "name": "server/application-server",
+      "path": "./server/application-server"
+    },
+    {
+      "name": "server/intelligence-service",
+      "path": "./server/intelligence-service"
+    },
+    {
+      "name": "server/webhook-ingest",
+      "path": "./server/webhook-ingest"
+    },
+  ],
+  "settings": {
+    "java.compile.nullAnalysis.mode": "automatic",
+    "python.terminal.activateEnvironment": true,
+    "python.terminal.activateEnvInCurrentTerminal": true,
+    "python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python"
+  }
+}

server/webhook-ingest/Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.12 as requirements-stage
+
+WORKDIR /tmp
+
+RUN pip install poetry
+COPY ./pyproject.toml ./poetry.lock* /tmp/
+RUN poetry export -f requirements.txt --output requirements.txt --without-hashes
+
+FROM python:3.12
+
+WORKDIR /code
+
+COPY --from=requirements-stage /tmp/requirements.txt /code/requirements.txt
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+COPY ./app /code/app
+
+CMD ["fastapi", "run", "app/main.py", "--port", "4200"]

server/webhook-ingest/README.md

@@ -0,0 +1,192 @@
+# WebHook Ingest
+
+## Overview
+
+A service to ingest GitHub webhooks and publish the data to NATS JetStream.
+
+## Setup
+
+### Prerequisites
+
+- **Python 3.12**
+- **Poetry** for dependency management
+- **Docker** for containerization
+
+### Installation
+
+Install dependencies using Poetry:
+
+```bash
+pip install poetry
+poetry install
+```
+
+## Running the Service
+
+### Development
+
+```bash
+fastapi dev
+```
+
+### Production
+
+```bash
+fastapi run
+```
+
+## Docker Deployment
+
+Build and run with Docker Compose:
+
+```bash
+docker-compose up --build
+```
+
+Service ports:
+- **Webhook Service**: `4200`
+- **NATS Server**: `4222`
+
+## Environment Variables
+
+- `NATS_URL`: NATS server URL
+- `NATS_AUTH_TOKEN`: Authorization token for NATS server
+- `WEBHOOK_SECRET`: HMAC secret for verifying GitHub webhooks
+- `TLS_CERT_FILE`: Path to the TLS certificate file (used by NATS server)
+- `TLS_KEY_FILE`: Path to the TLS key file (used by NATS server)
+
+## Usage
+
+Configure your GitHub webhooks to POST to:
+
+```
+https://<server>:4200/github
+```
+
+### Event Handling
+
+Events are published to NATS with the subject:
+
+```
+github.<owner>.<repo>.<event_type>
+```
+
+## NATS Configuration with TLS
+
+
+
+You're absolutely right. The NATS configuration with TLS and Let's Encrypt, along with the corresponding environment variables, is crucial for ensuring secure communication and should be highlighted in the README. Here’s an updated version:
+
+---
+
+# WebHook Ingest
+
+## Overview
+
+A service to ingest GitHub webhooks and publish the data to NATS JetStream.
+
+## Setup
+
+### Prerequisites
+
+- **Python 3.12**
+- **Poetry** for dependency management
+- **Docker** for containerization
+
+### Installation
+
+Install dependencies using Poetry:
+
+```bash
+pip install poetry
+poetry install
+```
+
+## Running the Service
+
+### Development
+
+```bash
+fastapi dev
+```
+
+### Production
+
+```bash
+fastapi run
+```
+
+## Docker Deployment
+
+Build and run with Docker Compose:
+
+```bash
+docker-compose up --build
+```
+
+Service ports:
+
+- **Webhook Service**: `4200`
+- **NATS Server**: `4222`
+
+## Environment Variables
+
+- `NATS_URL`: NATS server URL
+- `NATS_AUTH_TOKEN`: Authorization token for NATS server
+- `WEBHOOK_SECRET`: HMAC secret for verifying GitHub webhooks
+- `TLS_CERT_FILE`: Path to the TLS certificate file (used by NATS server)
+- `TLS_KEY_FILE`: Path to the TLS key file (used by NATS server)
+
+## NATS Configuration with TLS
+
+For secure communication in production, NATS can be configured with TLS using Let's Encrypt certificates.
+
+### Steps to Create TLS Certificates
+
+1. **Install Certbot** on your server:
+
+```bash
+sudo apt-get install certbot
+```
+
+2. **Obtain a Certificate**:
+
+```bash
+sudo certbot certonly --standalone -d <your.domain.com>
+```
+
+Replace `<your.domain.com>` with your actual domain name.
+
+3. **Configure NATS** to use the certificate and key in the environment variables:
+
+```bash
+TLS_CERT_FILE=/etc/letsencrypt/live/<your.domain.com>/fullchain.pem
+TLS_KEY_FILE=/etc/letsencrypt/live/<your.domain.com>/privkey.pem
+
+NATS_URL=tls://<your.domain.com>
+```
+
+For more detailed instructions and options, refer to the [Certbot documentation](https://certbot.eff.org/).
+
+### NATS Authorization Token
+
+1. **Generate a Token**:
+
+```bash
+openssl rand -base64 32
+```
+
+2. **Set the Token** as an environment variable:
+
+```bash
+NATS_AUTH_TOKEN=<your_generated_token>
+```
+
+### Important Notes
+
+- The service automatically sets up a NATS JetStream stream named `github` to store events.
+- Ensure your firewall allows traffic on port 4222 (NATS) and ports 80/443 (Let's Encrypt challenge).
+- TLS is essential so no sensitive data can be intercepted during communication (such as webhook payloads).
+- Authentication tokens are crucial for securing the NATS server and ensuring only authorized clients can connect.
+- The webhook ingest service connects to the NATS server like any other client using the specified URL and token.
+- Allowing unauthenticated non-TLS connections from within the internal Docker network does not seem to be possible with the NATS server.

server/webhook-ingest/app/config.py

@@ -0,0 +1,12 @@
+from pydantic_settings import BaseSettings
+
+
+class Settings(BaseSettings):
+    NATS_URL: str = "localhost"
+    NATS_AUTH_TOKEN: str = ""
+    WEBHOOK_SECRET: str = ""
+
+    class Config:
+        env_file = ".env"
+
+settings = Settings()

server/webhook-ingest/app/logger.py

@@ -0,0 +1,11 @@
+import logging
+import sys
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+stream_handler = logging.StreamHandler(sys.stdout)
+log_formatter = logging.Formatter("%(levelname)s:\t  %(message)s")
+stream_handler.setFormatter(log_formatter)
+logger.addHandler(stream_handler)
+
+logger.info("Logger initialized")

server/webhook-ingest/app/main.py

@@ -0,0 +1,60 @@
+import hmac
+import hashlib
+from contextlib import asynccontextmanager
+from fastapi import Body, FastAPI, HTTPException, Header, Request
+from nats.js.api import StreamConfig
+from app.config import settings
+from app.nats_client import nats_client
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    await nats_client.connect()
+    await nats_client.js.add_stream(name="github", subjects=["github.>"], config=StreamConfig(storage="file"))
+    yield
+    await nats_client.close()
+
+
+app = FastAPI(lifespan=lifespan)
+
+
+def verify_github_signature(signature, secret, body):
+    mac = hmac.new(secret.encode(), body, hashlib.sha1)
+    expected_signature = "sha1=" + mac.hexdigest()
+    return hmac.compare_digest(signature, expected_signature)
+
+
+@app.post("/github")
+async def github_webhook(
+    request: Request, 
+    signature: str = Header(
+        None, 
+        alias="X-Hub-Signature", 
+        description="GitHub's HMAC hex digest of the payload, used for verifying the webhook's authenticity"
+    ), 
+    event_type: str = Header(
+        None, 
+        alias="X-Github-Event",
+        description="The type of event that triggered the webhook, such as 'push', 'pull_request', etc.",
+    ),
+    body = Body(...),
+):    
+    body = await request.body()
+
+    if not verify_github_signature(signature, settings.WEBHOOK_SECRET, body):
+        raise HTTPException(status_code=401, detail="Invalid signature")
+
+    # Ignore ping events
+    if event_type == "ping":
+        return { "status": "pong" }
+
+    # Extract subject from the payload
+    payload = await request.json()
+    owner = payload["repository"]["owner"]["login"]
+    repo = payload["repository"]["name"]
+    subject = f"github.{owner}.{repo}.{event_type}"
+
+    # Publish the payload to NATS JetStream
+    await nats_client.js.publish(subject, body)
+
+    return { "status": "ok" }