Update Sentryflow

- Initial Uploads

.github/workflows/ci-test-py.yml

@@ -0,0 +1,52 @@
+name: ci-test-py
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  py-pip-ai-sentryflow:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+
+      - name: check Python pip3 requirements
+        run: |
+          pip install -r requirements.txt
+        working-directory: ai-engine
+
+  py-lint-ai-sentryflow:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+        working-directory: ai-engine
+
+      - name: Lint with Ruff
+        run: |
+          pip install ruff
+          ruff --output-format=github .
+        continue-on-error: true
+        working-directory: ai-engine
+
+  py-pep8-ai-sentryflow:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: 'Run PEP8'
+        uses: quentinguidee/pep8-action@v1
+        with:
+          arguments: '--max-line-length=120'    

.github/workflows/sentryflow-pr-checks.yml

@@ -24,7 +24,13 @@ jobs:
           echo "tag=tmp" >> $GITHUB_OUTPUT
         fi
 
-    - name: Build Docker Image
+    - name: Build SentryFlow Docker Image
       working-directory: ./sentryflow
       run: |
          make TAG=${{ steps.tag.outputs.tag }} image
+
+    - name: Build SentryFlow AI Engine Docker Image
+      working-directory: ./ai-engine
+      run: |
+         make TAG=${{ steps.tag.outputs.tag }} build
+         

.github/workflows/sentryflow-release-image.yml

@@ -30,11 +30,16 @@ jobs:
           echo "tag=tmp" >> $GITHUB_OUTPUT
         fi
 
-    - name: Build Docker Image
+    - name: Build SentryFlow Docker Image
       working-directory: ./sentryflow
       run: |
          make TAG=${{ steps.tag.outputs.tag }} image
 
+    - name: Build SentryFlow AI Engine Docker Image
+      working-directory: ./ai-engine
+      run: |
+         make TAG=${{ steps.tag.outputs.tag }} build
+
     # - name: Push Docker Image
     #   run: |
     #      docker push boanlab/sentryflow:${{ steps.tag.outputs.tag }}

README.md

@@ -1,13 +1,12 @@
-
 # SentryFlow
 
-[![SentryFlow Docker Build](https://github.com/5GSEC/sentryflow/actions/workflows/sentryflow-release-image.yml/badge.svg)](https://github.com/5GSEC/sentryflow/actions/workflows/sentryflow-release-image.yml) [![CI Test](https://github.com/5GSEC/sentryflow/actions/workflows/ci-test-go.yml/badge.svg)](https://github.com/5GSEC/sentryflow/actions/workflows/ci-test-go.yml)
+[![SentryFlow Docker Build](https://github.com/boanlab/numbat/actions/workflows/sentryflow-release-image.yml/badge.svg)](https://github.com/boanlab/numbat/actions/workflows/sentryflow-release-image.yml) [![CI Test](https://github.com/boanlab/numbat/actions/workflows/ci-test-go.yml/badge.svg)](https://github.com/boanlab/numbat/actions/workflows/ci-test-go.yml) [![ci-test-py](https://github.com/boanlab/sentryflow/actions/workflows/ci-test-py.yml/badge.svg)](https://github.com/boanlab/sentryflow/actions/workflows/ci-test-py.yml)
 
 SentryFlow is a cloud-native system for API observability and security, specializing in log collection, metric production, and data exportation.
 
 ## Architecture Overview
 
-![Sentryflow Overview](docs/sentryflow_overview.png)
+![Numbat_Overview](docs/sentryflow_overview.png)
 
 ### Features
 - Generation of API Access Logs

ai-engine/.dockerignore

@@ -0,0 +1,6 @@
+.idea
+.git
+.gitignore
+protobuf
+Dockerfile
+__pycache__/

ai-engine/.gitignore

@@ -0,0 +1,3 @@
+.idea/
+__pycache__/
+protobuf/

ai-engine/Dockerfile

@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: Apache-2.0
+
+# Dockerfile
+FROM ubuntu:latest
+
+RUN apt-get update && apt-get -y install python3 python3-pip wget git
+
+RUN git clone https://github.com/isu-kim/stringlifier.git
+WORKDIR ./stringlifier
+RUN pip install .
+
+RUN mkdir /app
+WORKDIR /app
+COPY /ai-engine .
+
+# Build protobuf for Python
+RUN pip install grpcio grpcio-tools
+RUN mkdir protobuf/
+COPY /protobuf ./protobuf
+
+# Due to python import bugs, we have to compile protoc using this command
+# Refer to https://github.com/protocolbuffers/protobuf/issues/1491#issuecomment-261621112 for more information on this
+RUN python3 -m grpc_tools.protoc --python_out=. --pyi_out=. --grpc_python_out=. -I=. protobuf/sentryflow_metrics.proto
+
+WORKDIR /app
+RUN pip install -r requirements.txt
+
+CMD ["python3", "ai-engine.py"]

ai-engine/Makefile

@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: Apache-2.0
+
+IMAGE_NAME = 5gsec/sentryflow-ai-engine
+TAG = v0.1
+
+.PHONY: build
+
+build:
+	docker build -t $(IMAGE_NAME):$(TAG) -f ./Dockerfile ../

ai-engine/ai-engine.py

@@ -0,0 +1,94 @@
+import os
+import grpc
+
+from stringlifier.api import Stringlifier
+from concurrent import futures
+
+from protobuf import sentryflow_metrics_pb2_grpc
+from protobuf import sentryflow_metrics_pb2
+
+
+class HandlerServer:
+    """
+    Class for gRPC Servers
+    """
+    def __init__(self):
+        try:
+            self.listen_addr = os.environ["AI_ENGINE_ADDRESS"]
+        except KeyError:
+            self.listen_addr = "0.0.0.0:5000"
+
+        self.server = None
+        self.grpc_servers = list()
+
+    def init_grpc_servers(self):
+        """
+        init_grpc_servers method that initializes and registers gRPC servers
+        :return: None
+        """
+        self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+        self.grpc_servers.append(APIClassificationServer())  # @todo: make this configurable
+
+        grpc_server: GRPCServer
+        for grpc_server in self.grpc_servers:
+            grpc_server.register(self.server)
+
+    def serve(self):
+        """
+        serve method that starts serving gRPC servers, this is blocking function.
+        :return: None
+        """
+        self.server.add_insecure_port(self.listen_addr)
+
+        print("[INFO] Starting to serve on {}".format(self.listen_addr))
+        self.server.start()
+        self.server.wait_for_termination()
+
+
+class GRPCServer:
+    """
+    Abstract class for an individual gRPC Server
+    """
+    def register(self, server):
+        """
+        register method that registers gRPC service to target server
+        :param server: The server
+        :return: None
+        """
+        pass
+
+
+class APIClassificationServer(sentryflow_metrics_pb2_grpc.SentryFlowMetricsServicer, GRPCServer):
+    """
+    Class for API Classification Server using Stringlifier
+    """
+
+    def __init__(self):
+        self.stringlifier = Stringlifier()
+        print("[Init] Successfully initialized APIClassificationServer")
+
+    def register(self, server):
+        sentryflow_metrics_pb2_grpc.add_SentryFlowMetricsServicer_to_server(self, server)
+
+    def GetAPIClassification(self, request_iterator, context):
+        """
+        GetAPIClassification method that runs multiple API ML Classification at once
+        :param request_iterator: The requests
+        :param context: The context
+        :return: The results
+        """
+
+        for req in request_iterator:
+            paths = req.paths
+            ml_results = self.stringlifier(paths)
+            print("{} -> {}".format(paths, ml_results))
+
+            results = [sentryflow_metrics_pb2.APIClassificationSingleResponse(merged=ml_result, fields=[]) for ml_result
+                       in ml_results]
+            yield sentryflow_metrics_pb2.APIClassificationResponse(response=results)
+
+
+if __name__ == '__main__':
+    hs = HandlerServer()
+    hs.init_grpc_servers()
+    hs.serve()

deployments/sentryflow.yaml

@@ -8,6 +8,43 @@ metadata:
     pod-security.kubernetes.io/enforce: privileged
     pod-security.kubernetes.io/warn: privileged
 ---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ai-engine
+  namespace: sentryflow
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: ai-engine
+  template:
+    metadata:
+      labels:
+        app: ai-engine
+    spec:
+      containers:
+      - name: sentryflow
+        image: 5gsec/sentryflow-ai-engine:v0.1
+        ports:
+        - containerPort: 5000
+          protocol: TCP
+          name: grpc-sentryflow
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ai-engine
+  namespace: sentryflow
+spec:
+  selector:
+    app: ai-engine
+  ports:
+    - protocol: TCP
+      port: 5000
+      targetPort: 5000
+      name: grpc-sentryflow    
+---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -54,7 +91,7 @@ spec:
       serviceAccountName: sa-sentryflow
       containers:
       - name: sentryflow
-        image: 5gsec/sentryflow:v0.0.1
+        image: 5gsec/sentryflow:v0.1
         ports:
         - containerPort: 4317
           protocol: TCP

protobuf/Makefile

@@ -1,4 +1,4 @@
-PROTO:=sentryflow.proto
+PROTO:=sentryflow.proto sentryflow_metrics.proto
 PBGO:=$(PROTO:.proto=.pb.go)
 
 .PHONY: build

protobuf/go.sum

@@ -2,14 +2,14 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
 golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
 golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
 golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 h1:wpZ8pe2x1Q3f2KyT5f8oP/fa9rHAKgFPr/HZdNuS+PQ=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA=
 google.golang.org/grpc v1.61.1 h1:kLAiWrZs7YeDM6MumDe7m3y4aM6wacLzM1Y/wiLP9XY=

protobuf/sentryflow_metrics.proto

@@ -0,0 +1,18 @@
+syntax = "proto3";
+
+package protobuf;
+
+option go_package = "sentryflow/protobuf";
+
+message APIClassificationRequest {
+  string path = 1;
+}
+
+message APIClassificationResponse {
+  string merged = 1;
+  repeated string fields = 2;
+}
+
+service SentryFlowMetrics {
+  rpc GetAPIClassification(stream APIClassificationRequest) returns (stream APIClassificationResponse);
+}

sentryflow/Dockerfile

@@ -6,6 +6,7 @@ FROM golang:1.19-alpine3.17 as builder
 
 RUN apk --no-cache update
 RUN apk add --no-cache git clang llvm make gcc protobuf make
+RUN apk add --update alpine-sdk
 RUN go install github.com/golang/protobuf/protoc-gen-go@latest
 RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
 
@@ -21,6 +22,7 @@ WORKDIR /app
 COPY /sentryflow .
 
 RUN go mod tidy
+RUN export CGO_ENABLED=1; export CC=gcc;
 RUN go build -o sentryflow
 
 ### Make executable image

sentryflow/Makefile

@@ -13,8 +13,12 @@ build:
 image:
 	docker build -t $(IMAGE_NAME):$(TAG) -f ./Dockerfile ../
 
-.PHONY: clean
-clean:
+.PHONY: clean-build
+clean-build:
+	rm -f sentryflow
+
+.PHONY: clean-image
+clean-image:
 	docker rmi $(IMAGE_NAME):$(TAG)
 
 .PHONY: run