feat: add emqx

prehor · May 23, 2024 · 7be9e26 · 7be9e26
1 parent 0829ead
commit 7be9e26
Show file tree

Hide file tree

Showing 18 changed files with 534 additions and 0 deletions.
diff --git a/kubernetes/main/apps/database/emqx/app/helmrelease.yaml b/kubernetes/main/apps/database/emqx/app/helmrelease.yaml
@@ -0,0 +1,35 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json
+apiVersion: helm.toolkit.fluxcd.io/v2beta2
+kind: HelmRelease
+metadata:
+  name: emqx-operator
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: emqx-operator
+      version: 2.2.22
+      sourceRef:
+        kind: HelmRepository
+        name: emqx
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      strategy: rollback
+      retries: 3
+  dependsOn:
+    - name: cert-manager
+      namespace: cert-manager
+  values:
+    image:
+      repository: ghcr.io/emqx/emqx-operator
+    resources:
+      requests:
+        cpu: 10m
+      limits:
+        memory: 512Mi
diff --git a/kubernetes/main/apps/database/emqx/app/kustomization.yaml b/kubernetes/main/apps/database/emqx/app/kustomization.yaml
@@ -0,0 +1,6 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./helmrelease.yaml
diff --git a/kubernetes/main/apps/database/emqx/cluster/apikey-secret.sops.yaml b/kubernetes/main/apps/database/emqx/cluster/apikey-secret.sops.yaml
@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: emqx-cluster-apikey-secret
+stringData:
+    EMQX_APIKEY_KEY: ENC[AES256_GCM,data:vvc6/LJ8zHDKgFDVtNgoNd9M8OA=,iv:qP2Y5JznB9VivNEyaVCy8+neCKHMVchIhqQoDxYOT+w=,tag:xZPMLgKI+WdKAnvUGr6AUw==,type:str]
+    EMQX_APIKEY_SECRET: ENC[AES256_GCM,data:JgyhuF9fhBUKO+ZHqXvq5gNx/8nXvQ3dBqP/n58t2/NiREoqFrS/hYNXDQR85KFMorJQPUGkBjP3ZcP0CgodXQ==,iv:CJIgJnFR+pk3xQ/mZmaRHS/nOCNH9OLe73EsSCVYRg4=,tag:kRWWicXvJkOx4syas1lrCg==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1ve9kzacrwq7l9l0emvs326uk6t576d75r596e083r2tq6xu28qcsacy3s7
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBheTNjZ1pWazNGblE1ZzlB
+            QjE2YmFvVzJ6MVUwanI0TnQvcXBWVUtnbkFrCmY1UnBXT1hRSlZ5MHRJLzQ2R0p5
+            NzdUS01sT3UwbG0wQjZpejhQVUhTc1kKLS0tIFlIZ3FjUFE2MG9CeVB6b3ZwTFMy
+            QStYM3c2MXBDcWtFRXMwOVQrUUt6KzgKZ4Lff78YgvkFXUgeMYrFtXiHU6OGcE4R
+            XkBvRAT/hK5S+l4rSnUxkRFrOUa11cFlg8yHYR7k310LAAOpnlL/JA==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-05-23T07:21:24Z"
+    mac: ENC[AES256_GCM,data:33668xe+jHhQQVEUkQyLFENfvepxe02EmQIDM7KdKLuNCgioTsOTxoQ3uafJssWckTDYjqkkGv8Cpy7SHVBDWo5zV8F4I7o8mYYvtAmT8DdX2JwuOBnB6f+/97VaAAjO00oeDrvWom9BmllGY0TYrDKMVel3vOlwQy46SLc8o40=,iv:9lCVsS64M5vd8kqOEOplUyog+15ybVTqTP0sQqwXbCs=,tag:BXtO+QqbIJsDGeQEbG36dw==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.8.1
diff --git a/kubernetes/main/apps/database/emqx/cluster/cluster.yaml b/kubernetes/main/apps/database/emqx/cluster/cluster.yaml
@@ -0,0 +1,70 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/apps.emqx.io/emqx_v2beta1.json
+apiVersion: apps.emqx.io/v2beta1
+kind: EMQX
+metadata:
+  name: emqx-cluster
+spec:
+  image: public.ecr.aws/emqx/emqx:5.6.1
+  config:
+    data: |
+      authentication {
+        backend = "built_in_database"
+        mechanism = "password_based"
+        password_hash_algorithm {
+            name = "bcrypt",
+        }
+        user_id_type = "username"
+      }
+      authorization {
+        sources = [
+          {
+            type = built_in_database
+            enable = true
+          }
+        ]
+        no_match: "deny"
+      }
+  bootstrapAPIKeys:
+    - secretRef:
+        key:
+          secretName: emqx-cluster-apikey-secret
+          secretKey: EMQX_APIKEY_KEY
+        secret:
+          secretName: emqx-cluster-apikey-secret
+          secretKey: EMQX_APIKEY_SECRET
+  coreTemplate:
+    metadata:
+      annotations:
+        secret.reloader.stakater.com/reload: "emqx-cluster-apikey-secret,emqx-cluster-dashboard-secret,mqtt-secret"
+    spec:
+      replicas: 3
+      envFrom: &env
+        - secretRef:
+            name: emqx-cluster-dashboard-secret
+      # extraContainers:
+      #   - name: init-mqtt
+      #     image: docker.io/library/python:3.12-alpine
+      #     env:
+      #       - name: EMQX_DASHBOARD_ENDPOINT
+      #         value: http://emqx-cluster-dashboard.database.svc.cluster.local:18083
+      #       - name: EMQX_MQTT_USERNAME
+      #         valueFrom:
+      #           secretKeyRef:
+      #             name: mqtt-secret
+      #             key: username
+      #       - name: EMQX_MQTT_PASSWORD
+      #         valueFrom:
+      #           secretKeyRef:
+      #             name: mqtt-secret
+      #             key: password
+      #     envFrom: *env
+      #     command: ["python", "/init-mqtt.py"]
+      #     volumeMounts:
+      #       - name: init-mqtt
+      #         mountPath: /init-mqtt.py
+      #         subPath: init-mqtt.py
+      # extraVolumes:
+      #   - name: init-mqtt
+      #     configMap:
+      #       name: emqx-cluster-init-mqtt-configmap
diff --git a/kubernetes/main/apps/database/emqx/cluster/dashboard-secret.sops.yaml b/kubernetes/main/apps/database/emqx/cluster/dashboard-secret.sops.yaml
@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: emqx-cluster-dashboard-secret
+stringData:
+    EMQX_DASHBOARD__DEFAULT_USERNAME: ENC[AES256_GCM,data:5fm+nA==,iv:WxYJwsJYLx09ggnKA9tcEdjGzCyVN5H3QI0iAEWuVi8=,tag:YeD2IOERHjCePFbWnVABTA==,type:str]
+    EMQX_DASHBOARD__DEFAULT_PASSWORD: ENC[AES256_GCM,data:V+vdKYjyTMyEjpJ9xZnXoPQ=,iv:ktfoZTvDsqATO7DBMpjxRbwVmH8WNswlhRs/PvA7beI=,tag:llzv1kQAlSKZkAieigEqQA==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1ve9kzacrwq7l9l0emvs326uk6t576d75r596e083r2tq6xu28qcsacy3s7
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBheTNjZ1pWazNGblE1ZzlB
+            QjE2YmFvVzJ6MVUwanI0TnQvcXBWVUtnbkFrCmY1UnBXT1hRSlZ5MHRJLzQ2R0p5
+            NzdUS01sT3UwbG0wQjZpejhQVUhTc1kKLS0tIFlIZ3FjUFE2MG9CeVB6b3ZwTFMy
+            QStYM3c2MXBDcWtFRXMwOVQrUUt6KzgKZ4Lff78YgvkFXUgeMYrFtXiHU6OGcE4R
+            XkBvRAT/hK5S+l4rSnUxkRFrOUa11cFlg8yHYR7k310LAAOpnlL/JA==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-05-23T07:21:32Z"
+    mac: ENC[AES256_GCM,data:1w3yqweqA/lir/+7BQvNPNUHBe/ek/CCnE6TiyIDX78TwnTIfl3fLHMdXL/E9X6j/S0HyoLrGtx4/voK3qn5iv4eppBuWzfdvuH0uj61IJfNxVp8j9rDupHRDjfeJGnCKoIglVcPC9VttuTFLj0FoY6q6/CIJqrZKmbfZW9V1eE=,iv:8Woz0IAgJ+/BQIioXi+kp90l1syBRWwVb5vVsSABtco=,tag:QQltUVdKbfv8ygQD9GRzmQ==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.8.1
diff --git a/kubernetes/main/apps/database/emqx/cluster/ingress.yaml b/kubernetes/main/apps/database/emqx/cluster/ingress.yaml
@@ -0,0 +1,21 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: emqx-cluster-dashboard
+spec:
+  ingressClassName: internal
+  rules:
+    - host: &host emqx.${SECRET_DOMAIN}
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: emqx-cluster-dashboard
+                port:
+                  number: 18083
+  tls:
+    - hosts:
+        - *host
diff --git a/kubernetes/main/apps/database/emqx/cluster/job.yaml b/kubernetes/main/apps/database/emqx/cluster/job.yaml
@@ -0,0 +1,39 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: emqx-cluster-init-mqtt-user
+spec:
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      automountServiceAccountToken: false
+      restartPolicy: OnFailure
+      containers:
+        - name: init-mqtt-user
+          image: docker.io/library/python:3.12-alpine
+          env:
+            - name: EMQX_DASHBOARD_ENDPOINT
+              value: http://emqx-cluster-dashboard.database.svc.cluster.local:18083
+            - name: EMQX_MQTT_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: mqtt-secret
+                  key: username
+            - name: EMQX_MQTT_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: mqtt-secret
+                  key: password
+          envFrom:
+            - secretRef:
+                name: emqx-cluster-dashboard-secret
+          command: ["python", "/init-mqtt.py"]
+          volumeMounts:
+            - name: init-mqtt-user
+              mountPath: /init-mqtt.py
+              subPath: init-mqtt.py
+      volumes:
+        - name: init-mqtt-user
+          configMap:
+            name: emqx-cluster-init-mqtt-user
diff --git a/kubernetes/main/apps/database/emqx/cluster/kustomization.yaml b/kubernetes/main/apps/database/emqx/cluster/kustomization.yaml
@@ -0,0 +1,17 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./apikey-secret.sops.yaml
+  - ./dashboard-secret.sops.yaml
+  - ./cluster.yaml
+  - ./ingress.yaml
+  # - ./job.yaml
+  - ./podmonitor.yaml
+configMapGenerator:
+  - name: emqx-cluster-init-mqtt-user
+    files:
+      - init-mqtt.py=./resources/init-mqtt-user.py
+generatorOptions:
+  disableNameSuffixHash: true
diff --git a/kubernetes/main/apps/database/emqx/cluster/podmonitor.yaml b/kubernetes/main/apps/database/emqx/cluster/podmonitor.yaml
@@ -0,0 +1,26 @@
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: emqx
+spec:
+  selector:
+    matchLabels:
+      apps.emqx.io/instance: emqx
+      apps.emqx.io/managed-by: emqx-operator
+  podMetricsEndpoints:
+    - port: dashboard
+      path: /api/v5/prometheus/stats
+      relabelings:
+        - action: replace
+          # user-defined cluster name, requires unique
+          replacement: emqx5
+          targetLabel: cluster
+        - action: replace
+          # fix value, don't modify
+          replacement: emqx
+          targetLabel: from
+        - action: replace
+          # fix value, don't modify
+          sourceLabels: ['pod']
+          targetLabel: "instance"
diff --git a/kubernetes/main/apps/database/emqx/cluster/resources/init-mqtt-user.py b/kubernetes/main/apps/database/emqx/cluster/resources/init-mqtt-user.py
@@ -0,0 +1,92 @@
+import os
+import json
+import time
+from typing import Optional
+from urllib.request import Request, urlopen
+from urllib.error import URLError
+
+class EMQXManager:
+    def __init__(self, emqx_endpoint: str, admin_username: str, admin_password: str,
+                 mqtt_username: str, mqtt_password: str) -> None:
+        self.emqx_endpoint = emqx_endpoint
+        self.admin_username = admin_username
+        self.admin_password = admin_password
+        self.mqtt_username = mqtt_username
+        self.mqtt_password = mqtt_password
+
+    def wait_for_emqx(self) -> None:
+        while True:
+            try:
+                response = urlopen(f"{self.emqx_endpoint}/api/v5/status")
+                if response.getcode() == 200:
+                    print("EMQX started, ready to initialize..")
+                    break
+                print("EMQX not ready...")
+                time.sleep(1)
+            except URLError:
+                print("Waiting for EMQX to start..")
+                time.sleep(1)
+
+    def get_api_token(self) -> Optional[str]:
+        data = json.dumps({"username": self.admin_username, "password": self.admin_password}).encode('utf-8')
+        req = Request(f"{self.emqx_endpoint}/api/v5/login", data=data, headers={'Content-Type': 'application/json'})
+        try:
+            with urlopen(req) as response:
+                response_data = json.loads(response.read().decode('utf-8'))
+                return response_data.get('token', None)
+        except URLError as e:
+            print(f"Error: {e}")
+            return None
+
+    def create_mqtt_user(self, api_token: str) -> bool:
+        data = json.dumps({"user_id": self.mqtt_username, "password": self.mqtt_password, "is_superuser": True}).encode('utf-8')
+        headers = {'Authorization': f'Bearer {api_token}', 'Content-Type': 'application/json'}
+        req = Request(f"{self.emqx_endpoint}/api/v5/authentication/password_based:built_in_database/users", data=data, headers=headers)
+        try:
+            with urlopen(req) as response:
+                return response.getcode() == 200
+        except URLError as e:
+            print(f"Error: {e}")
+            return False
+
+def main() -> None:
+    emqx_endpoint = os.environ.get('EMQX_DASHBOARD_ENDPOINT')
+    admin_username = os.environ.get('EMQX_DASHBOARD__DEFAULT_USERNAME')
+    admin_password = os.environ.get('EMQX_DASHBOARD__DEFAULT_PASSWORD')
+    mqtt_username = os.environ.get('EMQX_MQTT_USERNAME')
+    mqtt_password = os.environ.get('EMQX_MQTT_PASSWORD')
+
+    if not emqx_endpoint:
+        print("Missing environment variable 'emqx_endpoint'.")
+        return
+    if not admin_username:
+        print("Missing environment variable 'admin_username'.")
+        return
+    if not admin_password:
+        print("Missing environment variable 'admin_password'.")
+        return
+    if not mqtt_username:
+        print("Missing environment variable 'mqtt_username'.")
+        return
+    if not mqtt_password:
+        print("Missing environment variable 'mqtt_password'.")
+        return
+
+    emqx_manager = EMQXManager(emqx_endpoint, admin_username, admin_password, mqtt_username, mqtt_password)
+    emqx_manager.wait_for_emqx()
+
+    api_token = emqx_manager.get_api_token()
+    if api_token:
+        success = emqx_manager.create_mqtt_user(api_token)
+        if success:
+            print(f"User {mqtt_username} created successfully.")
+        else:
+            print(f"Error creating user {mqtt_username} or user already exists.")
+    else:
+        print("Login failed.")
+
+    while True:
+        time.sleep(1)
+
+if __name__ == "__main__":
+    main()