feat: add usbip-server

prehor · Jun 17, 2024 · 130630b · 130630b
1 parent e4fc435
commit 130630b
Show file tree

Hide file tree

Showing 14 changed files with 323 additions and 8 deletions.
diff --git a/.gitignore b/.gitignore
@@ -9,8 +9,10 @@ kubeconfig
 *.pub
 *.key
 # Private
-.private
+.archive
 .bin
+.private
+TODO.md
 # Ansible
 .venv*
 # Taskfile

diff --git a/ansible/main/playbooks/cluster-prepare.yaml b/ansible/main/playbooks/cluster-prepare.yaml
@@ -24,7 +24,7 @@
             name: apt-transport-https,ca-certificates,conntrack,curl,dirmngr,gdisk,gnupg,hdparm,htop,
               iptables,iputils-ping,ipvsadm,libseccomp2,lm-sensors,net-tools,nfs-common,
               nvme-cli,open-iscsi,parted,psmisc,python3,python3-apt,python3-kubernetes,python3-yaml,
-              smartmontools,socat,software-properties-common,unzip,util-linux
+              smartmontools,socat,software-properties-common,unzip,usbip,util-linux
             install_recommends: false
 
     - name: Network Configuration
@@ -84,7 +84,7 @@
             dest: "/etc/modules-load.d/{{ item }}.conf"
             mode: "0644"
             content: "{{ item }}"
-          loop: ["br_netfilter", "ceph", "ip_vs", "ip_vs_rr", "iptable_mangle", "iptable_raw", "nbd", "overlay", "rbd", "xt_socket"]
+          loop: ["br_netfilter", "ceph", "ip_vs", "ip_vs_rr", "iptable_mangle", "iptable_raw", "nbd", "overlay", "rbd", "vhci-hcd", "xt_socket"]
           register: modules_status
         - name: System Configuration | Reload Kernel modules # noqa: no-changed-when no-handler
           when: modules_status.changed

diff --git a/ansible/storage/playbooks/cluster-prepare.yaml b/ansible/storage/playbooks/cluster-prepare.yaml
@@ -24,7 +24,7 @@
             name: apt-transport-https,ca-certificates,conntrack,curl,dirmngr,gdisk,gnupg,hdparm,htop,
               iptables,iputils-ping,ipvsadm,libseccomp2,lm-sensors,net-tools,nfs-common,
               nvme-cli,open-iscsi,parted,psmisc,python3,python3-apt,python3-kubernetes,python3-yaml,
-              smartmontools,socat,software-properties-common,unzip,util-linux
+              smartmontools,socat,software-properties-common,unzip,usbip,util-linux
             install_recommends: false
 
     - name: Network Configuration
@@ -84,7 +84,7 @@
             dest: "/etc/modules-load.d/{{ item }}.conf"
             mode: "0644"
             content: "{{ item }}"
-          loop: ["br_netfilter", "ceph", "ip_vs", "ip_vs_rr", "iptable_mangle", "iptable_raw", "nbd", "overlay", "rbd", "xt_socket"]
+          loop: ["br_netfilter", "ceph", "ip_vs", "ip_vs_rr", "iptable_mangle", "iptable_raw", "nbd", "overlay", "rbd", "usbip_host", "xt_socket"]
           register: modules_status
         - name: System Configuration | Reload Kernel modules # noqa: no-changed-when no-handler
           when: modules_status.changed

diff --git a/kubernetes/main/apps/home-automation/home-assistant/app/helmrelease.yaml b/kubernetes/main/apps/home-automation/home-assistant/app/helmrelease.yaml
@@ -40,8 +40,8 @@ spec:
             "ips": ["192.168.101.254/24"]
           }]
         reloader.stakater.com/auto: "true"
-      nodeSelector:
-        nabucasa.feature.node.kubernetes.io/skyconnect: "true"
+      # nodeSelector:
+      #   nabucasa.feature.node.kubernetes.io/skyconnect: "true"
     controllers:
       home-assistant:
         containers:
@@ -73,7 +73,7 @@ spec:
               limits:
                 cpu: 2000m
                 memory: 2Gi
-                device/nabucasa-skyconnect: 1
+                # device/nabucasa-skyconnect: 1
           code-server:
             image:
               repository: ghcr.io/coder/code-server

diff --git a/kubernetes/storage/apps/kube-system/generic-device-plugin/app/configmap.yaml b/kubernetes/storage/apps/kube-system/generic-device-plugin/app/configmap.yaml
@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: generic-device-plugin-config
+data:
+  config.yaml: |
+    devices:
+      - name: nabucasa-skyconnect
+        groups:
+          - paths:
+              - path: /dev/serial/by-id/usb-Nabu_Casa_SkyConnect*
+                mountPath: /dev/ttyUSB0
diff --git a/kubernetes/storage/apps/kube-system/generic-device-plugin/app/helmrelease.yaml b/kubernetes/storage/apps/kube-system/generic-device-plugin/app/helmrelease.yaml
@@ -0,0 +1,103 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: &app generic-device-plugin
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: app-template
+      version: 3.2.1
+      sourceRef:
+        kind: HelmRepository
+        name: bjw-s
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  values:
+    defaultPodOptions:
+      annotations:
+        reloader.stakater.com/auto: "true"
+      priorityClassName: system-node-critical
+      tolerations:
+        - operator: "Exists"
+          effect: "NoExecute"
+        - operator: "Exists"
+          effect: "NoSchedule"
+    controllers:
+      generic-device-plugin:
+        type: daemonset
+        containers:
+          app:
+            image:
+              repository: ghcr.io/squat/generic-device-plugin
+              tag: latest
+            args:
+              - --config=/config/config.yaml
+              - --domain=device
+              - --log-level=debug
+            probes:
+              liveness: &probes
+                enabled: true
+                custom: true
+                spec:
+                  httpGet:
+                    path: /health
+                    port: &port 8080
+                  initialDelaySeconds: 0
+                  periodSeconds: 10
+                  timeoutSeconds: 1
+                  failureThreshold: 3
+              readiness: *probes
+              startup: *probes
+            resources:
+              requests:
+                cpu: 5m
+                memory: 16Mi
+              limits:
+                cpu: 100m
+                memory: 64Mi
+            securityContext:
+              privileged: true
+    service:
+      app:
+        controller: *app
+        ports:
+          http:
+            port: *port
+    serviceMonitor:
+      app:
+        serviceName: *app
+        endpoints:
+          - port: http
+            scheme: http
+            path: /metrics
+            interval: 1m
+            scrapeTimeout: 10s
+    serviceAccount:
+      create: true
+      name: *app
+    persistence:
+      config:
+        type: configMap
+        name: generic-device-plugin-config
+        globalMounts:
+          - path: /config/config.yaml
+            subPath: config.yaml
+      device-plugins:
+        type: hostPath
+        hostPath: /var/lib/kubelet/device-plugins
+        globalMounts:
+          - path: /var/lib/kubelet/device-plugins
+      dev:
+        type: hostPath
+        hostPath: /dev
+        globalMounts:
+          - path: /dev
diff --git a/kubernetes/storage/apps/kube-system/generic-device-plugin/app/kustomization.yaml b/kubernetes/storage/apps/kube-system/generic-device-plugin/app/kustomization.yaml
@@ -0,0 +1,7 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./configmap.yaml
+  - ./helmrelease.yaml
diff --git a/kubernetes/storage/apps/kube-system/generic-device-plugin/ks.yaml b/kubernetes/storage/apps/kube-system/generic-device-plugin/ks.yaml
@@ -0,0 +1,21 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app generic-device-plugin
+  namespace: flux-system
+spec:
+  targetNamespace: kube-system
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  path: ./kubernetes/storage/apps/kube-system/generic-device-plugin/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-ops
+  wait: true
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
diff --git a/kubernetes/storage/apps/kube-system/kustomization.yaml b/kubernetes/storage/apps/kube-system/kustomization.yaml
@@ -5,6 +5,7 @@ resources:
   - ./namespace.yaml
   - ./cilium/ks.yaml
   - ./coredns/ks.yaml
+  - ./generic-device-plugin/ks.yaml
   - ./k8tz/ks.yaml
   - ./kube-vip/ks.yaml
   - ./metrics-server/ks.yaml

diff --git a/kubernetes/storage/apps/network/kustomization.yaml b/kubernetes/storage/apps/network/kustomization.yaml
@@ -6,3 +6,4 @@ resources:
   - ./echo-server/ks.yaml
   - ./ingress-nginx/ks.yaml
   - ./k8s-gateway/ks.yaml
+  - ./usbip-server/ks.yaml
diff --git a/kubernetes/storage/apps/network/usbip-server/app/configmap.yaml b/kubernetes/storage/apps/network/usbip-server/app/configmap.yaml
@@ -0,0 +1,26 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: usbip-server-config
+data:
+  crontab: |
+    */1 * * * * /etc/usbip/usbip-bind.sh
+  usbip-bind.sh: |
+    #!/bin/bash
+
+    PATH=/bin:/sbin:/usr/bin/:/usr/sbin
+
+    ID_LIST=$(usbip list -p -l | cut -d'#' -f1 | cut -d'=' -f2)
+    for id in $ID_LIST; do
+      usbip bind -b $id 2>/dev/null
+    done
+  usbip-unbind.sh: |
+    #!/bin/bash
+
+    PATH=/bin:/sbin:/usr/bin:/usr/sbin
+
+    ID_LIST=$(usbip list -p -l | cut -d'#' -f1 | cut -d'=' -f2)
+    for id in $ID_LIST; do
+        usbip unbind -b $id 2>/dev/null
+    done
diff --git a/kubernetes/storage/apps/network/usbip-server/app/helmrelease.yaml b/kubernetes/storage/apps/network/usbip-server/app/helmrelease.yaml
@@ -0,0 +1,115 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: &app usbip-server
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: app-template
+      version: 3.2.1
+      sourceRef:
+        kind: HelmRepository
+        name: bjw-s
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  values:
+    defaultPodOptions:
+      annotations:
+        reloader.stakater.com/auto: "true"
+    controllers:
+      usbip-server:
+        containers:
+          app:
+            image:
+              repository: ghcr.io/prehor/usbip
+              tag: 6.6.31
+            lifecycle:
+              postStart:
+                exec:
+                  command:
+                    - /bin/sh
+                    - -c
+                    - |
+                        (
+                          set -ex
+                          /etc/usbip/usbip-bind.sh
+                        ) > /tmp/postStart.log 2>&1 || true
+              preStop:
+                exec:
+                  command:
+                    - /bin/sh
+                    - -c
+                    -   (
+                          set -ex
+                          /etc/usbip/usbip-unbind.sh
+                        ) > /tmp/preStop.log 2>&1 || true
+            probes:
+              liveness: &probes
+                enabled: true
+                custom: true
+                spec:
+                  tcpSocket:
+                    port: &port 3240
+                  initialDelaySeconds: 0
+                  periodSeconds: 10
+                  timeoutSeconds: 1
+                  failureThreshold: 3
+              readiness: *probes
+            resources:
+              requests:
+                cpu: 10m
+                memory: 16Mi
+              limits:
+                cpu: 100m
+                memory: 128Mi
+                device/nabucasa-skyconnect: 1
+            securityContext:
+              privileged: true
+              runAsUser: 0
+              runAsGroup: 0
+              # readOnlyRootFilesystem: true
+          bind:
+            image:
+              repository: ghcr.io/prehor/usbip
+              tag: 6.6.31
+            args:
+              - /usr/sbin/crond
+              - -l2
+              - -f
+    persistence:
+      config:
+        type: configMap
+        name: usbip-server-config
+        defaultMode: 0o555
+        globalMounts:
+          - path: /etc/crontabs/root
+            subPath: crontab
+          - path: /etc/usbip/usbip-bind.sh
+            subPath: usbip-bind.sh
+          - path: /etc/usbip/usbip-unbind.sh
+            subPath: usbip-unbind.sh
+      sys:
+        type: hostPath
+        hostPath: /sys
+        globalMounts:
+          - path: /sys
+    service:
+      app:
+        controller: *app
+        type: NodePort
+        externalTrafficPolicy: Local
+        ports:
+          tcp:
+            port: *port
+            nodePort: 32400
+    serviceAccount:
+      create: true
+      name: *app
diff --git a/kubernetes/storage/apps/network/usbip-server/app/kustomization.yaml b/kubernetes/storage/apps/network/usbip-server/app/kustomization.yaml
@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./configmap.yaml
+  - ./helmrelease.yaml
diff --git a/kubernetes/storage/apps/network/usbip-server/ks.yaml b/kubernetes/storage/apps/network/usbip-server/ks.yaml
@@ -0,0 +1,20 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app usbip-server
+  namespace: flux-system
+spec:
+  targetNamespace: network
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  path: ./kubernetes/storage/apps/network/usbip-server/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-ops
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m