diff --git a/charts/orb/.helmignore b/charts/orb/.helmignore index 50af031..ddc10d2 100644 --- a/charts/orb/.helmignore +++ b/charts/orb/.helmignore @@ -4,6 +4,7 @@ .DS_Store # Common VCS dirs .git/ +issuers/ .gitignore .bzr/ .bzrignore diff --git a/charts/orb/Chart.yaml b/charts/orb/Chart.yaml index c86e3cb..09dc170 100644 --- a/charts/orb/Chart.yaml +++ b/charts/orb/Chart.yaml @@ -10,7 +10,7 @@ name: orb description: Orb Observability Platform icon: https://avatars1.githubusercontent.com/u/13207490 type: application -version: 1.0.54 +version: 1.0.55 appVersion: "0.27.0" home: https://getorb.io sources: diff --git a/charts/orb/README.md b/charts/orb/README.md index 40f527f..6d94e6b 100644 --- a/charts/orb/README.md +++ b/charts/orb/README.md @@ -25,10 +25,11 @@ helm repo update helm dependency update ``` -* Create `orb` namespace +* Create `orb` and `otelcollectors` namespace ``` kubectl create namespace orb +kubectl create namespace otelcollectors ``` * Create JWT signing key secret @@ -44,7 +45,7 @@ kubectl create secret generic orb-sinks-encryption-key --from-literal=key=mainfl * Create keto dsn secret ``` -kubectl create secret generic orb-keto-dsn --from-literal=dsn='postgres://postgres:password@db.host.com:5432/keto' -n orb +kubectl create secret generic orb-keto-dsn --from-literal=dsn='postgres://postgres:orb@orb-postgresql-keto:5432/keto' -n orb ``` * Create admin user secrets @@ -53,12 +54,28 @@ kubectl create secret generic orb-keto-dsn --from-literal=dsn='postgres://postgr kubectl create secret generic orb-user-service --from-literal=adminEmail=user@example.com --from-literal=adminPassword=12345678 -n orb ``` +* Install orb. Replace `orb` with your helm release name, also set your HOSTNAME as a valid domain to expose service properly, remember that should generate a certificate for that. +Check the [optional variables](#optional-variables-to-set) for more options. + +``` +helm install --set ingress.hostname=HOSTNAME -n orb orb . +``` + +On AWS EKS: +Once that you can update your ingress controller (AWS LoadBalancer) using helm, a good solution could be you open the MQTT port on the cluster loadbalancer and redirect it to orb-nginx-internal pod as below: * Deploy [ingres-nginx helm](https://kubernetes.github.io/ingress-nginx/deploy/#using-helm) (to default namespace) with tcp config map configured from helm for 8883 (MQTTS). Note you need to reference both namespace and helm release name here! ``` -helm install --set tcp.8883=orb/my-orb-nginx-internal:8883 ingress-nginx ingress-nginx/ingress-nginx +helm install --set tcp.8883=orb/orb-nginx-internal:8883 ingress-nginx ingress-nginx/ingress-nginx +``` + +On On-Premise kubernetes cluster: +The best approach is use nginx-internal as service type LoadBalancer on your values.yaml to expose your MQTT port externally + +``` +helm install --set tcp.8883=orb/orb-nginx-internal:8883 ingress-nginx ingress-nginx/ingress-nginx ``` * Wait for an external IP to be available @@ -76,16 +93,38 @@ helm install cert-manager jetstack/cert-manager --namespace cert-manager --creat ``` * Create Issuer CRDs (in the `orb` namespace!) - * `cp issuers/production-issuer-tpt.yaml issuers/production-issuer.yaml` - * edit `issuers/production-issuer.yaml` and change `spec.acme.email` to a real email address - * `kubectl create -f issuers/production-issuer.yaml -n orb` -* Install orb. Replace `my-orb` with your helm release name. -Check the [optional variables](#optional-variables-to-set) for more options. +If you are using nginx as ingress controller: +``` +cp issuers/production-issuer-nginx.yaml issuers/production-issuer.yaml +``` +If you are using traefik as ingress controller: +``` +cp issuers/production-issuer-traefik.yaml issuers/production-issuer.yaml +``` +* edit `issuers/production-issuer.yaml` and change `spec.acme.email` to a real email address +``` +kubectl apply -f issuers/production-issuer.yaml -n orb +``` +* Create Certificate (in the `orb` namespace!) +``` +kubectl apply -f issuers/production-issuer.yaml -n orb +``` + +To restart entire deployment: + +``` +kubectl rollout restart deployment -n orb +``` + +## Known-bug: +Sometimes on the first run, postgres can have a problem to seed your password. To fix this, you have to manually remove the persistent volume claim (PVC) which will free up the database storage. ``` -helm install --set ingress.hostname=HOSTNAME -n orb my-orb . +kubectl delete pvc data-my-db-postgresql-0 ``` +(Or whatever the PVC associated with your initial Helm install was named.) +After remove the pvc, you need to restart the respective pod. ### Optional variables to set - **SMTP** @@ -96,4 +135,4 @@ helm install --set ingress.hostname=HOSTNAME -n orb my-orb . - `smtp.fromName`: E-mail sender display name. Defaults to `Orb`. - `smtp.fromAddress`: E-mail address of the sender. - `smtp.usernmame`: username used when authenticating to the SMTP server used for sending e-emails. - - `smtp.password`: password used when authenticating to the SMTP server used for sending e-emails. \ No newline at end of file + - `smtp.password`: password used when authenticating to the SMTP server used for sending e-emails. diff --git a/charts/orb/issuers/certificate.yaml b/charts/orb/issuers/certificate.yaml new file mode 100644 index 0000000..6c88ce6 --- /dev/null +++ b/charts/orb/issuers/certificate.yaml @@ -0,0 +1,12 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: orb-tls + namespace: orb +spec: + dnsNames: + - orb.example.com + secretName: orb-tls + issuerRef: + name: letsencrypt-prod + kind: ClusterIssuer diff --git a/charts/orb/issuers/production-issuer-tpt.yaml b/charts/orb/issuers/production-issuer-nginx.yaml similarity index 100% rename from charts/orb/issuers/production-issuer-tpt.yaml rename to charts/orb/issuers/production-issuer-nginx.yaml diff --git a/charts/orb/issuers/production-issuer-traefik.yaml b/charts/orb/issuers/production-issuer-traefik.yaml new file mode 100644 index 0000000..d7ce2a1 --- /dev/null +++ b/charts/orb/issuers/production-issuer-traefik.yaml @@ -0,0 +1,24 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: letsencrypt-prod +spec: + acme: + # The ACME server URL + server: https://acme-v02.api.letsencrypt.org/directory + # Email address used for ACME registration + email: user@example.com + # Name of a secret used to store the ACME account private key + privateKeySecretRef: + name: letsencrypt-prod + # Enable the HTTP-01 challenge provider + solvers: + - http01: + ingress: + class: traefik + ingressTemplate: + metadata: + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: "web" + traefik.ingress.kubernetes.io/router.tls: "false" + traefik.ingress.kubernetes.io/router.priority: "42" diff --git a/charts/orb/templates/ingress.yaml b/charts/orb/templates/ingress.yaml index 7c2bc07..cedefd0 100644 --- a/charts/orb/templates/ingress.yaml +++ b/charts/orb/templates/ingress.yaml @@ -1,6 +1,6 @@ # Copyright (c) Mainflux # SPDX-License-Identifier: Apache-2.0 - +{{- if eq .Values.ingress.ingressClassName "nginx" }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -34,6 +34,8 @@ spec: - {{ .Values.ingress.hostname }} secretName: {{ .Values.ingress.secret }} --- +{{- end }} +{{- if eq .Values.ingress.ingressClassName "nginx" }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -122,3 +124,113 @@ spec: - hosts: - {{ .Values.ingress.hostname }} secretName: {{ .Values.ingress.secret }} +{{- end }} +{{- if eq .Values.ingress.ingressClassName "traefik" }} +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: orb-stripprefix + namespace: {{ .Release.Namespace }} +spec: + stripPrefix: + prefixes: + - /api/v1 +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + annotations: + {{- if .Values.ingress.annotationsTraefik }} + {{ toYaml .Values.ingress.annotationsTraefik | indent 4 }} + {{- end }} + name: {{ .Release.Name }}-traefik-ingress + namespace: {{ .Release.Namespace }} +spec: + entryPoints: + - web + - websecure + routes: + - match: Host(`{{ .Values.ingress.hostname }}`) && PathPrefix(`/api/v1/users`) + kind: Rule + services: + - name: {{ .Release.Name }}-users + port: {{ .Values.users.httpPort }} + middlewares: + - name: orb-stripprefix + - match: Host(`{{ .Values.ingress.hostname }}`) && PathPrefix(`/api/v1/password`) + kind: Rule + services: + - name: {{ .Release.Name }}-users + port: {{ .Values.users.httpPort }} + middlewares: + - name: orb-stripprefix + - match: Host(`{{ .Values.ingress.hostname }}`) && PathPrefix(`/api/v1/tokens`) + kind: Rule + services: + - name: {{ .Release.Name }}-users + port: {{ .Values.users.httpPort }} + middlewares: + - name: orb-stripprefix + - match: Host(`{{ .Values.ingress.hostname }}`) && PathPrefix(`/api/v1/keys`) + kind: Rule + services: + - name: {{ .Release.Name }}-auth + port: {{ default .Values.auth.httpPort }} + middlewares: + - name: orb-stripprefix + - match: Host(`{{ .Values.ingress.hostname }}`) && PathPrefix(`/api/v1/agents`) + kind: Rule + services: + - name: {{ .Release.Name }}-fleet + port: {{ default .Values.fleet.httpPort }} + middlewares: + - name: orb-stripprefix + - match: Host(`{{ .Values.ingress.hostname }}`) && PathPrefix(`/api/v1/agent_groups`) + kind: Rule + services: + - name: {{ .Release.Name }}-fleet + port: {{ default .Values.fleet.httpPort }} + middlewares: + - name: orb-stripprefix + - match: Host(`{{ .Values.ingress.hostname }}`) && PathPrefix(`/api/v1/sinks`) + kind: Rule + services: + - name: {{ .Release.Name }}-sinks + port: {{ default .Values.sinks.httpPort }} + middlewares: + - name: orb-stripprefix + - match: Host(`{{ .Values.ingress.hostname }}`) && PathPrefix(`/api/v1/policies`) + kind: Rule + services: + - name: {{ .Release.Name }}-policies + port: {{ default .Values.policies.httpPort }} + middlewares: + - name: orb-stripprefix + - match: Host(`{{ .Values.ingress.hostname }}`) && PathPrefix(`/api/v1/features/sinks`) + kind: Rule + services: + - name: {{ .Release.Name }}-sinks + port: {{ default .Values.sinks.httpPort }} + middlewares: + - name: orb-stripprefix + - match: Host(`{{ .Values.ingress.hostname }}`) && PathPrefix(`/api/v1/features/authenticationtypes`) + kind: Rule + services: + - name: {{ .Release.Name }}-sinks + port: {{ default .Values.sinks.httpPort }} + middlewares: + - name: orb-stripprefix + - match: Host(`{{ .Values.ingress.hostname }}`) && PathPrefix(`/version`) + kind: Rule + services: + - name: {{ .Release.Name }}-fleet + port: {{ .Values.fleet.httpPort }} + - match: Host(`{{ required "an ingress.hostname is required!" .Values.ingress.hostname }}`) && PathPrefix(`/`) + kind: Rule + services: + - name: {{ .Release.Name }}-ui + port: {{ .Values.ui.port }} + tls: + secretName: {{ .Values.ingress.secret }} +{{- end }} diff --git a/charts/orb/templates/maestro-deployment.yaml b/charts/orb/templates/maestro-deployment.yaml index 191a820..2e3616b 100644 --- a/charts/orb/templates/maestro-deployment.yaml +++ b/charts/orb/templates/maestro-deployment.yaml @@ -18,7 +18,7 @@ spec: app: {{ .Release.Name }} component: maestro spec: - serviceAccountName: k8s-maestro-role + serviceAccountName: {{ .Values.maestro.rbac.serviceAccountName }} containers: - env: - name: ORB_SINKS_SECRET_KEY diff --git a/charts/orb/templates/maestro-service-account.yaml b/charts/orb/templates/maestro-service-account.yaml index d679de9..6c43ffb 100644 --- a/charts/orb/templates/maestro-service-account.yaml +++ b/charts/orb/templates/maestro-service-account.yaml @@ -2,32 +2,74 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ .Values.maestro.rbac.serviceAccountName }} + name: k8s-maestro-sa +{{- if .Values.maestro.rbac.createServiceAccountTokenSecret }} +secrets: + - name: {{ .Release.Name }}-maestro-k8s-secret +{{ end }} +{{- if .Values.maestro.rbac.createServiceAccountTokenSecret }} --- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-maestro-k8s-secret + annotations: + kubernetes.io/service-account.name: k8s-maestro-sa +type: kubernetes.io/service-account-token +{{ end }} {{- if .Values.maestro.rbac.ClusterRoleBindingCreate }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: maestro-cluster-role + # "namespace" omitted since ClusterRoles are not namespaced +rules: + - apiGroups: [""] + resources: ["*"] + verbs: ["*"] + - apiGroups: ["extensions"] + resources: ["*"] + verbs: ["*"] +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: {{ .Values.maestro.rbac.serviceAccountName }} + name: k8s-maestro-rb subjects: - kind: ServiceAccount - name: {{ .Values.maestro.rbac.serviceAccountName }} + name: k8s-maestro-sa namespace: {{ .Values.maestro.rbac.serviceAccountNamespace }} roleRef: kind: ClusterRole - name: {{ .Values.maestro.rbac.ClusterRole }} + name: maestro-cluster-role apiGroup: rbac.authorization.k8s.io {{ else }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: maestro-role + namespace: {{ .Values.maestro.rbac.serviceAccountNamespace }} +rules: + - apiGroups: [""] + resources: ["*"] + verbs: ["*"] + - apiGroups: ["extensions"] + resources: ["*"] + verbs: ["*"] +--- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: {{ .Values.maestro.rbac.serviceAccountName }} + name: k8s-maestro-rb namespace: {{ .Values.maestro.rbac.serviceAccountNamespace }} subjects: - kind: ServiceAccount - name: {{ .Values.maestro.rbac.serviceAccountName }} + name: k8s-maestro-sa + namespace: {{ .Values.maestro.rbac.serviceAccountNamespace }} roleRef: - kind: ClusterRole - name: {{ .Values.maestro.rbac.ClusterRole }} + kind: Role + name: maestro-role apiGroup: rbac.authorization.k8s.io {{ end }} diff --git a/charts/orb/templates/nginx-internal.yaml b/charts/orb/templates/nginx-internal.yaml index 6dbfd33..86768e5 100644 --- a/charts/orb/templates/nginx-internal.yaml +++ b/charts/orb/templates/nginx-internal.yaml @@ -430,12 +430,16 @@ spec: secret: secretName: {{ .Values.ingress.secret }} {{- end }} ---- +--- apiVersion: v1 kind: Service metadata: name: {{ .Release.Name }}-nginx-internal + annotations: + {{- with .Values.nginx_internal.service.annotations }} + {{- toYaml . | nindent 8 }} + {{- end }} spec: selector: app: {{ .Release.Name }} @@ -444,6 +448,18 @@ spec: - port: 8883 protocol: TCP name: mqtt + type: {{ .Values.nginx_internal.service.serviceType }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-nginx-internal-metrics +spec: + selector: + app: {{ .Release.Name }} + component: nginx-internal + ports: - port: 8080 protocol: TCP name: metrics diff --git a/charts/orb/values.yaml b/charts/orb/values.yaml index fe1b056..538e5a4 100644 --- a/charts/orb/values.yaml +++ b/charts/orb/values.yaml @@ -27,10 +27,10 @@ smtp: ingress: ingressClassName: "nginx" annotationsNginx: {} # set this for certificate - # cert-manager.io/issuer: "letsencrypt-prod" + # cert-manager.io/clusterissuer: "letsencrypt-prod" annotationsNginxRewrite: # set this for certificate nginx.ingress.kubernetes.io/rewrite-target: /$1 - # cert-manager.io/issuer: "letsencrypt-prod" + # cert-manager.io/clusterissuer: "letsencrypt-prod" hostname: "" secret: "orb-tls" @@ -186,7 +186,7 @@ nats: clusterAuth: enabled: false maxPayload: 67108864 - replicaCount: 3 + replicaCount: 1 postgresql-users: enabled: true # dependency install, disable if you want to use external services @@ -194,6 +194,10 @@ postgresql-users: postgresqlUsername: postgres postgresqlPassword: mainflux postgresqlDatabase: users + livenessProbe: + initialDelaySeconds: 200 + readinessProbe: + initialDelaySeconds: 200 resources: requests: cpu: 25m @@ -203,11 +207,13 @@ postgresql-users: postgresql-fleet: enabled: true # dependency install, disable if you want to use external services name: postgresql-fleet - image: - tag: 13 postgresqlUsername: postgres postgresqlPassword: orb postgresqlDatabase: fleet + livenessProbe: + initialDelaySeconds: 200 + readinessProbe: + initialDelaySeconds: 200 resources: requests: cpu: 25m @@ -217,11 +223,13 @@ postgresql-fleet: postgresql-policies: enabled: true # dependency install, disable if you want to use external services name: postgresql-policies - image: - tag: 13 postgresqlUsername: postgres postgresqlPassword: orb postgresqlDatabase: policies + livenessProbe: + initialDelaySeconds: 200 + readinessProbe: + initialDelaySeconds: 200 resources: requests: cpu: 25m @@ -231,11 +239,13 @@ postgresql-policies: postgresql-sinks: enabled: true # dependency install, disable if you want to use external services name: postgresql-sinks - image: - tag: 13 postgresqlUsername: postgres postgresqlPassword: orb postgresqlDatabase: sinks + livenessProbe: + initialDelaySeconds: 200 + readinessProbe: + initialDelaySeconds: 200 resources: requests: cpu: 25m @@ -248,6 +258,10 @@ postgresql-things: postgresqlUsername: postgres postgresqlPassword: mainflux postgresqlDatabase: things + livenessProbe: + initialDelaySeconds: 200 + readinessProbe: + initialDelaySeconds: 200 resources: requests: cpu: 25m @@ -260,6 +274,10 @@ postgresql-auth: postgresqlUsername: postgres postgresqlPassword: mainflux postgresqlDatabase: auth + livenessProbe: + initialDelaySeconds: 200 + readinessProbe: + initialDelaySeconds: 200 resources: requests: cpu: 25m @@ -272,6 +290,10 @@ postgresql-keto: postgresqlUsername: postgres postgresqlPassword: orb postgresqlDatabase: keto + livenessProbe: + initialDelaySeconds: 200 + readinessProbe: + initialDelaySeconds: 200 resources: requests: cpu: 25m @@ -287,6 +309,10 @@ postgresql-maestro: postgresqlUsername: postgres postgresqlPassword: orb postgresqlDatabase: maestro + livenessProbe: + initialDelaySeconds: 200 + readinessProbe: + initialDelaySeconds: 200 resources: requests: cpu: 25m @@ -294,7 +320,7 @@ postgresql-maestro: size: 1Gi commonAnnotations: helm.sh/hook: "pre-install, pre-upgrade" - helm.sh/hook-weight: "-1" + helm.sh/hook-weight: "-1" redis-streams: enabled: true # dependency install, disable if you want to use external services @@ -329,7 +355,7 @@ jaeger-operator: jaeger: create: true rbac: - pspEnabled: true + pspEnabled: false clusterRole: true envoy: @@ -341,6 +367,9 @@ envoy: annotations: {} nginx_internal: + service: + serviceType: "ClusterIP" # use LoadBalancer for on-premise solution + annotations: {} # use for external dns integration image: pullPolicy: "IfNotPresent" repository: "nginx" @@ -349,6 +378,9 @@ nginx_internal: annotations: {} keto: + secret: + enabled: false + nameOverride: "orb-keto-dsn" keto: autoMigrate: true config: @@ -402,9 +434,8 @@ maestro: port: 9092 rbac: ClusterRoleBindingCreate: false # set this true to create ClusterRoleBinding instead RoleBinding - ClusterRole: "admin" - serviceAccountName: "k8s-maestro-role" serviceAccountNamespace: "otelcollectors" + createServiceAccountTokenSecret: false # required true for kubernetes > 1.27 image: name: "orb-maestro" metadata: