diff --git a/.github/workflows/server-cicd.yml b/.github/workflows/server-cicd.yml index 98be42df8..0f237dc1b 100644 --- a/.github/workflows/server-cicd.yml +++ b/.github/workflows/server-cicd.yml @@ -60,12 +60,3 @@ jobs: permissions: contents: read packages: write - feedback-deployment: - uses: ./.github/workflows/_restart-argocd.yml - if: ${{ github.ref }} == 'refs/heads/main' - needs: - - server-build - with: - deployment: feedback - secrets: - ARGOCD_TOKEN: ${{ secrets.ARGOCD_TOKEN }} diff --git a/README.md b/README.md index 7d05ac548..f1d598504 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # NavigaTUM [![Deployment Status](https://argocd.nav.tum.sexy/api/badge?name=navigatum-prod)](https://argocd.nav.tum.sexy/applications/navigatum-prod) -[![Website Uptime over 30 days](https://nav-monitoring.mm.rbg.tum.de/api/badge/5/uptime/720?label=Website%20Uptime/30&labelSuffix=d)](https://uptime.nav.tum.sexy/status/navigatum) -[![API Uptime over 30 days](https://nav-monitoring.mm.rbg.tum.de/api/badge/2/uptime/720?label=API%20Uptime/30&labelSuffix=d)](https://uptime.nav.tum.sexy/status/navigatum) -[![CDN Uptime over 30 days](https://nav-monitoring.mm.rbg.tum.de/api/badge/1/uptime/720?label=CDN%20Uptime/30&labelSuffix=d)](https://uptime.nav.tum.sexy/status/navigatum) +[![Website Uptime over 30 days](https://nav-monitoring.mm.rbg.tum.de/api/badge/5/uptime/720?label=Website%20Uptime/30&labelSuffix=d)](https://nav-monitoring.mm.rbg.tum.de/status/navigatum) +[![API Uptime over 30 days](https://nav-monitoring.mm.rbg.tum.de/api/badge/2/uptime/720?label=API%20Uptime/30&labelSuffix=d)](https://nav-monitoring.mm.rbg.tum.de/status/navigatum) +[![CDN Uptime over 30 days](https://nav-monitoring.mm.rbg.tum.de/api/badge/1/uptime/720?label=CDN%20Uptime/30&labelSuffix=d)](https://nav-monitoring.mm.rbg.tum.de/status/navigatum) NavigaTUM is a tool developed by students for students, to help you get around at [TUM](https://tum.de). Feel free to contribute, we are open to new people 😄. diff --git a/deployment/k3s/templates/deployments/feedback-deployment.yaml b/deployment/k3s/templates/deployments/feedback-deployment.yaml deleted file mode 100644 index 52395cfe0..000000000 --- a/deployment/k3s/templates/deployments/feedback-deployment.yaml +++ /dev/null @@ -1,100 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: feedback - labels: - app.kubernetes.io/part-of: navigatum - app.kubernetes.io/name: feedback - namespace: {{ $.Values.namespace }} -spec: - replicas: 1 - revisionHistoryLimit: 0 - selector: - matchLabels: - app.kubernetes.io/part-of: navigatum - app.kubernetes.io/name: feedback - strategy: - rollingUpdate: - maxSurge: {{ if eq "nav.tum.de" $.Values.url }}50%{{ else }}100%{{ end }} - maxUnavailable: {{ if eq "nav.tum.de" $.Values.url }}50%{{ else }}0%{{ end }} - type: RollingUpdate - template: - metadata: - labels: - app.kubernetes.io/part-of: navigatum - app.kubernetes.io/name: feedback - {{- if eq "nav.tum.de" $.Values.url }} - annotations: - prometheus.io/path: /metrics - prometheus.io/port: '3004' - prometheus.io/scrape: 'true' - {{- end }} - spec: - priorityClassName: {{ $.Values.url }} - affinity: # no two pods on the same node - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - navigatum - - key: deployment - operator: In - values: - - feedback - topologyKey: "kubernetes.io/hostname" - containers: - - name: feedback - image: "ghcr.io/tum-dev/navigatum-server:{{ $.Values.tag }}" - imagePullPolicy: Always - command: ["/bin/navigatum-feedback"] - {{ if or (.Values.server.GITHUB_TOKEN) (.Values.server.JWT_KEY) }} - envFrom: - - secretRef: - name: feedback-api-keys # GITHUB_TOKEN, JWT_KEY - {{ end }} - ports: - - containerPort: 3004 - name: feedback - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - resources: - requests: - cpu: 1m - memory: 40Mi - limits: - memory: 40Mi - livenessProbe: - httpGet: - path: /api/feedback/status - port: feedback - failureThreshold: 5 - periodSeconds: 1 - startupProbe: - httpGet: - path: /api/feedback/status - port: feedback - failureThreshold: 60 - periodSeconds: 1 -{{ if or (.Values.server.GITHUB_TOKEN) (.Values.server.JWT_KEY) }} ---- -apiVersion: v1 -kind: Secret -type: Opaque -metadata: - name: feedback-api-keys - namespace: {{ $.Values.namespace }} - labels: - app.kubernetes.io/part-of: navigatum - app.kubernetes.io/name: feedback -data: - {{ if $.Values.server.GITHUB_TOKEN }} - GITHUB_TOKEN: {{ $.Values.server.GITHUB_TOKEN }} - {{ end }} - {{ if $.Values.server.JWT_KEY }} - JWT_KEY: {{ $.Values.server.JWT_KEY }} - {{ end }} -{{ end }} diff --git a/deployment/k3s/templates/deployments/webclient-deployment.yaml b/deployment/k3s/templates/deployments/webclient-deployment.yaml index bc76f6433..cd0abfe09 100644 --- a/deployment/k3s/templates/deployments/webclient-deployment.yaml +++ b/deployment/k3s/templates/deployments/webclient-deployment.yaml @@ -5,7 +5,7 @@ metadata: labels: app.kubernetes.io/part-of: navigatum app.kubernetes.io/name: web - namespace: {{ $.Values.namespace }} + namespace: { { $.Values.namespace } } spec: replicas: 1 revisionHistoryLimit: 0 @@ -24,7 +24,7 @@ spec: app.kubernetes.io/part-of: navigatum app.kubernetes.io/name: web spec: - priorityClassName: {{ $.Values.url }} + priorityClassName: { { $.Values.url } } containers: - name: webclient image: "ghcr.io/tum-dev/navigatum-webclient:{{ $.Values.tag }}" @@ -39,8 +39,6 @@ spec: value: http://cdn-svc.navigatum.svc.cluster.local:3002 - name: MAIN_API_URL value: http://api-svc.navigatum.svc.cluster.local:3003 - - name: FEEDBACK_API_URL - value: http://feedback-svc.navigatum.svc.cluster.local:3004 resources: requests: cpu: 50m @@ -63,9 +61,9 @@ spec: allowPrivilegeEscalation: false capabilities: drop: - - ALL + - ALL add: - - CHOWN - - DAC_OVERRIDE - - SETGID - - SETUID + - CHOWN + - DAC_OVERRIDE + - SETGID + - SETUID diff --git a/deployment/k3s/templates/networking/ingress.yaml b/deployment/k3s/templates/networking/ingress.yaml index ec4169a8d..0904f8b27 100644 --- a/deployment/k3s/templates/networking/ingress.yaml +++ b/deployment/k3s/templates/networking/ingress.yaml @@ -4,16 +4,11 @@ metadata: name: ingress labels: app.kubernetes.io/part-of: navigatum - namespace: {{ $.Values.namespace }} + namespace: { { $.Values.namespace } } spec: entryPoints: - websecure routes: - - kind: Rule - match: Host(`{{ $.Values.url }}`) && PathPrefix(`/api/feedback/`) - services: - - name: feedback-svc - port: 3004 - kind: Rule match: Host(`{{ $.Values.url }}`) && PathPrefix(`/api/`) services: @@ -53,7 +48,7 @@ spec: middlewares: - name: nav-tum-de-redirect tls: - secretName: {{ $.Values.url }} + secretName: { { $.Values.url } } --- apiVersion: traefik.containo.us/v1alpha1 kind: IngressRoute @@ -61,7 +56,7 @@ metadata: name: ingress-http labels: app.kubernetes.io/part-of: navigatum - namespace: {{ $.Values.namespace }} + namespace: { { $.Values.namespace } } spec: entryPoints: - web diff --git a/deployment/k3s/templates/networking/service.yaml b/deployment/k3s/templates/networking/service.yaml index 297fe590e..d98c421c6 100644 --- a/deployment/k3s/templates/networking/service.yaml +++ b/deployment/k3s/templates/networking/service.yaml @@ -5,7 +5,7 @@ metadata: labels: app.kubernetes.io/part-of: navigatum app.kubernetes.io/name: api - namespace: {{ $.Values.namespace }} + namespace: { { $.Values.namespace } } spec: type: ClusterIP selector: @@ -18,30 +18,12 @@ spec: --- apiVersion: v1 kind: Service -metadata: - name: feedback-svc - labels: - app.kubernetes.io/part-of: navigatum - app.kubernetes.io/name: feedback - namespace: {{ $.Values.namespace }} -spec: - type: ClusterIP - selector: - app.kubernetes.io/part-of: navigatum - app.kubernetes.io/name: feedback - ports: - - name: http - port: 3004 - targetPort: 3004 ---- -apiVersion: v1 -kind: Service metadata: name: maps-svc labels: app.kubernetes.io/part-of: navigatum app.kubernetes.io/name: maps - namespace: {{ $.Values.namespace }} + namespace: { { $.Values.namespace } } spec: type: ClusterIP selector: @@ -62,7 +44,7 @@ metadata: labels: app.kubernetes.io/part-of: navigatum app.kubernetes.io/name: cdn - namespace: {{ $.Values.namespace }} + namespace: { { $.Values.namespace } } spec: type: ClusterIP selector: @@ -80,7 +62,7 @@ metadata: labels: app.kubernetes.io/part-of: navigatum app.kubernetes.io/name: web - namespace: {{ $.Values.namespace }} + namespace: { { $.Values.namespace } } spec: type: ClusterIP selector: @@ -90,7 +72,7 @@ spec: - name: http port: 3000 targetPort: 3000 -{{- if eq "nav.tum.de" $.Values.url }} + { { - if eq "nav.tum.de" $.Values.url } } --- apiVersion: v1 kind: Service @@ -99,7 +81,7 @@ metadata: labels: app.kubernetes.io/part-of: navigatum app.kubernetes.io/name: postgres - namespace: {{ $.Values.namespace }} + namespace: { { $.Values.namespace } } spec: type: ClusterIP selector: @@ -109,4 +91,4 @@ spec: - name: postgres port: 5432 targetPort: 5432 - {{ end }} + { { end } } diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 52efcd76e..38355924a 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,140 +1,124 @@ services: - webclient: - restart: unless-stopped - build: ./webclient - ports: - - "3000:3000" - environment: - NUXT_PUBLIC_MAPS_URL: http://tileserver:8080 - NUXT_PUBLIC_CDN_URL: http://data:3002 - NUXT_PUBLIC_API_URL: http://main-api:3003 - NUXT_PUBLIC_FEEDBACK_URL: http://feedback-api:3004 - depends_on: - tileserver: - condition: service_healthy - data: - condition: service_healthy - main-api: - condition: service_healthy - feedback-api: - condition: service_healthy - # maps - tileserver-init-sprites: - image: alpine:latest - command: sh -c "mkdir -p /data/sprites/ && rm -f /data/sprites/* && wget -P /data/sprites ${TILE_SPRITES_URL}/osm-liberty.json ${TILE_SPRITES_URL}/osm-liberty@2x.json ${TILE_SPRITES_URL}/osm-liberty.png ${TILE_SPRITES_URL}/osm-liberty@2x.png" - volumes: - - tileserver-src:/data - tileserver-init-config: - image: alpine:latest - command: sh -c "apk update --no-progress --quiet && apk add rsync --no-progress --quiet && rsync /to_data/ /data/" - volumes: - - tileserver-src:/data - - ./deployment/k3s/files/:/to_data/:ro - depends_on: - tileserver-init-sprites: - condition: service_completed_successfully - tileserver-srv-src: # needed for simpler initialisation of dev environments - image: nginx:1.26 - restart: on-failure - volumes: - - tileserver-src:/usr/share/nginx/html/maps/vol:ro - depends_on: - tileserver-init-config: - condition: service_completed_successfully - tileserver-init-sprites: - condition: service_completed_successfully - healthcheck: - test: service nginx status || exit 1 - retries: 2 - interval: 10s - start_period: 10s - tileserver: - image: maptiler/tileserver-gl:latest - restart: unless-stopped - command: /usr/src/app/docker-entrypoint.sh --public_url=https://nav.tum.de/maps/ - tmpfs: - - /tmp - volumes: - - tileserver-src:/data/ - read_only: true - ports: - - "8080:8080" - depends_on: - tileserver-init-config: - condition: service_completed_successfully - tileserver-init-sprites: - condition: service_completed_successfully - # cdn - data: - restart: unless-stopped - build: ./data - ports: - - "3002:3002" - # server - main-api: - restart: unless-stopped - build: ./server - command: /bin/navigatum-main-api - ports: - - "3003:3003" - volumes: - - type: tmpfs - target: /home/navigatum/.cache - user: 1000:3000 - environment: - MIELI_URL: http://meilisearch:7700 - TILESERVER_URL: http://tileserver:8080 - CDN_URL: http://data:3002/cdn - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_URL: db:5432 - depends_on: - meilisearch: - condition: service_healthy - db: - condition: service_healthy - healthcheck: - test: wget -q --spider http://localhost:3003/api/status - retries: 5 - interval: 10s - start_period: 60s - meilisearch: - image: getmeili/meilisearch:v1.7.6 - restart: unless-stopped - ports: - - "7700:7700" - healthcheck: - test: wget -q --spider http://localhost:7700/health - retries: 5 - interval: 10s - start_period: 10s - feedback-api: - restart: unless-stopped - build: ./server - command: /bin/navigatum-feedback - ports: - - "3004:3004" - healthcheck: - test: wget -q --spider http://localhost:3004/api/feedback/status - retries: 5 - start_period: 10s - db: - image: postgres:16 - restart: unless-stopped - environment: - PGDATA: /var/lib/postgresql/data/pgdata - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_DB: ${POSTGRES_DB} - ports: - - "5432:5432" - healthcheck: - test: [ "CMD", "pg_isready", "-U", "${POSTGRES_USER}" ] - retries: 5 - interval: 10s - start_period: 10s + webclient: + restart: unless-stopped + build: ./webclient + ports: + - "3000:3000" + environment: + NUXT_PUBLIC_MAPS_URL: http://tileserver:8080 + NUXT_PUBLIC_CDN_URL: http://data:3002 + NUXT_PUBLIC_API_URL: http://main-api:3003 + depends_on: + tileserver: + condition: service_healthy + data: + condition: service_healthy + main-api: + condition: service_healthy + feedback-api: + condition: service_healthy + # maps + tileserver-init-sprites: + image: alpine:latest + command: sh -c "mkdir -p /data/sprites/ && rm -f /data/sprites/* && wget -P /data/sprites ${TILE_SPRITES_URL}/osm-liberty.json ${TILE_SPRITES_URL}/osm-liberty@2x.json ${TILE_SPRITES_URL}/osm-liberty.png ${TILE_SPRITES_URL}/osm-liberty@2x.png" + volumes: + - tileserver-src:/data + tileserver-init-config: + image: alpine:latest + command: sh -c "apk update --no-progress --quiet && apk add rsync --no-progress --quiet && rsync /to_data/ /data/" + volumes: + - tileserver-src:/data + - ./deployment/k3s/files/:/to_data/:ro + depends_on: + tileserver-init-sprites: + condition: service_completed_successfully + tileserver-srv-src: # needed for simpler initialisation of dev environments + image: nginx:1.26 + restart: on-failure + volumes: + - tileserver-src:/usr/share/nginx/html/maps/vol:ro + depends_on: + tileserver-init-config: + condition: service_completed_successfully + tileserver-init-sprites: + condition: service_completed_successfully + healthcheck: + test: service nginx status || exit 1 + retries: 2 + interval: 10s + start_period: 10s + tileserver: + image: maptiler/tileserver-gl:latest + restart: unless-stopped + command: /usr/src/app/docker-entrypoint.sh --public_url=https://nav.tum.de/maps/ + tmpfs: + - /tmp + volumes: + - tileserver-src:/data/ + read_only: true + ports: + - "8080:8080" + depends_on: + tileserver-init-config: + condition: service_completed_successfully + tileserver-init-sprites: + condition: service_completed_successfully + # cdn + data: + restart: unless-stopped + build: ./data + ports: + - "3002:3002" + # server + main-api: + restart: unless-stopped + build: ./server + command: /bin/navigatum-main-api + ports: + - "3003:3003" + volumes: + - type: tmpfs + target: /home/navigatum/.cache + user: 1000:3000 + environment: + MIELI_URL: http://meilisearch:7700 + TILESERVER_URL: http://tileserver:8080 + CDN_URL: http://data:3002/cdn + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_URL: db:5432 + depends_on: + meilisearch: + condition: service_healthy + db: + condition: service_healthy + meilisearch: + image: getmeili/meilisearch:v1.7.6 + restart: unless-stopped + ports: + - "7700:7700" + healthcheck: + test: wget -q --spider http://localhost:7700/health + retries: 5 + interval: 10s + start_period: 10s + db: + image: postgres:16 + restart: unless-stopped + environment: + PGDATA: /var/lib/postgresql/data/pgdata + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_DB: ${POSTGRES_DB} + ports: + - "5432:5432" + healthcheck: + test: [ "CMD", "pg_isready", "-U", "${POSTGRES_USER}" ] + retries: 5 + interval: 10s + start_period: 10s volumes: - tileserver-src: - driver: local + tileserver-src: + driver: local diff --git a/docker-compose.yml b/docker-compose.yml index 87c972e36..98940d183 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,235 +1,211 @@ services: - webclient: - image: ghcr.io/tum-dev/navigatum-webclient:main - restart: unless-stopped - labels: - - "traefik.enable=true" - - "traefik.http.routers.navigatum-webclient.entrypoints=webs" - - "traefik.http.routers.navigatum-webclient.tls.certresolver=leacme" - - "traefik.http.routers.navigatum-webclient.rule=Host(`nav.tum.de`)" - - "traefik.http.services.navigatum-webclient.loadbalancer.server.port=3000" - networks: - - traefik_traefik - - navigatum - expose: - - "3000" - environment: - NUXT_PUBLIC_MAPS_URL: https://nav.tum.de - NUXT_PUBLIC_CDN_URL: https://nav.tum.de - NUXT_PUBLIC_API_URL: https://nav.tum.de - NUXT_PUBLIC_FEEDBACK_URL: https://nav.tum.de - depends_on: - tileserver: - condition: service_healthy - data: - condition: service_healthy - main-api: - condition: service_healthy - feedback-api: - condition: service_healthy - # maps - tileserver-init-sprites: - image: alpine:latest - command: sh -c "mkdir -p /data/sprites/ && rm -f /data/sprites/* && wget -P /data/sprites ${TILE_SPRITES_URL}/osm-liberty.json ${TILE_SPRITES_URL}/osm-liberty@2x.json ${TILE_SPRITES_URL}/osm-liberty.png ${TILE_SPRITES_URL}/osm-liberty@2x.png" - volumes: - - tileserver-src:/data - tileserver-init-config: - image: alpine:latest - command: sh -c "apk update --no-progress --quiet && apk add rsync --no-progress --quiet && rsync /to_data/ /data/" - volumes: - - tileserver-src:/data - - ./deployment/k3s/files/:/to_data/:ro - depends_on: - tileserver-init-sprites: - condition: service_completed_successfully - tileserver-srv-src: # needed for simpler initialisation of dev environments - image: nginx:1.26 - restart: unless-stopped - labels: - - "traefik.enable=true" - - "traefik.http.routers.navigatum-tileserver-srv.entrypoints=webs" - - "traefik.http.routers.navigatum-tileserver-srv.tls.certresolver=leacme" - - "traefik.http.routers.navigatum-tileserver-srv.rule=Host(`nav.tum.de`) && PathPrefix(`/maps/vol/`)" - - "traefik.http.services.navigatum-tileserver-srv.loadbalancer.server.port=80" - networks: - - traefik_traefik - - navigatum - expose: - - 80 - volumes: - - tileserver-src:/usr/share/nginx/html/maps/vol:ro - depends_on: - tileserver-init-config: - condition: service_completed_successfully - tileserver-init-sprites: - condition: service_completed_successfully - healthcheck: - test: service nginx status || exit 1 - retries: 2 - interval: 10s - start_period: 10s - tileserver: - image: maptiler/tileserver-gl:latest - restart: unless-stopped - labels: - - "traefik.enable=true" - - "traefik.http.routers.navigatum-tileserver.entrypoints=webs" - - "traefik.http.routers.navigatum-tileserver.tls.certresolver=leacme" - - "traefik.http.routers.navigatum-tileserver.rule=Host(`nav.tum.de`) && PathPrefix(`/maps/`)" - - "traefik.http.routers.navigatum-tileserver.middlewares=navigatum-stripprefix@docker, navigatum-compress@docker, navigatum-compress@docker, navigatum-cache-2m@docker" - - "traefik.http.middlewares.navigatum-stripprefix.stripprefix.prefixes=/maps" - - "traefik.http.middlewares.navigatum-compress.compress=true" - - "traefik.http.middlewares.navigatum-cache-2m.headers.customrequestheaders.Cache-Control=public, max-age=5184000" - - "traefik.http.services.navigatum-tileserver.loadbalancer.server.port=8080" - networks: - - traefik_traefik - - navigatum - expose: - - "8080" - command: /usr/src/app/docker-entrypoint.sh --public_url=https://nav.tum.de/maps/ - tmpfs: - - /tmp - volumes: - - tileserver-src:/data/:ro - read_only: true - depends_on: - tileserver-init-config: - condition: service_completed_successfully - tileserver-init-sprites: - condition: service_completed_successfully - # cdn - data: - image: ghcr.io/tum-dev/navigatum-data:main - restart: unless-stopped - labels: - - "traefik.enable=true" - - "traefik.http.routers.navigatum-data.entrypoints=webs" - - "traefik.http.routers.navigatum-data.tls.certresolver=leacme" - - "traefik.http.routers.navigatum-data.rule=Host(`nav.tum.de`) && PathPrefix(`/cdn/`)" - - "traefik.http.services.navigatum-data.loadbalancer.server.port=3002" - networks: - - traefik_traefik - - navigatum - expose: - - "3002" - # server - main-api: - image: ghcr.io/tum-dev/navigatum-server:main - restart: unless-stopped - labels: - - "traefik.enable=true" - - "traefik.http.routers.navigatum-main-api.entrypoints=webs" - - "traefik.http.routers.navigatum-main-api.tls.certresolver=leacme" - - "traefik.http.routers.navigatum-main-api.rule=Host(`nav.tum.de`) && PathPrefix(`/api/`)" - - "traefik.http.services.navigatum-main-api.loadbalancer.server.port=3003" - networks: - - traefik_traefik - - navigatum - expose: - - "3003" - command: /bin/navigatum-main-api - volumes: - - type: tmpfs - target: /home/navigatum/.cache - user: 1000:3000 - environment: - MIELI_URL: http://meilisearch:7700 - TILESERVER_URL: http://tileserver:8080 - CDN_URL: http://data:3002/cdn - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_URL: db:5432 - depends_on: - meilisearch: - condition: service_healthy - db: - condition: service_healthy - healthcheck: - test: wget -q --spider http://localhost:3003/api/status - retries: 5 - interval: 10s - start_period: 60s - meilisearch: - image: getmeili/meilisearch:v1.7.6 - restart: unless-stopped - networks: - - navigatum - expose: - - "7700" - healthcheck: - test: wget -q --spider http://localhost:7700/health - retries: 5 - interval: 10s - start_period: 10s - feedback-api: - image: ghcr.io/tum-dev/navigatum-server:main - restart: unless-stopped - labels: - - "traefik.enable=true" - - "traefik.http.routers.navigatum-feedback-api.entrypoints=webs" - - "traefik.http.routers.navigatum-feedback-api.tls.certresolver=leacme" - - "traefik.http.routers.navigatum-feedback-api.rule=Host(`nav.tum.de`) && PathPrefix(`/api/feedback`)" - - "traefik.http.services.navigatum-feedback-api.loadbalancer.server.port=3004" - networks: - - traefik_traefik - - navigatum - expose: - - "3004" - command: /bin/navigatum-feedback - healthcheck: - test: wget -q --spider http://localhost:3004/api/feedback/status - retries: 5 - start_period: 10s - db: - image: postgres:16 - restart: unless-stopped - environment: - PGDATA: /var/lib/postgresql/data/pgdata - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_DB: ${POSTGRES_DB} - networks: - - navigatum - expose: - - "5432" - volumes: - - postgres-data:/var/lib/postgresql/data - healthcheck: - test: [ "CMD", "pg_isready", "-U", "${POSTGRES_USER}" ] - retries: 5 - interval: 10s - start_period: 10s - redirect: - image: traefik/whoami - restart: unless-stopped - labels: - - "traefik.enable=true" - - "traefik.http.routers.navigatum-redirect-unsecure.entrypoints=web" - - "traefik.http.routers.navigatum-redirect-unsecure.rule=Host(`nav.tum.sexy`) || Host(`www.nav.tum.de`) || Host(`nav.tum.de`)" - - "traefik.http.routers.navigatum-redirect-unsecure.middlewares=navigatum-webs-redirectscheme" - - "traefik.http.middlewares.navigatum-webs-redirectscheme.redirectscheme.scheme=https" - - "traefik.http.middlewares.navigatum-webs-redirectscheme.redirectscheme.permanent=true" - - "traefik.http.routers.navigatum-webs-redirectscheme.service=noop@internal" + webclient: + image: ghcr.io/tum-dev/navigatum-webclient:main + restart: unless-stopped + labels: + - "traefik.enable=true" + - "traefik.http.routers.navigatum-webclient.entrypoints=webs" + - "traefik.http.routers.navigatum-webclient.tls.certresolver=leacme" + - "traefik.http.routers.navigatum-webclient.rule=Host(`nav.tum.de`)" + - "traefik.http.services.navigatum-webclient.loadbalancer.server.port=3000" + networks: + - traefik_traefik + - navigatum + expose: + - "3000" + environment: + NUXT_PUBLIC_MAPS_URL: https://nav.tum.de + NUXT_PUBLIC_CDN_URL: https://nav.tum.de + NUXT_PUBLIC_API_URL: https://nav.tum.de + NUXT_PUBLIC_FEEDBACK_URL: https://nav.tum.de + depends_on: + tileserver: + condition: service_healthy + data: + condition: service_healthy + main-api: + condition: service_healthy + feedback-api: + condition: service_healthy + # maps + tileserver-init-sprites: + image: alpine:latest + command: sh -c "mkdir -p /data/sprites/ && rm -f /data/sprites/* && wget -P /data/sprites ${TILE_SPRITES_URL}/osm-liberty.json ${TILE_SPRITES_URL}/osm-liberty@2x.json ${TILE_SPRITES_URL}/osm-liberty.png ${TILE_SPRITES_URL}/osm-liberty@2x.png" + volumes: + - tileserver-src:/data + tileserver-init-config: + image: alpine:latest + command: sh -c "apk update --no-progress --quiet && apk add rsync --no-progress --quiet && rsync /to_data/ /data/" + volumes: + - tileserver-src:/data + - ./deployment/k3s/files/:/to_data/:ro + depends_on: + tileserver-init-sprites: + condition: service_completed_successfully + tileserver-srv-src: # needed for simpler initialisation of dev environments + image: nginx:1.26 + restart: unless-stopped + labels: + - "traefik.enable=true" + - "traefik.http.routers.navigatum-tileserver-srv.entrypoints=webs" + - "traefik.http.routers.navigatum-tileserver-srv.tls.certresolver=leacme" + - "traefik.http.routers.navigatum-tileserver-srv.rule=Host(`nav.tum.de`) && PathPrefix(`/maps/vol/`)" + - "traefik.http.services.navigatum-tileserver-srv.loadbalancer.server.port=80" + networks: + - traefik_traefik + - navigatum + expose: + - 80 + volumes: + - tileserver-src:/usr/share/nginx/html/maps/vol:ro + depends_on: + tileserver-init-config: + condition: service_completed_successfully + tileserver-init-sprites: + condition: service_completed_successfully + healthcheck: + test: service nginx status || exit 1 + retries: 2 + interval: 10s + start_period: 10s + tileserver: + image: maptiler/tileserver-gl:latest + restart: unless-stopped + labels: + - "traefik.enable=true" + - "traefik.http.routers.navigatum-tileserver.entrypoints=webs" + - "traefik.http.routers.navigatum-tileserver.tls.certresolver=leacme" + - "traefik.http.routers.navigatum-tileserver.rule=Host(`nav.tum.de`) && PathPrefix(`/maps/`)" + - "traefik.http.routers.navigatum-tileserver.middlewares=navigatum-stripprefix@docker, navigatum-compress@docker, navigatum-compress@docker, navigatum-cache-2m@docker" + - "traefik.http.middlewares.navigatum-stripprefix.stripprefix.prefixes=/maps" + - "traefik.http.middlewares.navigatum-compress.compress=true" + - "traefik.http.middlewares.navigatum-cache-2m.headers.customrequestheaders.Cache-Control=public, max-age=5184000" + - "traefik.http.services.navigatum-tileserver.loadbalancer.server.port=8080" + networks: + - traefik_traefik + - navigatum + expose: + - "8080" + command: /usr/src/app/docker-entrypoint.sh --public_url=https://nav.tum.de/maps/ + tmpfs: + - /tmp + volumes: + - tileserver-src:/data/:ro + read_only: true + depends_on: + tileserver-init-config: + condition: service_completed_successfully + tileserver-init-sprites: + condition: service_completed_successfully + # cdn + data: + image: ghcr.io/tum-dev/navigatum-data:main + restart: unless-stopped + labels: + - "traefik.enable=true" + - "traefik.http.routers.navigatum-data.entrypoints=webs" + - "traefik.http.routers.navigatum-data.tls.certresolver=leacme" + - "traefik.http.routers.navigatum-data.rule=Host(`nav.tum.de`) && PathPrefix(`/cdn/`)" + - "traefik.http.services.navigatum-data.loadbalancer.server.port=3002" + networks: + - traefik_traefik + - navigatum + expose: + - "3002" + # server + main-api: + image: ghcr.io/tum-dev/navigatum-server:main + restart: unless-stopped + labels: + - "traefik.enable=true" + - "traefik.http.routers.navigatum-main-api.entrypoints=webs" + - "traefik.http.routers.navigatum-main-api.tls.certresolver=leacme" + - "traefik.http.routers.navigatum-main-api.rule=Host(`nav.tum.de`) && PathPrefix(`/api/`)" + - "traefik.http.services.navigatum-main-api.loadbalancer.server.port=3003" + networks: + - traefik_traefik + - navigatum + expose: + - "3003" + command: /bin/navigatum-main-api + volumes: + - type: tmpfs + target: /home/navigatum/.cache + user: 1000:3000 + environment: + MIELI_URL: http://meilisearch:7700 + TILESERVER_URL: http://tileserver:8080 + CDN_URL: http://data:3002/cdn + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_URL: db:5432 + depends_on: + meilisearch: + condition: service_healthy + db: + condition: service_healthy + meilisearch: + image: getmeili/meilisearch:v1.7.6 + restart: unless-stopped + networks: + - navigatum + expose: + - "7700" + healthcheck: + test: wget -q --spider http://localhost:7700/health + retries: 5 + interval: 10s + start_period: 10s + db: + image: postgres:16 + restart: unless-stopped + environment: + PGDATA: /var/lib/postgresql/data/pgdata + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_DB: ${POSTGRES_DB} + networks: + - navigatum + expose: + - "5432" + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: [ "CMD", "pg_isready", "-U", "${POSTGRES_USER}" ] + retries: 5 + interval: 10s + start_period: 10s + redirect: + image: traefik/whoami + restart: unless-stopped + labels: + - "traefik.enable=true" + - "traefik.http.routers.navigatum-redirect-unsecure.entrypoints=web" + - "traefik.http.routers.navigatum-redirect-unsecure.rule=Host(`nav.tum.sexy`) || Host(`www.nav.tum.de`) || Host(`nav.tum.de`)" + - "traefik.http.routers.navigatum-redirect-unsecure.middlewares=navigatum-webs-redirectscheme" + - "traefik.http.middlewares.navigatum-webs-redirectscheme.redirectscheme.scheme=https" + - "traefik.http.middlewares.navigatum-webs-redirectscheme.redirectscheme.permanent=true" + - "traefik.http.routers.navigatum-webs-redirectscheme.service=noop@internal" - - "traefik.http.routers.navigatum-redirect.entrypoints=webs" - - "traefik.http.routers.navigatum-redirect.tls.certresolver=leacme" - - "traefik.http.routers.navigatum-redirect.rule=Host(`nav.tum.sexy`) || Host(`www.nav.tum.de`)" - - "traefik.http.routers.navigatum-redirect.middlewares=navigatum-redirectregex@docker" - - "traefik.http.routers.navigatum-redirect.service=noop@internal" - - "traefik.http.middlewares.navigatum-redirectregex.redirectregex.regex=^(https?://.*)nav.tum.(app|sexy)/(.*)$$" - - "traefik.http.middlewares.navigatum-redirectregex.redirectregex.replacement=$${1}nav.tum.de/$${3}" - - "traefik.http.middlewares.navigatum-redirectregex.redirectregex.permanent=true" - networks: - - traefik_traefik + - "traefik.http.routers.navigatum-redirect.entrypoints=webs" + - "traefik.http.routers.navigatum-redirect.tls.certresolver=leacme" + - "traefik.http.routers.navigatum-redirect.rule=Host(`nav.tum.sexy`) || Host(`www.nav.tum.de`)" + - "traefik.http.routers.navigatum-redirect.middlewares=navigatum-redirectregex@docker" + - "traefik.http.routers.navigatum-redirect.service=noop@internal" + - "traefik.http.middlewares.navigatum-redirectregex.redirectregex.regex=^(https?://.*)nav.tum.(app|sexy)/(.*)$$" + - "traefik.http.middlewares.navigatum-redirectregex.redirectregex.replacement=$${1}nav.tum.de/$${3}" + - "traefik.http.middlewares.navigatum-redirectregex.redirectregex.permanent=true" + networks: + - traefik_traefik volumes: - tileserver-src: - driver: local - postgres-data: - driver: local + tileserver-src: + driver: local + postgres-data: + driver: local networks: - navigatum: - traefik_traefik: - external: true + navigatum: + traefik_traefik: + external: true diff --git a/server/Cargo.lock b/server/Cargo.lock index ba838a62f..889d3415b 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -2217,51 +2217,30 @@ dependencies = [ "typenum", ] -[[package]] -name = "navigatum-feedback" -version = "1.0.0" -dependencies = [ - "actix-cors", - "actix-governor", - "actix-web", - "actix-web-prom", - "base64 0.22.0", - "chrono", - "image", - "jsonwebtoken", - "log", - "octocrab", - "pretty_assertions", - "rand", - "regex", - "rustls 0.23.5", - "serde", - "serde_json", - "serde_yaml", - "structured-logger", - "tempfile", - "tokio", -] - [[package]] name = "navigatum-main-api" version = "1.0.0" dependencies = [ "ab_glyph", "actix-cors", + "actix-governor", "actix-web", "actix-web-prom", + "base64 0.22.0", "cached", "chrono", "futures", "image", "imageproc", + "jsonwebtoken", "lazy_static", "log", "logos", "meilisearch-sdk", "oauth2", + "octocrab", "pretty_assertions", + "rand", "regex", "reqwest 0.12.4", "rustls 0.23.5", @@ -2270,6 +2249,7 @@ dependencies = [ "serde_yaml", "sqlx", "structured-logger", + "tempfile", "tokio", "unicode-truncate", ] diff --git a/server/Cargo.toml b/server/Cargo.toml index 68d6725c1..e62e35ab5 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -1,29 +1,7 @@ [workspace] -members = ["feedback", "main-api"] +members = ["main-api"] resolver = "2" [profile.release] strip = true lto = "thin" - -[workspace.dependencies] -# logging/obeservability -log = "0.4.21" -structured-logger = "1.0.3" -actix-web-prom = { version = "0.8.0", default-features = false, features = [] } - -# runtime + webserver -tokio = { version = "1.37", features = ["full"] } -actix-web = { version = "4.5.1", default-features = false, features = ["macros", "compress-gzip", "cookies", "http2"] } -actix-cors = "0.7.0" -rustls = { version = "0.23.5", default-features = false, features = ["ring"] } # the aws' fips complient libary has weird bingen issues which require deeper looking into - -#serialisation -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" - -# testing -pretty_assertions = "1.4.0" - -# images -image = { version = "0.25.1", default-features = false, features = ["jpeg", "png", "webp"] } diff --git a/server/Dockerfile b/server/Dockerfile index 3961c9e44..4ba60e021 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -15,11 +15,8 @@ ENV RUSTFLAGS="-C target-feature=-crt-static" # (probably cached) first run of the image build => only dependencies COPY ./Cargo.* ./ COPY main-api/Cargo.* ./main-api/ -COPY feedback/Cargo.* ./feedback/ -RUN mkdir ./main-api/src/ ./main-api/src/setup/ ./feedback/src/ -RUN echo "fn main() { println!(\"Hello, world!\");}" > ./main-api/src/setup/mod.rs -RUN echo "fn main() { println!(\"Hello, world!\");}" > ./main-api/src/main.rs -RUN echo "fn main() { println!(\"Hello, world!\");}" > ./feedback/src/main.rs +RUN mkdir -p ./main-api/src/ \ + && echo "fn main() { println!(\"Hello, world!\");}" > ./main-api/src/main.rs RUN cargo build --release --workspace \ && rm -fr target/release/deps/navigatum* @@ -27,7 +24,6 @@ RUN cargo build --release --workspace \ # second run of the image build (including our code) COPY .sqlx .sqlx -COPY feedback/src ./feedback/src COPY main-api/src ./main-api/src COPY main-api/migrations ./main-api/migrations RUN cargo build --release --workspace @@ -54,13 +50,5 @@ USER navigatum ENTRYPOINT ["tini", "--"] -# Healthchecks and exopsing ports is ommited from the dockerfile, as this -# - is done by the kubernetes deployment -# - is not needed for local development -# - would prevent us merging the different servers into one dockerfile - -# These are coomands that can be chosen to run -# - a respective server or -# - an admin task -# CMD /bin/navigatum-feedback +HEALTHCHECK CMD curl --fail localhost:3003/api/status || exit 1 CMD /bin/navigatum-main-api diff --git a/server/README.md b/server/README.md index 756e7389a..7ff2d52b8 100644 --- a/server/README.md +++ b/server/README.md @@ -1,10 +1,7 @@ # Server -Our server is architected in different microservices, each of which is responsible for a specific task. +Our server is architected as a single monolith (with low coupling, chill). +The reason behind this is that we are a very small team (=mostly a singe dev) and +limit ourselves to the service itself. -- [main-api](/server/main-api): The main API server, which serves the API endpoints -- [feedback](/server/feedback): The feedback microservice, which allows users to submit feedback - This is separated from the server because: - - it has virtually no shared dependencies (natural faultline) - - this way, we can deploy the feedback-API independently of the main server (both in time, scaling and reliability) - - security: this way, we can increase our isolation and protect the GitHub token better ;) +Please see the [main-api](/server/main-api) for the main API server, which serves the API endpoints diff --git a/server/feedback/Cargo.toml b/server/feedback/Cargo.toml deleted file mode 100644 index 238cb54dd..000000000 --- a/server/feedback/Cargo.toml +++ /dev/null @@ -1,44 +0,0 @@ -[package] -name = "navigatum-feedback" -version = "1.0.0" -authors = ["Markus A ", "Frank Elsinga "] -edition = "2021" -description = "Navigating around TUM with excellence – A feedback system to submit feedback via github issues" -repository = "https://github.com/TUM-Dev/navigatum" -readme = "README.md" -license = "GPL-3.0" -keywords = ["website", "feedback", "api-rest", "tum"] - -[[bin]] -name = "navigatum-feedback" -path = "src/main.rs" - -[dependencies] -# shared -log.workspace = true -structured-logger.workspace = true -tokio.workspace = true -actix-web.workspace = true -actix-cors.workspace = true -actix-web-prom.workspace = true -serde.workspace = true -serde_json.workspace = true -rustls.workspace = true - -rand = "0.8.5" -regex = "1.10.4" -octocrab = { version = "0.38.0", default-features = false, features = ["rustls", "rustls-webpki-tokio", "retry", "default-client"] } - -# web -jsonwebtoken = { version = "9.3.0", default-features = false, features = [] } -chrono = { version = "0.4.38", default-features = false, features = [] } -actix-governor = { version = "0.5.0", features = ["logger"] } - -# proposing feedback -tempfile = "3.10.1" -image.workspace = true -base64 = "0.22.0" -serde_yaml = "0.9" - -[dev-dependencies] -pretty_assertions.workspace = true diff --git a/server/feedback/README.md b/server/feedback/README.md deleted file mode 100644 index 25b7831df..000000000 --- a/server/feedback/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# Feedback - -This folder contains the feedback-microservice for NavigaTUM. - -## Getting started - -### Prerequisites - -For getting started, there are some system dependencys which you will need. -Please follow the [system dependencys docs](/resources/documentation/Dependencys.md) before trying to run this part of our project. - -### Starting the server - -The following environment variables are required for all features to work: - -| variable | usage/description | -| -------------- | -------------------------------------------------------------------------------------------------- | -| `GITHUB_TOKEN` | A GitHub token with `write` access to `repo`. This is used to create issues/PRs on the repository. | -| `JWT_KEY` | A key used to sign JWTs. This is used to authenticate that feedback tokens were given out by us. | - -Run `cargo run` to start the server. -The server should now be available on `localhost:3004`. - -Note that `cargo run --release` is used to start the server for an optimised production build (use this if you want to profile performance, it makes quite a difference). - -### API-Changes - -#### Editing - -If you have made changes to the API, you need to update the API documentation. - -There are two editors for the API documentation (both are imperfect): - -- [Swagger Editor](https://editor.swagger.io/?url=https://raw.githubusercontent.com/TUM-Dev/navigatum/main/openapi.yaml) -- [stoplight](https://stoplight.io/) - -#### Testing - -Of course documentation is one part of the process. If the changes are substantial, you should also run an API-Fuzz-Test: -To make sure that this specification is up-to-date and without holes, we run [schemathesis](https://github.com/schemathesis/schemathesis) using the following command on API Server: - -```bash -python -m venv venv -source venv/bin/activate -pip install schemathesis -st run --workers=auto --base-url=http://localhost:3004 --checks=all ../openapi.yaml -``` - -Some fuzzing-goals may not be available for you locally, as they require prefix-routing (f.ex.`/cdn` to the CDN) and some fuzzing-goals are automatically tested in our CI. -You can exchange `--base-url=http://localhost:3004` to `--base-url=https://nav.tum.sexy` for the full public API, or restrict your scope using a option like `--endpoint=/api/feedback/`. - -## License - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - ---- diff --git a/server/feedback/src/main.rs b/server/feedback/src/main.rs deleted file mode 100644 index b1d013150..000000000 --- a/server/feedback/src/main.rs +++ /dev/null @@ -1,82 +0,0 @@ -use actix_cors::Cors; -use actix_governor::{GlobalKeyExtractor, Governor, GovernorConfigBuilder}; -use std::collections::HashMap; -use std::error::Error; - -use crate::tokens::RecordedTokens; -use actix_web::{get, middleware, web, App, HttpResponse, HttpServer}; -use actix_web_prom::PrometheusMetricsBuilder; -use structured_logger::async_json::new_writer; -use structured_logger::Builder; - -mod github; -mod post_feedback; -mod proposed_edits; -mod tokens; -type BoxedError = Box; - -const MAX_JSON_PAYLOAD: usize = 1024 * 1024; // 1 MB - -#[get("/api/feedback/status")] -async fn health_status_handler() -> HttpResponse { - let github_link = match std::env::var("GIT_COMMIT_SHA") { - Ok(hash) => format!("https://github.com/TUM-Dev/navigatum/tree/{hash}"), - Err(_) => "unknown commit hash, probably running in development".to_string(), - }; - HttpResponse::Ok() - .content_type("text/plain") - .body(format!("healthy\nsource_code: {github_link}")) -} - -const SECONDS_PER_DAY: u64 = 60 * 60 * 24; -#[tokio::main] -async fn main() -> std::io::Result<()> { - Builder::with_level("info") - .with_target_writer("*", new_writer(tokio::io::stdout())) - .init(); - - let feedback_ratelimit = GovernorConfigBuilder::default() - .key_extractor(GlobalKeyExtractor) - .per_second(SECONDS_PER_DAY / 300) // replenish new token every .. seconds - .burst_size(50) - .finish() - .expect("Invalid configuration of the governor"); - - // metrics - let labels = HashMap::from([( - "revision".to_string(), - std::env::var("GIT_COMMIT_SHA").unwrap_or_else(|_| "development".to_string()), - )]); - let prometheus = PrometheusMetricsBuilder::new("navigatum_feedback") - .endpoint("/api/feedback/metrics") - .const_labels(labels) - .build() - .unwrap(); - - let recorded_tokens = web::Data::new(RecordedTokens::default()); - HttpServer::new(move || { - let cors = Cors::default() - .allow_any_origin() - .allow_any_header() - .allowed_methods(vec!["GET", "POST"]) - .max_age(3600); - App::new() - .wrap(prometheus.clone()) - .wrap(cors) - .wrap(middleware::Logger::default().exclude("/api/feedback/status")) - .wrap(middleware::Compress::default()) - .app_data(web::JsonConfig::default().limit(MAX_JSON_PAYLOAD)) - .service(health_status_handler) - .app_data(recorded_tokens.clone()) - .service(post_feedback::send_feedback) - .service(proposed_edits::propose_edits) - .service( - web::scope("/api/feedback/get_token") - .wrap(Governor::new(&feedback_ratelimit)) - .route("", web::post().to(tokens::get_token)), - ) - }) - .bind(std::env::var("BIND_ADDRESS").unwrap_or_else(|_| "0.0.0.0:3004".to_string()))? - .run() - .await -} diff --git a/server/main-api/Cargo.toml b/server/main-api/Cargo.toml index c11167097..b13f164f2 100644 --- a/server/main-api/Cargo.toml +++ b/server/main-api/Cargo.toml @@ -15,24 +15,26 @@ name = "navigatum-main-api" path = "src/main.rs" [dependencies] -# shared -log.workspace = true -structured-logger.workspace = true -tokio.workspace = true -actix-web.workspace = true -actix-cors.workspace = true -actix-web-prom.workspace = true -serde.workspace = true -serde_json.workspace = true -rustls.workspace = true +# logging/obeservability +log = "0.4.21" +structured-logger = "1.0.3" +actix-web-prom = { version = "0.8.0", default-features = false, features = [] } + +#serialisation +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +serde_yaml = "0.9" + +# runtime + webserver +tokio = { version = "1.37", features = ["full"] } +actix-web = { version = "4.5.1", default-features = false, features = ["macros", "compress-gzip", "cookies", "http2"] } +actix-cors = "0.7.0" +rustls = { version = "0.23.5", default-features = false, features = ["ring"] } # the aws' fips complient libary has weird bingen issues which require deeper looking into cached = { version = "0.50.0", features = ["default", "async", "tokio", "disk_store"] } futures = "0.3.30" unicode-truncate = "1.0.0" -# setup -serde_yaml = "0.9" - # database sqlx = { version = "0.7.4", features = ["postgres", "runtime-tokio-rustls", "migrate", "macros", "chrono"] } chrono = { version = "0.4.38", default-features = false, features = ["serde"] } @@ -47,13 +49,24 @@ oauth2 = { version = "4.4.2", default-features = false, features = ["rustls-tls" reqwest = { version = "0.12.4", default-features = false, features = ["rustls-tls", "http2", "charset", "json", "gzip", "hickory-dns"] } # image production -image.workspace = true +image = { version = "0.25.1", default-features = false, features = ["jpeg", "png", "webp"] } imageproc = "0.24.0" ab_glyph = "0.2.25" lazy_static = "1.4.0" +rand = "0.8.5" +octocrab = { version = "0.38.0", default-features = false, features = ["rustls", "rustls-webpki-tokio", "retry", "default-client"] } + +# auth/security +jsonwebtoken = { version = "9.3.0", default-features = false, features = [] } +actix-governor = { version = "0.5.0", features = ["logger"] } + +# proposing feedback +tempfile = "3.10.1" +base64 = "0.22.0" + [dev-dependencies] -pretty_assertions.workspace = true +pretty_assertions = "1.4.0" [features] skip_db_setup = [] diff --git a/server/main-api/README.md b/server/main-api/README.md index 10cb3e430..897c9f593 100644 --- a/server/main-api/README.md +++ b/server/main-api/README.md @@ -7,14 +7,17 @@ This folder contains the main backend server for NavigaTUM. ### Prerequisites For getting started, there are some system dependencys which you will need. -Please follow the [system dependencys docs](/resources/documentation/Dependencys.md) before trying to run this part of our project. +Please follow the [system dependencys docs](/resources/documentation/Dependencys.md) before trying to run this part of +our project. ### Additional dependency We have a few API endpoints which require additional dependencies. -As a general rule of thumb, if you probably want to **skip the tileserver**, but need to **do the SQLite Database** and **MeiliSearch** setup. -The reason for this is, that the `preview` endpoint is the only endpoint, which requires the tileserver and said endpoint is a non-essential part of the project. +As a general rule of thumb, if you probably want to **skip the tileserver**, but need to **do the SQLite Database** and +**MeiliSearch** setup. +The reason for this is, that the `preview` endpoint is the only endpoint, which requires the tileserver and said +endpoint is a non-essential part of the project. #### How to Set up the Databases @@ -22,7 +25,8 @@ At the beginning of the main api we set up both meilisearch and the database. This will ensure that the sqlite database and meilisearch index is created. This requires meilisearch to be online. -To set up [MeiliSearch](https://github.com/meilisearch/MeiliSearch), either follow their installation instructions or use +To set up [MeiliSearch](https://github.com/meilisearch/MeiliSearch), either follow their installation instructions or +use ```bash docker run -it --rm -p 7700:7700 getmeili/meilisearch:latest @@ -39,10 +43,22 @@ docker run -it --rm -e POSTGRES_PASSWORD=password -p 5432:5432 postgres:latest ### Starting the server Run `cargo run` to start the server. -The server should now be available on `localhost:8080`. +The server should now be available on `localhost:8080` if you have configured the correct environment. > [!NOTE] -> `cargo run --release` is used to start the server for an optimised production build (use this if you want to profile the `search` or `preview` functions, it makes quite a difference). +> `cargo run --release` is used to start the server for an optimised production build (use this if you want to profile +> the `search` or `preview` functions, it makes quite a difference). + +### Environment Variables + +| variable | module | | usage/description | +|-----------------------------------|----------------------------------|-----------------------------------------|--------------------------------------------------------------------------------------------------------| +| `POSTGRES_{USER,PASSWORD,URL,DB}` | [`all`](./main.rs) | required | Used to connect to the db | +| `GIT_COMMIT_SHA` | [`main`](./main.rs) | optional | Shown in the status endpint (also set at build time in docker) | +| `GITHUB_TOKEN` | [`feedback`](./feeedback/mod.rs) | | A GitHub token with `write` access to `repo`.
This is used to create issues/PRs on the repository. | +| `JWT_KEY` | [`feedback`](./feeedback/mod.rs) | | A key used to sign JWTs.
This is used to authenticate that feedback tokens were given out by us. | +| `MIELI_{URL,MASTER_KEY}` | [`search`](./search/mod.rs) | | Allows searching via meiliserch | +| `CDN_URL` | [`setup`](./setup/mod.rs) | required
can be skipped via flags | Source of truth of the data | ### Adding Migrations @@ -71,7 +87,8 @@ cargo sqlx prepare --database-url postgres://postgres:password@localhost:5432/po ### How to Set up the tileserver (needed for the `preview` endpoint) -To set up your tileserver, head over to the [`map`](https://github.com/TUM-Dev/NavigaTUM/tree/main/map) folder and follow the instructions there. +To set up your tileserver, head over to the [`map`](https://github.com/TUM-Dev/NavigaTUM/tree/main/map) folder and +follow the instructions there. ### API-Changes @@ -87,18 +104,21 @@ There are two editors for the API documentation (both are imperfect): #### Testing Of course documentation is one part of the process. -Run API-Fuzz-Test and [schemathesis](https://github.com/schemathesis/schemathesis) on API Server to ensure specification is up-to-date and without holes. +Run API-Fuzz-Test and [schemathesis](https://github.com/schemathesis/schemathesis) on API Server to ensure specification +is up-to-date and without holes. To do so, run the following commands against the API Server: ```bash python -m venv venv source venv/bin/activate pip install schemathesis -st run --workers=auto --base-url=http://localhost:8080 --checks=all ../openapi.yaml +st run --workers=auto --base-url=http://localhost:3003 --checks=all ../openapi.yaml ``` -Some fuzzing-goals may not be available for you locally, as they require prefix-routing (f.ex.`/cdn` to the CDN) and some fuzzing-goals are automatically tested in our CI. -You can exchange `--base-url=http://localhost:8080` to `--base-url=https://nav.tum.sexy` for the full public API, or restrict your scope using an option like `--endpoint=/api/search`. +Some fuzzing-goals may not be available for you locally, as they require prefix-routing (f.ex.`/cdn` to the CDN) and +some fuzzing-goals are automatically tested in our CI. +You can exchange `--base-url=http://localhost:3003` to `--base-url=https://nav.tum.de` for the full public API, or +restrict your scope using an option like `--endpoint=/api/search`. ## License diff --git a/server/feedback/src/github.rs b/server/main-api/src/feedback/github.rs similarity index 100% rename from server/feedback/src/github.rs rename to server/main-api/src/feedback/github.rs diff --git a/server/main-api/src/feedback/mod.rs b/server/main-api/src/feedback/mod.rs new file mode 100644 index 000000000..1530a62b3 --- /dev/null +++ b/server/main-api/src/feedback/mod.rs @@ -0,0 +1,28 @@ +use actix_governor::{GlobalKeyExtractor, Governor, GovernorConfigBuilder}; +use actix_web::web; + +mod github; +mod post_feedback; +mod proposed_edits; +mod tokens; + +const SECONDS_PER_DAY: u64 = 60 * 60 * 24; + +pub fn configure(cfg: &mut web::ServiceConfig) { + let feedback_ratelimit = GovernorConfigBuilder::default() + .key_extractor(GlobalKeyExtractor) + .per_second(SECONDS_PER_DAY / 300) // replenish new token every .. seconds + .burst_size(50) + .finish() + .expect("Invalid configuration of the governor"); + + let recorded_tokens = web::Data::new(tokens::RecordedTokens::default()); + cfg.app_data(recorded_tokens.clone()) + .service(post_feedback::send_feedback) + .service(proposed_edits::propose_edits) + .service( + web::scope("/get_token") + .wrap(Governor::new(&feedback_ratelimit)) + .route("", web::post().to(tokens::get_token)), + ); +} diff --git a/server/feedback/src/post_feedback.rs b/server/main-api/src/feedback/post_feedback.rs similarity index 95% rename from server/feedback/src/post_feedback.rs rename to server/main-api/src/feedback/post_feedback.rs index e6f4cf884..17711874d 100644 --- a/server/feedback/src/post_feedback.rs +++ b/server/main-api/src/feedback/post_feedback.rs @@ -1,11 +1,11 @@ -use crate::github; +use actix_web::post; use actix_web::web::{Data, Json}; use actix_web::HttpResponse; - -use crate::tokens::RecordedTokens; -use actix_web::post; use serde::Deserialize; +use super::github; +use super::tokens::RecordedTokens; + #[derive(Deserialize)] pub struct FeedbackPostData { token: String, @@ -16,7 +16,7 @@ pub struct FeedbackPostData { deletion_requested: bool, } -#[post("/api/feedback/feedback")] +#[post("/feedback")] pub async fn send_feedback( recorded_tokens: Data, req_data: Json, diff --git a/server/feedback/src/proposed_edits/coordinate.rs b/server/main-api/src/feedback/proposed_edits/coordinate.rs similarity index 99% rename from server/feedback/src/proposed_edits/coordinate.rs rename to server/main-api/src/feedback/proposed_edits/coordinate.rs index f631f869b..73b5d0fd2 100644 --- a/server/feedback/src/proposed_edits/coordinate.rs +++ b/server/main-api/src/feedback/proposed_edits/coordinate.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use serde::Deserialize; -use crate::proposed_edits::AppliableEdit; +use super::AppliableEdit; struct CoordinateFile { path: PathBuf, diff --git a/server/feedback/src/proposed_edits/discription.rs b/server/main-api/src/feedback/proposed_edits/discription.rs similarity index 96% rename from server/feedback/src/proposed_edits/discription.rs rename to server/main-api/src/feedback/proposed_edits/discription.rs index bb90bca41..955ff09a8 100644 --- a/server/feedback/src/proposed_edits/discription.rs +++ b/server/main-api/src/feedback/proposed_edits/discription.rs @@ -1,7 +1,8 @@ -use crate::proposed_edits::AppliableEdit; use std::collections::HashMap; use std::path::Path; +use super::AppliableEdit; + #[derive(Default)] pub struct Description { pub title: String, @@ -37,7 +38,6 @@ impl Description { #[cfg(test)] mod test_discription { - use crate::proposed_edits::AppliableEdit; use std::collections::HashMap; use std::path::Path; diff --git a/server/feedback/src/proposed_edits/image.rs b/server/main-api/src/feedback/proposed_edits/image.rs similarity index 99% rename from server/feedback/src/proposed_edits/image.rs rename to server/main-api/src/feedback/proposed_edits/image.rs index b68c55cce..f5377bae0 100644 --- a/server/feedback/src/proposed_edits/image.rs +++ b/server/main-api/src/feedback/proposed_edits/image.rs @@ -9,7 +9,7 @@ use log::error; use serde::Deserialize; use serde::Serialize; -use crate::proposed_edits::AppliableEdit; +use super::AppliableEdit; #[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq)] pub struct Source { diff --git a/server/feedback/src/proposed_edits/mod.rs b/server/main-api/src/feedback/proposed_edits/mod.rs similarity index 94% rename from server/feedback/src/proposed_edits/mod.rs rename to server/main-api/src/feedback/proposed_edits/mod.rs index 11ecd5298..7fb91d368 100644 --- a/server/feedback/src/proposed_edits/mod.rs +++ b/server/main-api/src/feedback/proposed_edits/mod.rs @@ -1,13 +1,15 @@ -use crate::github; -use crate::proposed_edits::coordinate::Coordinate; -use crate::proposed_edits::image::Image; -use crate::proposed_edits::tmp_repo::TempRepo; -use crate::tokens::RecordedTokens; +use std::collections::HashMap; +use std::path::Path; + use actix_web::web::{Data, Json}; use actix_web::{post, HttpResponse}; use serde::Deserialize; -use std::collections::HashMap; -use std::path::Path; + +use super::github; +use super::proposed_edits::coordinate::Coordinate; +use super::proposed_edits::image::Image; +use super::proposed_edits::tmp_repo::TempRepo; +use super::tokens::RecordedTokens; mod coordinate; mod discription; @@ -81,7 +83,7 @@ impl EditRequest { } } -#[post("/api/feedback/propose_edit")] +#[post("/propose_edit")] pub async fn propose_edits( recorded_tokens: Data, req_data: Json, diff --git a/server/feedback/src/proposed_edits/tmp_repo.rs b/server/main-api/src/feedback/proposed_edits/tmp_repo.rs similarity index 98% rename from server/feedback/src/proposed_edits/tmp_repo.rs rename to server/main-api/src/feedback/proposed_edits/tmp_repo.rs index 4e97d254e..6edcab330 100644 --- a/server/feedback/src/proposed_edits/tmp_repo.rs +++ b/server/main-api/src/feedback/proposed_edits/tmp_repo.rs @@ -1,8 +1,8 @@ -use crate::proposed_edits::EditRequest; use log::{debug, info}; use tokio::process::Command; -use crate::proposed_edits::discription::Description; +use super::discription::Description; +use super::EditRequest; pub struct TempRepo { dir: tempfile::TempDir, @@ -110,9 +110,10 @@ impl TempRepo { #[cfg(test)] mod tests { - use super::*; use std::fs; + use super::*; + const GIT_URL: &str = "https://github.com/CommanderStorm/dotfiles.git"; #[tokio::test] async fn test_new() { diff --git a/server/feedback/src/tokens.rs b/server/main-api/src/feedback/tokens.rs similarity index 100% rename from server/feedback/src/tokens.rs rename to server/main-api/src/feedback/tokens.rs diff --git a/server/main-api/src/main.rs b/server/main-api/src/main.rs index 6c172a5ef..20ff1196b 100644 --- a/server/main-api/src/main.rs +++ b/server/main-api/src/main.rs @@ -13,6 +13,7 @@ use structured_logger::Builder; mod calendar; mod details; +mod feedback; mod maps; mod models; mod search; @@ -56,7 +57,7 @@ fn connection_string() -> String { } #[tokio::main] -async fn main() -> Result<(), crate::BoxedError> { +async fn main() -> Result<(), BoxedError> { Builder::with_level("info") .with_target_writer("*", new_writer(tokio::io::stdout())) .init(); @@ -96,6 +97,7 @@ async fn main() -> Result<(), crate::BoxedError> { .service(health_status_handler) .service(calendar::calendar_handler) .service(web::scope("/api/preview").configure(maps::configure)) + .service(web::scope("/api/feedback").configure(feedback::configure)) .service(details::get_handler) .service(search::search_handler) })