diff --git a/.github/workflows/server-cicd.yml b/.github/workflows/server-cicd.yml index cbb0df9d3..d390ebd43 100644 --- a/.github/workflows/server-cicd.yml +++ b/.github/workflows/server-cicd.yml @@ -69,21 +69,3 @@ jobs: deployment: feedback secrets: ARGOCD_TOKEN: ${{ secrets.ARGOCD_TOKEN }} - calendar-deployment: - uses: ./.github/workflows/_restart-argocd.yml - if: ${{ github.ref }} == 'refs/heads/main' - needs: - - server-build - with: - deployment: calendar - secrets: - ARGOCD_TOKEN: ${{ secrets.ARGOCD_TOKEN }} - server-deployment: - uses: ./.github/workflows/_restart-argocd.yml - if: ${{ github.ref }} == 'refs/heads/main' - needs: - - server-build - with: - deployment: server - secrets: - ARGOCD_TOKEN: ${{ secrets.ARGOCD_TOKEN }} diff --git a/deployment/k3s/templates/deployments/calendar/calendar-cronjob.yaml b/deployment/k3s/templates/deployments/calendar/calendar-cronjob.yaml deleted file mode 100644 index 12d418fc5..000000000 --- a/deployment/k3s/templates/deployments/calendar/calendar-cronjob.yaml +++ /dev/null @@ -1,56 +0,0 @@ -{{- if eq "nav.tum.de" $.Values.url }} -{{ range $scrape_task := $.Values.calendar.scrape_tasks }} ---- -apiVersion: batch/v1 -kind: CronJob -metadata: - name: {{ $scrape_task.name }} - labels: - app.kubernetes.io/part-of: navigatum - app.kubernetes.io/name: maps - namespace: {{ $.Values.namespace }} -spec: - schedule: {{ $scrape_task.schedule | quote }} - jobTemplate: - spec: - template: - spec: - priorityClassName: {{ $.Values.url }} - containers: - - name: {{ $scrape_task.name }} - image: "ghcr.io/tum-dev/navigatum-server:{{ $.Values.tag }}" - imagePullPolicy: Always - command: ["/bin/navigatum-calendar-scraper"] - env: - - name: PUSHGATEWAY_URL - value: prometheus-pushgateway.monitoring.svc.cluster.local:9091 - - name: POSTGRES_URL - value: postgres-rw.navigatum.svc.cluster.local - - name: POSTGRES_USER - valueFrom: - secretKeyRef: - name: postgres-db-secret - key: user - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: postgres-db-secret - key: password - - name: SCRAPED_TIME_WINDOW_MONTHS - value: {{ $scrape_task.scraped_time_window_months | quote }} - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: - - ALL - resources: - requests: - cpu: 500m - memory: 100Mi - limits: - cpu: 1000m - memory: 500Mi - restartPolicy: OnFailure -{{ end }} -{{ end }} diff --git a/deployment/k3s/templates/deployments/calendar/calendar-deployment.yaml b/deployment/k3s/templates/deployments/calendar/calendar-deployment.yaml deleted file mode 100644 index 8117886c2..000000000 --- a/deployment/k3s/templates/deployments/calendar/calendar-deployment.yaml +++ /dev/null @@ -1,73 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: calendar - labels: - app.kubernetes.io/part-of: navigatum - app.kubernetes.io/name: calendar - namespace: {{ $.Values.namespace }} -spec: - replicas: 1 - revisionHistoryLimit: 0 - selector: - matchLabels: - app.kubernetes.io/part-of: navigatum - app.kubernetes.io/name: calendar - template: - metadata: - labels: - app.kubernetes.io/part-of: navigatum - app.kubernetes.io/name: calendar - {{- if eq "nav.tum.de" $.Values.url }} - annotations: - prometheus.io/path: /metrics - prometheus.io/port: '8060' - prometheus.io/scrape: 'true' - {{- end }} - spec: - priorityClassName: {{ $.Values.url }} - containers: - - name: calendar - image: "ghcr.io/tum-dev/navigatum-server:{{ $.Values.tag }}" - imagePullPolicy: Always - command: ["/bin/navigatum-calendar"] - ports: - - containerPort: 3005 - name: calendar - env: - - name: POSTGRES_URL - value: postgres-rw.navigatum.svc.cluster.local - - name: POSTGRES_USER - valueFrom: - secretKeyRef: - name: postgres-db-secret - key: user - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: postgres-db-secret - key: password - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: - - ALL - resources: - requests: - cpu: 10m - memory: 50Mi - limits: - memory: 500Mi - livenessProbe: - httpGet: - path: /api/calendar/status - port: calendar - failureThreshold: 5 - periodSeconds: 1 - startupProbe: - httpGet: - path: /api/calendar/status - port: calendar - failureThreshold: 240 - periodSeconds: 1 diff --git a/deployment/k3s/templates/deployments/calendar/posgres-deployment.yaml b/deployment/k3s/templates/deployments/postgres/posgres-deployment.yaml similarity index 100% rename from deployment/k3s/templates/deployments/calendar/posgres-deployment.yaml rename to deployment/k3s/templates/deployments/postgres/posgres-deployment.yaml diff --git a/deployment/k3s/templates/deployments/calendar/postgres-volume.yaml b/deployment/k3s/templates/deployments/postgres/postgres-volume.yaml similarity index 100% rename from deployment/k3s/templates/deployments/calendar/postgres-volume.yaml rename to deployment/k3s/templates/deployments/postgres/postgres-volume.yaml diff --git a/deployment/k3s/templates/deployments/webclient-deployment.yaml b/deployment/k3s/templates/deployments/webclient-deployment.yaml index 7eaeb7aa9..ff7c91826 100644 --- a/deployment/k3s/templates/deployments/webclient-deployment.yaml +++ b/deployment/k3s/templates/deployments/webclient-deployment.yaml @@ -41,8 +41,6 @@ spec: value: http://api-svc.navigatum.svc.cluster.local:3003 - name: FEEDBACK_API_URL value: http://feedback-svc.navigatum.svc.cluster.local:3004 - - name: CALENDAR_API_URL - value: http://calendar-svc.navigatum.svc.cluster.local:3005 resources: requests: cpu: 5m diff --git a/docker-compose.yml b/docker-compose.yml index 1d3d822bd..5e7e95e6f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,6 @@ services: CDN_URL: http://data:3002 MAIN_API_URL: http://main-api:3003 FEEDBACK_API_URL: http://feedback-api:3004 - CALENDAR_API_URL: http://calendar-api:3005 depends_on: tileserver: condition: service_healthy @@ -20,8 +19,6 @@ services: condition: service_healthy feedback-api: condition: service_healthy - calendar-api: - condition: service_healthy # maps tileserver-init-sprites: image: alpine:latest @@ -112,25 +109,6 @@ services: test: wget -q --spider http://localhost:3004/api/feedback/status retries: 5 start_period: 10s - calendar-api: - image: ghcr.io/tum-dev/navigatum-server:latest - restart: always - build: ./server - command: /bin/navigatum-calendar - environment: - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_URL: db:5432 - depends_on: - db: - condition: service_healthy - ports: - - "3005:3005" - healthcheck: - test: wget -q --spider http://localhost:3005/api/calendar/status - retries: 5 - start_period: 10s db: image: postgres:16 restart: unless-stopped diff --git a/openapi.yaml b/openapi.yaml index a641b2ba1..d9b84b3c6 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -733,26 +733,42 @@ paths: '/api/calendar/{id}': get: operationId: calendar - summary: Get a entry-preview + summary: Retrieve Calendar Entries description: | - This returns the a 1200x630px preview for the entry (room/building/..). + Retrieves calendar entries for a specific `id` within the requested time span. + The time span is defined by the `start_after` and `end_before` query parameters. + Ensure to provide valid date-time formats for these parameters. - This is usefull for implementing custom OpenGraph images for detail previews. + If successful, returns additional entries in the requested time span. parameters: - - name: start + - name: start_after in: query description: The first allowed time the calendar would like to display required: true schema: type: string format: date-time - - name: end + examples: + use_tz: + summary: how timezones are defined + value: '2039-01-19T03:14:07+01:00' + utc: + summary: how utc is defined + value: '2042-01-07T00:00:00 UTC' + - name: end_before in: query description: The last allowed time the calendar would like to display required: true schema: type: string format: date-time + examples: + use_tz: + summary: how timezones are defined + value: '2039-01-19T03:14:07+01:00' + utc: + summary: how utc is defined + value: '2042-01-07T00:00:00 UTC' - name: id in: path description: string you want to search for @@ -822,9 +838,9 @@ paths: operationId: previews summary: Get a entry-preview description: | - This returns the a 1200x630px preview for the entry (room/building/..). + This returns a 1200x630px preview for the location (room/building/..). - This is usefull for implementing custom OpenGraph images for detail previews. + This is usefully for implementing custom OpenGraph images for detail previews. parameters: - name: lang in: query @@ -1235,26 +1251,6 @@ paths: description: Service Unavailable tags: - health - /api/calendar/status: - get: - operationId: calendar-health - summary: calendar-API healthcheck - description: | - If this endpoint does not return 200, the API is experiencing a catastrophic outage. Should never happen. - responses: - '200': - description: Ok - content: - text/plain: - schema: - type: string - example: | - healthy - source code: https://github.com/TUM-Dev/navigatum/tree/8a0fb71819ac88c8af35683cfb46291f0d0c9b0a - '503': - description: Service Unavailable - tags: - - health /cdn/health: get: operationId: cdn-health diff --git a/server/.sqlx/query-12f3c3e7dbfbd4a7b5f48ddd580dddf031da92cd84bd0be70a7d5c7f9a405eaa.json b/server/.sqlx/query-12f3c3e7dbfbd4a7b5f48ddd580dddf031da92cd84bd0be70a7d5c7f9a405eaa.json new file mode 100644 index 000000000..01fc9bbac --- /dev/null +++ b/server/.sqlx/query-12f3c3e7dbfbd4a7b5f48ddd580dddf031da92cd84bd0be70a7d5c7f9a405eaa.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM calendar WHERE room_code = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "12f3c3e7dbfbd4a7b5f48ddd580dddf031da92cd84bd0be70a7d5c7f9a405eaa" +} diff --git a/server/.sqlx/query-190bd7b2e8c82bf248caf3daa7e835f6c069abf769cc9d2980c96b163654ddd5.json b/server/.sqlx/query-190bd7b2e8c82bf248caf3daa7e835f6c069abf769cc9d2980c96b163654ddd5.json deleted file mode 100644 index 0535d371f..000000000 --- a/server/.sqlx/query-190bd7b2e8c82bf248caf3daa7e835f6c069abf769cc9d2980c96b163654ddd5.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO calendar(key, dtstart, dtend, dtstamp, event_id, event_title, single_event_id, single_event_type_id, single_event_type_name, event_type_id, event_type_name, course_type_name, course_type, course_code, course_semester_hours, group_id, xgroup, status_id, status, comment, last_scrape)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Timestamp", - "Timestamp", - "Timestamp", - "Int4", - "Text", - "Int4", - "Text", - "Text", - "Text", - "Text", - "Text", - "Text", - "Text", - "Int4", - "Text", - "Text", - "Text", - "Text", - "Text", - "Timestamp" - ] - }, - "nullable": [] - }, - "hash": "190bd7b2e8c82bf248caf3daa7e835f6c069abf769cc9d2980c96b163654ddd5" -} diff --git a/server/.sqlx/query-324732134abda5929ac9b25d78b022731f939e1ed4268a7b675504e42a4ca004.json b/server/.sqlx/query-324732134abda5929ac9b25d78b022731f939e1ed4268a7b675504e42a4ca004.json deleted file mode 100644 index 69255892e..000000000 --- a/server/.sqlx/query-324732134abda5929ac9b25d78b022731f939e1ed4268a7b675504e42a4ca004.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO rooms(key,tumonline_org_id,tumonline_calendar_id,tumonline_room_id,last_scrape)\n VALUES ($1,$2,$3,$4,$5)\n ON CONFLICT (key) DO UPDATE SET\n tumonline_org_id=$2,\n tumonline_calendar_id=$3,\n tumonline_room_id=$4,\n last_scrape=$5\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int4", - "Int4", - "Int4", - "Timestamp" - ] - }, - "nullable": [] - }, - "hash": "324732134abda5929ac9b25d78b022731f939e1ed4268a7b675504e42a4ca004" -} diff --git a/server/.sqlx/query-334f5e3016bbe104ba8f360d61ef91a95272231c6aac78c0207fddc062e8dd13.json b/server/.sqlx/query-334f5e3016bbe104ba8f360d61ef91a95272231c6aac78c0207fddc062e8dd13.json deleted file mode 100644 index ed335486f..000000000 --- a/server/.sqlx/query-334f5e3016bbe104ba8f360d61ef91a95272231c6aac78c0207fddc062e8dd13.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM calendar WHERE dtstart > $1 AND dtend < $2 AND last_scrape < $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Timestamp", - "Timestamp", - "Timestamp" - ] - }, - "nullable": [] - }, - "hash": "334f5e3016bbe104ba8f360d61ef91a95272231c6aac78c0207fddc062e8dd13" -} diff --git a/server/.sqlx/query-4d4a2f19ff44defa3d05dbcf7ede3925b8a07f3de56607e58bf8b29ca9d9d424.json b/server/.sqlx/query-4d4a2f19ff44defa3d05dbcf7ede3925b8a07f3de56607e58bf8b29ca9d9d424.json new file mode 100644 index 000000000..3120e110c --- /dev/null +++ b/server/.sqlx/query-4d4a2f19ff44defa3d05dbcf7ede3925b8a07f3de56607e58bf8b29ca9d9d424.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE de SET last_calendar_scrape_at = $1 WHERE key=$2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Timestamptz", + "Text" + ] + }, + "nullable": [] + }, + "hash": "4d4a2f19ff44defa3d05dbcf7ede3925b8a07f3de56607e58bf8b29ca9d9d424" +} diff --git a/server/.sqlx/query-7ad82242b6dc392f71b63b4513da83339d69e3e4c4cbe67b44b94871754cdf13.json b/server/.sqlx/query-7ad82242b6dc392f71b63b4513da83339d69e3e4c4cbe67b44b94871754cdf13.json deleted file mode 100644 index e9d15fa1a..000000000 --- a/server/.sqlx/query-7ad82242b6dc392f71b63b4513da83339d69e3e4c4cbe67b44b94871754cdf13.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT *\n FROM calendar\n WHERE key = $1 AND dtstart >= $2 AND dtend <= $3\n ORDER BY dtstart", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "key", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "dtstart", - "type_info": "Timestamp" - }, - { - "ordinal": 2, - "name": "dtend", - "type_info": "Timestamp" - }, - { - "ordinal": 3, - "name": "dtstamp", - "type_info": "Timestamp" - }, - { - "ordinal": 4, - "name": "event_id", - "type_info": "Int4" - }, - { - "ordinal": 5, - "name": "event_title", - "type_info": "Text" - }, - { - "ordinal": 6, - "name": "single_event_id", - "type_info": "Int4" - }, - { - "ordinal": 7, - "name": "single_event_type_id", - "type_info": "Text" - }, - { - "ordinal": 8, - "name": "single_event_type_name", - "type_info": "Text" - }, - { - "ordinal": 9, - "name": "event_type_id", - "type_info": "Text" - }, - { - "ordinal": 10, - "name": "event_type_name", - "type_info": "Text" - }, - { - "ordinal": 11, - "name": "course_type_name", - "type_info": "Text" - }, - { - "ordinal": 12, - "name": "course_type", - "type_info": "Text" - }, - { - "ordinal": 13, - "name": "course_code", - "type_info": "Text" - }, - { - "ordinal": 14, - "name": "course_semester_hours", - "type_info": "Int4" - }, - { - "ordinal": 15, - "name": "group_id", - "type_info": "Text" - }, - { - "ordinal": 16, - "name": "xgroup", - "type_info": "Text" - }, - { - "ordinal": 17, - "name": "status_id", - "type_info": "Text" - }, - { - "ordinal": 18, - "name": "status", - "type_info": "Text" - }, - { - "ordinal": 19, - "name": "comment", - "type_info": "Text" - }, - { - "ordinal": 20, - "name": "last_scrape", - "type_info": "Timestamp" - } - ], - "parameters": { - "Left": [ - "Text", - "Timestamp", - "Timestamp" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - true, - true, - true, - true, - false, - false, - false, - false - ] - }, - "hash": "7ad82242b6dc392f71b63b4513da83339d69e3e4c4cbe67b44b94871754cdf13" -} diff --git a/server/.sqlx/query-874f7f834952dc9935690debfad3d161757d7ce6fcee59c23c23abbf45e803cb.json b/server/.sqlx/query-874f7f834952dc9935690debfad3d161757d7ce6fcee59c23c23abbf45e803cb.json index 37694e0dc..f1a625cd9 100644 --- a/server/.sqlx/query-874f7f834952dc9935690debfad3d161757d7ce6fcee59c23c23abbf45e803cb.json +++ b/server/.sqlx/query-874f7f834952dc9935690debfad3d161757d7ce6fcee59c23c23abbf45e803cb.json @@ -42,6 +42,11 @@ "ordinal": 7, "name": "data", "type_info": "Text" + }, + { + "ordinal": 8, + "name": "last_calendar_scrape_at", + "type_info": "Timestamptz" } ], "parameters": { @@ -57,7 +62,8 @@ false, false, false, - false + false, + true ] }, "hash": "874f7f834952dc9935690debfad3d161757d7ce6fcee59c23c23abbf45e803cb" diff --git a/server/.sqlx/query-a09ff7e07e367385a4c9b7cfbfbc75fda0eb5321958547cb8a16a6892af10f76.json b/server/.sqlx/query-a09ff7e07e367385a4c9b7cfbfbc75fda0eb5321958547cb8a16a6892af10f76.json new file mode 100644 index 000000000..3871a1c01 --- /dev/null +++ b/server/.sqlx/query-a09ff7e07e367385a4c9b7cfbfbc75fda0eb5321958547cb8a16a6892af10f76.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO calendar (id,room_code,start_at,end_at,stp_title_de,stp_title_en,stp_type,entry_type,detailed_entry_type)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n ON CONFLICT (id) DO UPDATE SET\n room_code = $2,\n start_at = $3,\n end_at = $4,\n stp_title_de = $5,\n stp_title_en = $6,\n stp_type = $7,\n entry_type = $8,\n detailed_entry_type = $9", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Varchar", + "Timestamptz", + "Timestamptz", + "Text", + "Text", + "Text", + { + "Custom": { + "name": "eventtype", + "kind": { + "Enum": [ + "lecture", + "exercise", + "exam", + "barred", + "other" + ] + } + } + }, + "Text" + ] + }, + "nullable": [] + }, + "hash": "a09ff7e07e367385a4c9b7cfbfbc75fda0eb5321958547cb8a16a6892af10f76" +} diff --git a/server/.sqlx/query-a87d33b55872d9135ab82391b36d1a495534817fb39a9e25f6720267c6025965.json b/server/.sqlx/query-a87d33b55872d9135ab82391b36d1a495534817fb39a9e25f6720267c6025965.json index 9357fbca4..89d01e799 100644 --- a/server/.sqlx/query-a87d33b55872d9135ab82391b36d1a495534817fb39a9e25f6720267c6025965.json +++ b/server/.sqlx/query-a87d33b55872d9135ab82391b36d1a495534817fb39a9e25f6720267c6025965.json @@ -42,6 +42,11 @@ "ordinal": 7, "name": "data", "type_info": "Text" + }, + { + "ordinal": 8, + "name": "last_calendar_scrape_at", + "type_info": "Timestamptz" } ], "parameters": { @@ -57,7 +62,8 @@ false, false, false, - false + false, + true ] }, "hash": "a87d33b55872d9135ab82391b36d1a495534817fb39a9e25f6720267c6025965" diff --git a/server/.sqlx/query-a975b979fb570d9cc18fb71425102c98ee5202b6d0dadd5be5c894d03b9e1354.json b/server/.sqlx/query-a975b979fb570d9cc18fb71425102c98ee5202b6d0dadd5be5c894d03b9e1354.json deleted file mode 100644 index a9e71f898..000000000 --- a/server/.sqlx/query-a975b979fb570d9cc18fb71425102c98ee5202b6d0dadd5be5c894d03b9e1354.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT * FROM rooms WHERE key = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "key", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "tumonline_org_id", - "type_info": "Int4" - }, - { - "ordinal": 2, - "name": "tumonline_calendar_id", - "type_info": "Int4" - }, - { - "ordinal": 3, - "name": "tumonline_room_id", - "type_info": "Int4" - }, - { - "ordinal": 4, - "name": "last_scrape", - "type_info": "Timestamp" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - false - ] - }, - "hash": "a975b979fb570d9cc18fb71425102c98ee5202b6d0dadd5be5c894d03b9e1354" -} diff --git a/server/.sqlx/query-d2978b1b6a3b39bedda8e40cc2e53f91b262d8c42e7a6886ce25a14975c87f70.json b/server/.sqlx/query-d2978b1b6a3b39bedda8e40cc2e53f91b262d8c42e7a6886ce25a14975c87f70.json new file mode 100644 index 000000000..3d1a0ec11 --- /dev/null +++ b/server/.sqlx/query-d2978b1b6a3b39bedda8e40cc2e53f91b262d8c42e7a6886ce25a14975c87f70.json @@ -0,0 +1,85 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id,room_code,start_at,end_at,stp_title_de,stp_title_en,stp_type,entry_type AS \"entry_type!:crate::calendar::models::EventType\",detailed_entry_type\n FROM calendar\n WHERE room_code = $1 AND start_at >= $2 AND end_at <= $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "room_code", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "start_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 3, + "name": "end_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "stp_title_de", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "stp_title_en", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "stp_type", + "type_info": "Text" + }, + { + "ordinal": 7, + "name": "entry_type!:crate::calendar::models::EventType", + "type_info": { + "Custom": { + "name": "eventtype", + "kind": { + "Enum": [ + "lecture", + "exercise", + "exam", + "barred", + "other" + ] + } + } + } + }, + { + "ordinal": 8, + "name": "detailed_entry_type", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Timestamptz", + "Timestamptz" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "d2978b1b6a3b39bedda8e40cc2e53f91b262d8c42e7a6886ce25a14975c87f70" +} diff --git a/server/.sqlx/query-f142eb3b0a7027d50af7f60df5288f82b10327b0ecca14f0954057ffc514629a.json b/server/.sqlx/query-f142eb3b0a7027d50af7f60df5288f82b10327b0ecca14f0954057ffc514629a.json deleted file mode 100644 index eddb287d7..000000000 --- a/server/.sqlx/query-f142eb3b0a7027d50af7f60df5288f82b10327b0ecca14f0954057ffc514629a.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM rooms WHERE last_scrape < $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Timestamp" - ] - }, - "nullable": [] - }, - "hash": "f142eb3b0a7027d50af7f60df5288f82b10327b0ecca14f0954057ffc514629a" -} diff --git a/server/Cargo.lock b/server/Cargo.lock index aaf09faa3..4ed2cb9ea 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -64,8 +64,8 @@ dependencies = [ "actix-service", "actix-utils", "ahash", - "base64", - "bitflags 2.4.1", + "base64 0.21.7", + "bitflags 2.4.2", "bytes", "bytestring", "derive_more", @@ -97,14 +97,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] name = "actix-router" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66ff4d247d2b160861fa2866457e85706833527840e4133f8f49aa423a38799" +checksum = "d22475596539443685426b6bdadb926ad0ecaefdfc5fb05e5e3441f15463c511" dependencies = [ "bytestring", "http", @@ -210,7 +210,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -243,12 +243,12 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" dependencies = [ "cfg-if", - "getrandom 0.2.11", + "getrandom 0.2.12", "once_cell", "version_check", "zerocopy", @@ -312,9 +312,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc2d0cfb2a7388d34f590e76686704c494ed7aaceed62ee1ba35cbf363abc2a5" +checksum = "a116f46a969224200a0a97f29cfd4c50e7534e4b4826bd23ea2c3c533039c82c" dependencies = [ "flate2", "futures-core", @@ -325,13 +325,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.75" +version = "0.1.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdf6721fb0140e4f897002dd086c06f6c27775df19cfe1fccb21181a48fd2c98" +checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -374,6 +374,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.7" @@ -406,9 +412,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" dependencies = [ "serde", ] @@ -521,8 +527,10 @@ checksum = "41daef31d7a747c5c847246f36de49ced6f7403b4cdabc807a97b5cc184cda7a" dependencies = [ "android-tzdata", "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", "windows-targets 0.52.0", ] @@ -600,9 +608,9 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] @@ -633,45 +641,37 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ - "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.16" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2fe95351b870527a5d09bf563ed3c97c0cffb87cf1c78a591bf48bb218d9aa" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset", ] [[package]] name = "crossbeam-queue" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9bcf5bdbfdd6030fb4a1c497b5d5fc5921aa2f60d359a17e249c0e6df3de153" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.17" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d96137f14f244c37f989d9fff8f95e6c18b918e71f36638f8c49112e4c78f" -dependencies = [ - "cfg-if", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crunchy" @@ -787,9 +787,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", "serde", @@ -864,9 +864,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "erased-serde" -version = "0.3.31" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c138974f9d5e7fe373eb04df7cae98833802ae4b11c24ac7039a21d5af4b26c" +checksum = "55d05712b2d8d88102bc9868020c9e5c7a1f5527c452b9b97450a1d006140ba7" dependencies = [ "serde", ] @@ -931,9 +931,9 @@ checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "fdeflate" -version = "0.3.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d6dafc854908ff5da46ff3f8f473c6984119a2876a383a860246dd7841a868" +checksum = "4f9bfee30e4dedf0ab8b422f03af778d9612b63f502710fc500a334ebe2de645" dependencies = [ "simd-adler32", ] @@ -971,21 +971,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1077,7 +1062,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -1139,9 +1124,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if", "js-sys", @@ -1248,9 +1233,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" [[package]] name = "hex" @@ -1366,24 +1351,11 @@ dependencies = [ "webpki-roots", ] -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", -] - [[package]] name = "iana-time-zone" -version = "0.1.58" +version = "0.1.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1551,18 +1523,18 @@ dependencies = [ [[package]] name = "jpeg-decoder" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc0000e42512c92e31c2252315bda326620a4e034105e900c98ec492fa077b3e" +checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" dependencies = [ "rayon", ] [[package]] name = "js-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" dependencies = [ "wasm-bindgen", ] @@ -1573,7 +1545,7 @@ version = "8.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" dependencies = [ - "base64", + "base64 0.21.7", "ring 0.16.20", "serde", "serde_json", @@ -1585,7 +1557,7 @@ version = "9.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c7ea04a7c5c055c175f189b6dc6ba036fd62306b58c66c9f6389036c503a3f4" dependencies = [ - "base64", + "base64 0.21.7", "js-sys", "pem", "ring 0.17.7", @@ -1617,9 +1589,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" [[package]] name = "libc" -version = "0.2.151" +version = "0.2.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" [[package]] name = "libm" @@ -1629,9 +1601,9 @@ checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] name = "libnghttp2-sys" -version = "0.1.8+1.55.1" +version = "0.1.9+1.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fae956c192dadcdb5dace96db71fa0b827333cce7c7b38dc71446f024d8a340" +checksum = "b57e858af2798e167e709b9d969325b6d8e9d50232fcbc494d7d54f976854a64" dependencies = [ "cc", "libc", @@ -1660,9 +1632,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.12" +version = "1.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +checksum = "295c17e837573c8c821dbaeb3cceb3d745ad082f7572191409e69cbc1b3fd050" dependencies = [ "cc", "libc", @@ -1672,9 +1644,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "local-channel" @@ -1733,7 +1705,7 @@ dependencies = [ "proc-macro2", "quote", "regex-syntax 0.6.29", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -1815,18 +1787,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" - -[[package]] -name = "memoffset" -version = "0.9.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" -dependencies = [ - "autocfg", -] +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "mime" @@ -1834,15 +1797,6 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "minidom" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f45614075738ce1b77a1768912a60c0227525971b03e09122a05b8a34a2a6278" -dependencies = [ - "rxml", -] - [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1886,49 +1840,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "native-tls" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - -[[package]] -name = "navigatum-calendar" -version = "1.0.0" -dependencies = [ - "actix-cors", - "actix-web", - "actix-web-prom", - "chrono", - "futures", - "lazy_static", - "log", - "minidom", - "pretty_assertions", - "prometheus", - "rand 0.8.5", - "regex", - "reqwest", - "rustls 0.22.2", - "serde", - "serde_json", - "sqlx", - "structured-logger", - "tokio", -] - [[package]] name = "navigatum-feedback" version = "1.0.0" @@ -1937,7 +1848,7 @@ dependencies = [ "actix-governor", "actix-web", "actix-web-prom", - "base64", + "base64 0.21.7", "chrono", "image", "jsonwebtoken 9.2.0", @@ -1962,6 +1873,7 @@ dependencies = [ "actix-web", "actix-web-prom", "cached", + "chrono", "futures", "image", "imageproc", @@ -1969,6 +1881,7 @@ dependencies = [ "log", "logos", "meilisearch-sdk", + "oauth2", "pretty_assertions", "regex", "reqwest", @@ -1989,7 +1902,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cfg-if", "libc", ] @@ -2120,11 +2033,31 @@ dependencies = [ "libc", ] +[[package]] +name = "oauth2" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c38841cdd844847e3e7c8d29cef9dcfed8877f8f56f9071f77843ecf3baf937f" +dependencies = [ + "base64 0.13.1", + "chrono", + "getrandom 0.2.12", + "http", + "rand 0.8.5", + "reqwest", + "serde", + "serde_json", + "serde_path_to_error", + "sha2", + "thiserror", + "url", +] + [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] @@ -2137,7 +2070,7 @@ checksum = "5dad7c6124666d1e64c6cd7dc2a9411ea70d3aee4bdf1f099720f70bd11bc3f1" dependencies = [ "arc-swap", "async-trait", - "base64", + "base64 0.21.7", "bytes", "cfg-if", "chrono", @@ -2169,32 +2102,6 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" -[[package]] -name = "openssl" -version = "0.10.61" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b8419dc8cc6d866deb801274bba2e6f8f6108c1bb7fcc10ee5ab864931dbb45" -dependencies = [ - "bitflags 2.4.1", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.42", -] - [[package]] name = "openssl-probe" version = "0.1.5" @@ -2203,9 +2110,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.97" +version = "0.9.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3eaad34cdd97d81de97964fc7f29e2d104f483840d906ef56daa1912338460b" +checksum = "22e1bf214306098e4832460f797824c05d25aacdf896f64a985fb0fd992454ae" dependencies = [ "cc", "libc", @@ -2263,7 +2170,7 @@ version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b8fcc794035347fb64beda2d3b462595dd2753e3f268d89c5aae77e8cf2c310" dependencies = [ - "base64", + "base64 0.21.7", "serde", ] @@ -2299,7 +2206,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -2337,15 +2244,15 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" +checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" [[package]] name = "png" -version = "0.17.10" +version = "0.17.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd75bf2d8dd3702b9707cdbc56a5b9ef42cec752eb8b3bafc01234558442aa64" +checksum = "1f6c3c3e617595665b8ea2ff95a86066be38fb121ff920a9c0eb282abcd1da5a" dependencies = [ "bitflags 1.3.2", "crc32fast", @@ -2394,9 +2301,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.71" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75cb1540fadbd5b8fbccc4dddad2734eba435053f725621c070711a14bb5f4b8" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] @@ -2410,20 +2317,11 @@ dependencies = [ "cfg-if", "fnv", "lazy_static", - "libc", "memchr", "parking_lot", - "protobuf", - "reqwest", "thiserror", ] -[[package]] -name = "protobuf" -version = "2.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" - [[package]] name = "qoi" version = "0.4.1" @@ -2451,9 +2349,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -2517,7 +2415,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.11", + "getrandom 0.2.12", ] [[package]] @@ -2555,9 +2453,9 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" [[package]] name = "rayon" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +checksum = "fa7237101a77a10773db45d62004a272517633fbcc3df19d96455ede1122e051" dependencies = [ "either", "rayon-core", @@ -2565,9 +2463,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -2624,7 +2522,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" dependencies = [ "async-compression", - "base64", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", @@ -2633,21 +2531,21 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-tls", + "hyper-rustls", "ipnet", "js-sys", "log", "mime", - "native-tls", "once_cell", "percent-encoding", "pin-project-lite", + "rustls 0.21.10", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", "system-configuration", "tokio", - "tokio-native-tls", "tokio-rustls", "tokio-util", "tower-service", @@ -2655,6 +2553,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", + "webpki-roots", "winreg", ] @@ -2680,7 +2579,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" dependencies = [ "cc", - "getrandom 0.2.11", + "getrandom 0.2.12", "libc", "spin 0.9.8", "untrusted 0.9.0", @@ -2724,11 +2623,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.28" +version = "0.38.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" +checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "errno", "libc", "linux-raw-sys", @@ -2779,14 +2678,14 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64", + "base64 0.21.7", ] [[package]] name = "rustls-pki-types" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7673e0aa20ee4937c6aacfc12bb8341cfbf054cdd21df6bec5fd0629fe9339b" +checksum = "9e9d979b3ce68192e42760c7810125eb6cf2ea10efae545a156063e61f314e2a" [[package]] name = "rustls-webpki" @@ -2819,23 +2718,6 @@ dependencies = [ "owned_ttf_parser", ] -[[package]] -name = "rxml" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a98f186c7a2f3abbffb802984b7f1dfd65dac8be1aafdaabbca4137f53f0dff7" -dependencies = [ - "bytes", - "rxml_validation", - "smartstring", -] - -[[package]] -name = "rxml_validation" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22a197350ece202f19a166d1ad6d9d6de145e1d2a8ef47db299abe164dbd7530" - [[package]] name = "ryu" version = "1.0.16" @@ -2853,11 +2735,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2910,28 +2792,28 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" +checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" [[package]] name = "serde" -version = "1.0.193" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.193" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -2945,9 +2827,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" dependencies = [ "itoa", "ryu", @@ -2956,9 +2838,9 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4beec8bce849d58d06238cb50db2e1c417cfeafa4c63f692b15c82b7c80f8335" +checksum = "ebd154a240de39fdebcf5775d2675c204d7c13cf39a4c697be6493c8e734337c" dependencies = [ "itoa", "serde", @@ -2978,9 +2860,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.29" +version = "0.9.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15e0ef66bf939a7c890a0bf6d5a733c70202225f9888a89ed5c62298b019129" +checksum = "b1bf28c79a99f70ee1f1d83d10c875d2e70618417fda01ad1785e027579d9d38" dependencies = [ "indexmap", "itoa", @@ -3083,20 +2965,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.2" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" - -[[package]] -name = "smartstring" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" -dependencies = [ - "autocfg", - "static_assertions", - "version_check", -] +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "snafu" @@ -3281,8 +3152,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e37195395df71fd068f6e2082247891bc11e3289624bbc776a0cdfa1ca7f1ea4" dependencies = [ "atoi", - "base64", - "bitflags 2.4.1", + "base64 0.21.7", + "bitflags 2.4.2", "byteorder", "bytes", "chrono", @@ -3324,8 +3195,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6ac0ac3b7ccd10cc96c7ab29791a7dd236bd94021f31eec7ba3d46a74aa1c24" dependencies = [ "atoi", - "base64", - "bitflags 2.4.1", + "base64 0.21.7", + "bitflags 2.4.2", "byteorder", "chrono", "crc", @@ -3381,12 +3252,6 @@ dependencies = [ "urlencoding", ] -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - [[package]] name = "stringprep" version = "0.1.4" @@ -3425,15 +3290,15 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "sval" -version = "2.10.2" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15df12a8db7c216a04b4b438f90d50d5335cd38f161b56389c9f5c9d96d0873" +checksum = "1604e9ab506f4805bc62d2868c6d20f23fa6ced4c7cfe695a1d20589ba5c63d0" [[package]] name = "sval_buffer" -version = "2.10.2" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57e80556bc8acea0446e574ce542ad6114a76a0237f28a842bc01ca3ea98f479" +checksum = "2831b6451148d344f612016d4277348f7721b78a0869a145fd34ef8b06b3fa2e" dependencies = [ "sval", "sval_ref", @@ -3441,18 +3306,18 @@ dependencies = [ [[package]] name = "sval_dynamic" -version = "2.10.2" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d93d2259edb1d7b4316179f0a98c62e3ffc726f47ab200e07cfe382771f57b8" +checksum = "238ac5832a23099a413ffd22e66f7e6248b9af4581b64c758ca591074be059fc" dependencies = [ "sval", ] [[package]] name = "sval_fmt" -version = "2.10.2" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "532f7f882226f7a5a4656f5151224aaebf8217e0d539cb1595b831bace921343" +checksum = "c8474862431bac5ac7aee8a12597798e944df33f489c340e17e886767bda0c4e" dependencies = [ "itoa", "ryu", @@ -3461,34 +3326,44 @@ dependencies = [ [[package]] name = "sval_json" -version = "2.10.2" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76e03bd8aa0ae6ee018f7ae95c9714577687a4415bd1a5f19b26e34695f7e072" +checksum = "d8f348030cc3d2a11eb534145600601f080cf16bf9ec0783efecd2883f14c21e" dependencies = [ "itoa", "ryu", "sval", ] +[[package]] +name = "sval_nested" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6659c3f6be1e5e99dc7c518877f48a8a39088ace2504b046db789bd78ce5969d" +dependencies = [ + "sval", + "sval_buffer", + "sval_ref", +] + [[package]] name = "sval_ref" -version = "2.10.2" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75ed054f2fb8c2a0ab5d36c1ec57b412919700099fc5e32ad8e7a38b23e1a9e1" +checksum = "829ad319bd82d0da77be6f3d547623686c453502f8eebdeb466cfa987972bd28" dependencies = [ "sval", ] [[package]] name = "sval_serde" -version = "2.10.2" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ff191c4ff05b67e3844c161021427646cde5d6624597958be158357d9200586" +checksum = "1a9da6c3efaedf8b8c0861ec5343e8e8c51d838f326478623328bd8728b79bca" dependencies = [ "serde", "sval", - "sval_buffer", - "sval_fmt", + "sval_nested", ] [[package]] @@ -3504,9 +3379,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.42" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b7d0a2c048d661a1a59fcd7355baa232f7ed34e0ee4df2eef3c1c1c0d3852d8" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -3549,29 +3424,29 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.51" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.51" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] name = "tiff" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d172b0f4d3fba17ba89811858b9d3d97f928aece846475bbda076ca46736211" +checksum = "ba1310fcea54c6a9a4fd1aad794ecc02c31682f6bfbecdf460bf19533eed1e3e" dependencies = [ "flate2", "jpeg-decoder", @@ -3649,17 +3524,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", -] - -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", + "syn 2.0.48", ] [[package]] @@ -3720,7 +3585,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "bytes", "futures-core", "futures-util", @@ -3765,7 +3630,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -3807,9 +3672,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-bidi" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" @@ -3891,18 +3756,18 @@ checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" [[package]] name = "uuid" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" dependencies = [ - "getrandom 0.2.11", + "getrandom 0.2.12", ] [[package]] name = "value-bag" -version = "1.4.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a72e1902dde2bd6441347de2b70b7f5d59bf157c6c62f0c44572607a1d55bbe" +checksum = "7cdbaf5e132e593e9fc1de6a15bbec912395b11fb9719e061cf64f804524c503" dependencies = [ "value-bag-serde1", "value-bag-sval2", @@ -3910,9 +3775,9 @@ dependencies = [ [[package]] name = "value-bag-serde1" -version = "1.4.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ba39dc791ecb35baad371a3fc04c6eab688c04937d2e0ac6c22b612c0357bf" +checksum = "92cad98b1b18d06b6f38b3cd04347a9d7a3a0111441a061f71377fb6740437e4" dependencies = [ "erased-serde", "serde", @@ -3921,9 +3786,9 @@ dependencies = [ [[package]] name = "value-bag-sval2" -version = "1.4.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3e06c10810a57bbf45778d023d432a50a1daa7d185991ae06bcfb6c654d0945" +checksum = "3dc7271d6b3bf58dd2e610a601c0e159f271ffdb7fbb21517c40b52138d64f8e" dependencies = [ "sval", "sval_buffer", @@ -3975,9 +3840,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -3985,24 +3850,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.39" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" +checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" dependencies = [ "cfg-if", "js-sys", @@ -4012,9 +3877,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4022,28 +3887,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" +checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" [[package]] name = "web-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" +checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" dependencies = [ "js-sys", "wasm-bindgen", @@ -4066,9 +3931,9 @@ checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" [[package]] name = "weezl" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9193164d4de03a926d909d3bc7c30543cecb35400c02114792c2cae20d5e2dbb" +checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082" [[package]] name = "whoami" @@ -4078,9 +3943,9 @@ checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" [[package]] name = "wide" -version = "0.7.13" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c68938b57b33da363195412cfc5fc37c9ed49aa9cfe2156fde64b8d2c9498242" +checksum = "b31891d644eba1789fb6715f27fbc322e4bdf2ecdc412ede1993246159271613" dependencies = [ "bytemuck", "safe_arch", @@ -4110,11 +3975,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.51.1" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.48.5", + "windows-targets 0.52.0", ] [[package]] @@ -4292,7 +4157,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] diff --git a/server/Cargo.toml b/server/Cargo.toml index f9cc678a4..6ed8b7a77 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["feedback", "calendar", "main-api"] +members = ["feedback", "main-api"] resolver = "2" [profile.release] diff --git a/server/Dockerfile b/server/Dockerfile index 3afdad71b..89cc12dbf 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -16,13 +16,10 @@ ENV RUSTFLAGS="-C target-feature=-crt-static" COPY ./Cargo.* ./ COPY main-api/Cargo.* ./main-api/ COPY feedback/Cargo.* ./feedback/ -COPY calendar/Cargo.* ./calendar/ -RUN mkdir ./main-api/src/ ./main-api/src/setup/ ./feedback/src/ ./calendar/src/ \ - && echo "fn main() { println!(\"Hello, world!\");}" > ./main-api/src/setup/mod.rs \ - && echo "fn main() { println!(\"Hello, world!\");}" > ./main-api/src/main.rs\ - && echo "fn main() { println!(\"Hello, world!\");}" > ./feedback/src/main.rs\ - && echo "fn main() { println!(\"Hello, world!\");}" > ./calendar/src/main.rs\ - && echo "fn main() { println!(\"Hello, world!\");}" > ./calendar/src/scraper.rs +RUN mkdir ./main-api/src/ ./main-api/src/setup/ ./feedback/src/ +RUN echo "fn main() { println!(\"Hello, world!\");}" > ./main-api/src/setup/mod.rs +RUN echo "fn main() { println!(\"Hello, world!\");}" > ./main-api/src/main.rs +RUN echo "fn main() { println!(\"Hello, world!\");}" > ./feedback/src/main.rs RUN cargo build --release --workspace \ && rm -fr target/release/deps/navigatum* @@ -31,7 +28,6 @@ RUN cargo build --release --workspace \ COPY .sqlx .sqlx COPY feedback/src ./feedback/src -COPY calendar/src ./calendar/src COPY main-api/src ./main-api/src COPY main-api/migrations ./main-api/migrations RUN cargo build --release --workspace @@ -66,7 +62,5 @@ ENTRYPOINT ["tini", "--"] # These are coomands that can be chosen to run # - a respective server or # - an admin task -# CMD /bin/navigatum-calendar-scraper -# CMD /bin/navigatum-calendar # CMD /bin/navigatum-feedback CMD /bin/navigatum-main-api diff --git a/server/README.md b/server/README.md index 4ab6bda9c..756e7389a 100644 --- a/server/README.md +++ b/server/README.md @@ -3,10 +3,6 @@ Our server is architected in different microservices, each of which is responsible for a specific task. - [main-api](/server/main-api): The main API server, which serves the API endpoints -- [calendar](/server/calendar): The calendar microservice, which scrapes and serves calendar data - This is separated from the server because: - - it has virtually no shared dependencies (natural fault line) - - we need a persistent database (postgres) for this microservice, but not for the main server (sqlite) - [feedback](/server/feedback): The feedback microservice, which allows users to submit feedback This is separated from the server because: - it has virtually no shared dependencies (natural faultline) diff --git a/server/calendar/.dockerignore b/server/calendar/.dockerignore deleted file mode 100644 index 495e95009..000000000 --- a/server/calendar/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -target -data -.hypothesis diff --git a/server/calendar/Cargo.toml b/server/calendar/Cargo.toml deleted file mode 100644 index eca942a36..000000000 --- a/server/calendar/Cargo.toml +++ /dev/null @@ -1,49 +0,0 @@ -[package] -name = "navigatum-calendar" -version = "1.0.0" -authors = ["Markus A ", "Frank Elsinga "] -edition = "2021" -description = "Navigating around TUM with excellence – An API and website to search for rooms, buildings and other places" -repository = "https://github.com/TUM-Dev/navigatum" -readme = "README.md" -license = "GPL-3.0" -keywords = ["website", "navigation", "api-rest", "tum"] - -[[bin]] -name = "navigatum-calendar" -path = "src/main.rs" - -[[bin]] -name = "navigatum-calendar-scraper" -path = "src/scraper.rs" - -[dependencies] -# shared -log.workspace = true -structured-logger.workspace = true -tokio.workspace = true -actix-web.workspace = true -actix-cors.workspace = true -actix-web-prom.workspace = true -serde.workspace = true -serde_json.workspace = true - -chrono = { version = "0.4.32", default-features = false, features = ["serde"] } - -# database -sqlx = { version = "0.7.3", features = ["postgres", "runtime-tokio-rustls", "migrate", "macros", "chrono"] } - -# metrics -lazy_static = "1.4.0" -prometheus = { version = "0.13.3", features = ["default", "push"] } - -# scraper -rand = "0.8.5" -futures = "0.3.30" -rustls = "0.22.2" -reqwest = { version= "0.11.23", default-features = false, features = ["tokio-rustls", "json", "gzip"] } -minidom = "0.15.2" -regex = "1.10.3" - -[dev-dependencies] -pretty_assertions.workspace = true diff --git a/server/calendar/README.md b/server/calendar/README.md deleted file mode 100644 index 446dd1063..000000000 --- a/server/calendar/README.md +++ /dev/null @@ -1,75 +0,0 @@ -# Calendar - -This folder contains the calendar-microservice for NavigaTUM. - -## Getting started - -### Prerequisites - -For getting started, there are some system dependencys which you will need. -Please follow the [system dependencys docs](resources/documentation/Dependencys.md) before trying to run this part of our project. - -### How to Set up the Sqlite Database - -#### Setting up the database - -To set up the database, you will need to run a `postgres` instance. We recommend this configuration: - -```bash -docker run -it --rm -e POSTGRES_PASSWORD=password -p 5432:5432 postgres:latest -``` - -Any migrations this database needs are applied on first run of the server. -In order to change the connection parameters that the calendar server uses you can use the environment variables: - -- `DATABASE_URL` (default `postgres`) - -### Starting the server - -Run `cargo run` to start the server. -The server should now be available on `localhost:8081`. - -Note that `cargo run --release` is used to start the server for an optimised production build (use this if you want to profile performance, it makes quite a difference). - -### API-Changes - -#### Editing - -If you have made changes to the API, you need to update the API documentation. - -There are two editors for the API documentation (both are imperfect): - -- [Swagger Editor](https://editor.swagger.io/?url=https://raw.githubusercontent.com/TUM-Dev/navigatum/main/openapi.yaml) -- [stoplight](stoplight.io) - -#### Testing - -Of course documentation is one part of the process. If the changes are substantial, you should also run an API-Fuzz-Test: -To make sure that this specification is up-to-date and without holes, we run [schemathesis](https://github.com/schemathesis/schemathesis) using the following command on API Server: - -```bash -python -m venv venv -source venv/bin/activate -pip install schemathesis -st run --workers=auto --base-url=http://localhost:8060 --checks=all ../openapi.yaml -``` - -Some fuzzing-goals may not be available for you locally, as they require prefix-routing (f.ex.`/cdn` to the CDN) and some fuzzing-goals are automatically tested in our CI. -You can exchange `--base-url=http://localhost:8081` to `--base-url=https://nav.tum.sexy` for the full public API, or restrict your scope using a option like `--endpoint=/api/calendar/`. - -## License - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - ---- diff --git a/server/calendar/src/calendar.rs b/server/calendar/src/calendar.rs deleted file mode 100644 index 11b7da7c0..000000000 --- a/server/calendar/src/calendar.rs +++ /dev/null @@ -1,158 +0,0 @@ -use crate::models::Room; -use crate::models::XMLEvent; -use actix_web::{get, web, HttpResponse}; -use chrono::NaiveDateTime; -use log::error; -use serde::{Deserialize, Serialize}; -use sqlx::PgPool; -use std::fmt::Debug; - -#[derive(Deserialize, Debug)] -pub struct QueryArguments { - start: NaiveDateTime, // eg. 2022-01-01T00:00:00 - end: NaiveDateTime, // eg. 2022-01-07T00:00:00 -} - -async fn get_room_information( - requested_key: &str, - conn: &PgPool, -) -> Result, sqlx::Error> { - let room = sqlx::query_as!(Room, "SELECT * FROM rooms WHERE key = $1", requested_key) - .fetch_optional(conn) - .await?; - match room { - Some(r) => { - let calendar_url = format!( - "https://campus.tum.de/tumonline/wbKalender.wbRessource?pResNr={id}", - id = r.tumonline_calendar_id - ); - Ok(Some((calendar_url, r.last_scrape))) - } - None => Ok(None), - } -} - -async fn get_entries( - requested_key: &str, - args: &QueryArguments, - conn: &PgPool, -) -> Result, sqlx::Error> { - sqlx::query_as!( - XMLEvent, - r#"SELECT * - FROM calendar - WHERE key = $1 AND dtstart >= $2 AND dtend <= $3 - ORDER BY dtstart"#, - requested_key, - args.start, - args.end - ) - .fetch_all(conn) - .await -} - -#[get("/api/calendar/{id}")] -pub async fn calendar_handler( - params: web::Path, - web::Query(args): web::Query, - data: web::Data, -) -> HttpResponse { - let id = params.into_inner(); - let results = get_entries(&id, &args, &data.db).await; - let room_information = get_room_information(&id, &data.db).await; - match (results, room_information) { - (Ok(results), Ok(Some((calendar_url, last_room_sync)))) => { - let last_calendar_sync = results.iter().map(|e| e.last_scrape).min(); - let events = results.into_iter().map(Event::from).collect(); - HttpResponse::Ok().json(Events { - events, - last_sync: last_calendar_sync.unwrap_or(last_room_sync), - calendar_url, - }) - } - (_, Ok(None)) => HttpResponse::NotFound() - .content_type("text/plain") - .body("Room not found"), - (Err(e), _) => { - error!("Error loading calendar entries: {e:?}"); - HttpResponse::InternalServerError() - .content_type("text/plain") - .body("Error loading calendar") - } - (_, Err(e)) => { - error!("Error loading calendar_url: {e:?}"); - HttpResponse::InternalServerError() - .content_type("text/plain") - .body("Error loading calendar") - } - } -} - -#[derive(Serialize, Debug)] -struct Events { - events: Vec, - last_sync: NaiveDateTime, - calendar_url: String, -} - -#[derive(Serialize, Debug)] -struct Event { - id: i32, - title: String, - start: NaiveDateTime, - end: NaiveDateTime, - entry_type: EventType, - detailed_entry_type: String, -} - -#[derive(Serialize, Debug)] -#[serde(rename_all = "lowercase")] -enum EventType { - Lecture, - Exercise, - Exam, - Barred, - Other, -} -impl EventType { - fn from(xml_event: &XMLEvent) -> (Self, String) { - // only used for the lecture type - let course_type_name = xml_event - .course_type_name - .clone() - .unwrap_or_else(|| "Course type is unknown".to_string()); - match xml_event.single_event_type_id.as_str() { - "SPERRE" => return (Self::Barred, String::new()), - "PT" => return (Self::Exam, String::new()), - "P" => return (Self::Lecture, course_type_name), // Prüfung (geplant) is sometimes used for lectures - _ => {} - } - match xml_event.event_type_id.as_str() { - "LV" => (Self::Lecture, course_type_name), - "PT" => (Self::Exam, String::new()), - "EX" => (Self::Exercise, String::new()), - _ => match &xml_event.event_type_name { - Some(event_type_name) => ( - Self::Other, - format!("{}: {}", xml_event.single_event_type_name, event_type_name), - ), - None => (Self::Other, xml_event.single_event_type_name.clone()), - }, - } - } -} - -impl From for Event { - fn from(xml_event: XMLEvent) -> Self { - let (entry_type, detailed_entry_type) = EventType::from(&xml_event); - let title = xml_event.event_title; - Self { - id: xml_event.single_event_id, - title, - start: xml_event.dtstart, - end: xml_event.dtend, - entry_type, - detailed_entry_type, - } - } -} diff --git a/server/calendar/src/main.rs b/server/calendar/src/main.rs deleted file mode 100644 index 7c570492c..000000000 --- a/server/calendar/src/main.rs +++ /dev/null @@ -1,85 +0,0 @@ -mod calendar; -mod models; -mod utils; - -use actix_cors::Cors; -use actix_web::{get, middleware, web, App, HttpResponse, HttpServer}; -use actix_web_prom::PrometheusMetricsBuilder; -use log::error; -use sqlx::postgres::PgPoolOptions; -use sqlx::{Executor, PgPool}; -use std::collections::HashMap; -use std::error::Error; -use structured_logger::async_json::new_writer; -use structured_logger::Builder; - -#[derive(Clone, Debug)] -pub struct AppData { - db: PgPool, -} - -const MAX_JSON_PAYLOAD: usize = 1024 * 1024; // 1 MB - -#[get("/api/calendar/status")] -async fn health_status_handler(data: web::Data) -> HttpResponse { - let github_link = match std::env::var("GIT_COMMIT_SHA") { - Ok(hash) => format!("https://github.com/TUM-Dev/navigatum/tree/{hash}"), - Err(_) => "unknown commit hash, probably running in development".to_string(), - }; - return match data.db.execute("SELECT 1").await { - Ok(_) => HttpResponse::Ok() - .content_type("text/plain") - .body(format!("healthy\nsource_code: {github_link}")), - Err(e) => { - error!("database error: {e:?}",); - HttpResponse::InternalServerError() - .content_type("text/plain") - .body(format!("unhealthy\nsource_code: {github_link}")) - } - }; -} - -#[tokio::main] -async fn main() -> Result<(), Box> { - Builder::with_level("info") - .with_target_writer("*", new_writer(tokio::io::stdout())) - .init(); - let uri = utils::connection_string(); - let pool = PgPoolOptions::new() - .max_connections(20) - .connect(&uri) - .await?; - - // metrics - let labels = HashMap::from([( - "revision".to_string(), - std::env::var("GIT_COMMIT_SHA").unwrap_or_else(|_| "development".to_string()), - )]); - let prometheus = PrometheusMetricsBuilder::new("navigatum_calendar") - .endpoint("/metrics") - .const_labels(labels) - .build() - .unwrap(); - - HttpServer::new(move || { - let cors = Cors::default() - .allow_any_origin() - .allow_any_header() - .allowed_methods(vec!["GET"]) - .max_age(3600); - - App::new() - .wrap(prometheus.clone()) - .wrap(cors) - .wrap(middleware::Logger::default().exclude("/api/calendar/status")) - .wrap(middleware::Compress::default()) - .app_data(web::JsonConfig::default().limit(MAX_JSON_PAYLOAD)) - .app_data(web::Data::new(AppData { db: pool.clone() })) - .service(health_status_handler) - .service(calendar::calendar_handler) - }) - .bind(std::env::var("BIND_ADDRESS").unwrap_or_else(|_| "0.0.0.0:3005".to_string()))? - .run() - .await?; - Ok(()) -} diff --git a/server/calendar/src/models.rs b/server/calendar/src/models.rs deleted file mode 100644 index 96bf997ba..000000000 --- a/server/calendar/src/models.rs +++ /dev/null @@ -1,35 +0,0 @@ -use chrono::NaiveDateTime; - -#[derive(Clone, Debug)] -pub struct XMLEvent { - pub key: String, - pub dtstart: NaiveDateTime, - pub dtend: NaiveDateTime, - pub dtstamp: NaiveDateTime, - pub event_id: i32, - pub event_title: String, - pub single_event_id: i32, - pub single_event_type_id: String, - pub single_event_type_name: String, - pub event_type_id: String, - pub event_type_name: Option, - pub course_type_name: Option, - pub course_type: Option, - pub course_code: Option, - pub course_semester_hours: Option, - pub group_id: Option, - pub xgroup: Option, - pub status_id: String, - pub status: String, - pub comment: String, - pub last_scrape: NaiveDateTime, -} - -#[derive(Clone, Debug)] -pub struct Room { - pub key: String, - pub tumonline_org_id: i32, - pub tumonline_calendar_id: i32, - pub tumonline_room_id: i32, - pub last_scrape: NaiveDateTime, -} diff --git a/server/calendar/src/scrape_task/main_api_connector.rs b/server/calendar/src/scrape_task/main_api_connector.rs deleted file mode 100644 index da28edc2e..000000000 --- a/server/calendar/src/scrape_task/main_api_connector.rs +++ /dev/null @@ -1,104 +0,0 @@ -use chrono::{NaiveDateTime, Utc}; -use log::{error, info}; -use regex::Regex; -use serde::Deserialize; -use sqlx::PgPool; - -fn api_url_from_env() -> Option { - let main_api_addr = std::env::var("CDN_SVC_SERVICE_HOST").ok()?; - let main_api_port = std::env::var("CDN_SVC_SERVICE_PORT_HTTP").ok()?; - - Some(format!( - "http://{main_api_addr}:{main_api_port}/cdn/api_data.json" - )) -} - -#[derive(Deserialize, Debug)] -pub struct ReducedRoom { - id: String, - props: ReducedRoomProps, -} - -#[derive(Deserialize, Debug)] -pub struct ReducedRoomProps { - calendar_url: Option, //tumonline_room_nr and calendar_url are sometimes not present, but only ever both - tumonline_room_nr: Option, -} - -#[derive(Clone, Debug, Default)] -pub struct Room { - pub sap_id: String, - pub tumonline_org_id: i32, - pub tumonline_calendar_id: i32, - pub tumonline_room_id: i32, -} - -impl Room { - fn from(room: ReducedRoom) -> Option { - let url = room.props.calendar_url?; - let regex = Regex::new(r".*cOrg=(?P\d+)&cRes=(?P\d+)\D.*").unwrap(); - let captures = regex.captures(&url)?; - Some(Room { - sap_id: room.id, - tumonline_org_id: captures.name("org")?.as_str().parse().ok()?, - tumonline_calendar_id: captures.name("cal")?.as_str().parse().ok()?, - tumonline_room_id: room.props.tumonline_room_nr?, - }) - } -} - -pub async fn get_all_ids(conn: &PgPool) -> Vec { - let url = - api_url_from_env().unwrap_or_else(|| "https://nav.tum.de/cdn/api_data.json".to_string()); - let res = reqwest::get(&url).await; - let rooms = match res { - Ok(res) => res.json::>().await, - Err(e) => { - error!("Failed to contact main-api at {url}: {e:#?}"); - return vec![]; - } - }; - let rooms: Vec = match rooms { - Ok(rooms) => rooms.into_iter().filter_map(Room::from).collect(), - Err(e) => panic!("Failed to parse main-api response: {e:#?}"), - }; - let start_time = Utc::now().naive_utc(); - store_in_db(conn, &rooms, &start_time).await; - delete_stale_results(conn, start_time).await; - rooms -} - -async fn store_in_db(conn: &PgPool, rooms_to_store: &[Room], start_time: &NaiveDateTime) { - info!( - "Storing {cnt} rooms in database", - cnt = rooms_to_store.len() - ); - for room in rooms_to_store { - if let Err(e) =sqlx::query!(r#" - INSERT INTO rooms(key,tumonline_org_id,tumonline_calendar_id,tumonline_room_id,last_scrape) - VALUES ($1,$2,$3,$4,$5) - ON CONFLICT (key) DO UPDATE SET - tumonline_org_id=$2, - tumonline_calendar_id=$3, - tumonline_room_id=$4, - last_scrape=$5 - "#, - room.sap_id.clone(), - room.tumonline_org_id, - room.tumonline_calendar_id, - room.tumonline_room_id, - *start_time, - ).execute(conn).await { - error!("Error inserting into database: {e:?}"); - } - } -} -async fn delete_stale_results(conn: &PgPool, start_time: NaiveDateTime) { - info!("Deleting stale rooms from the database"); - if let Err(e) = sqlx::query!("DELETE FROM rooms WHERE last_scrape < $1", start_time) - .execute(conn) - .await - { - error!("Error deleting stale rooms from database: {e:?}"); - } -} diff --git a/server/calendar/src/scrape_task/mod.rs b/server/calendar/src/scrape_task/mod.rs deleted file mode 100644 index a47efa256..000000000 --- a/server/calendar/src/scrape_task/mod.rs +++ /dev/null @@ -1,158 +0,0 @@ -mod main_api_connector; -mod scrape_room_task; -pub mod tumonline_calendar_connector; - -use crate::scrape_task::main_api_connector::{get_all_ids, Room}; -use crate::scrape_task::scrape_room_task::ScrapeRoomTask; -use crate::scrape_task::tumonline_calendar_connector::{Strategy, XMLEvents}; -use chrono::{DateTime, NaiveDate, Utc}; -use futures::stream::FuturesUnordered; -use futures::StreamExt; -use lazy_static::lazy_static; -use log::{info, warn}; -use prometheus::{register_counter, register_histogram, Counter, Histogram}; -use sqlx::PgPool; -use std::time::{Duration, Instant}; -use tokio::time::sleep; -lazy_static! { - static ref SCRAPED_CALENDAR_ENTRIES_COUNTER: Counter = register_counter!( - "navigatum_calendarscraper_total_entries", - "Total number of calendar entries scraped." - ) - .unwrap(); - static ref REQ_SUCCESS_HISTOGRAM: Histogram = register_histogram!( - "navigatum_calendarscraper_resulting_entries_buckets", - "Amount of entries retrieved", - prometheus::exponential_buckets(10.0, 2.0, 15).unwrap(), - ) - .unwrap(); - static ref REQ_TIME_HISTOGRAM: Histogram = register_histogram!( - "navigatum_calendarscraper_request_duration_ms_buckets", - "The scrape request latencies in seconds", - prometheus::linear_buckets(20.0, 20.0, 15).unwrap(), - ) - .unwrap(); -} - -pub struct ScrapeTask { - rooms_to_scrape: Vec, - rooms_cnt: usize, - time_window: chrono::Duration, - scraping_start: DateTime, -} - -const CONCURRENT_REQUESTS: usize = 2; -impl ScrapeTask { - pub async fn new(conn: &PgPool, time_window: chrono::Duration) -> Self { - let rooms_to_scrape = get_all_ids(conn).await; - let rooms_cnt = rooms_to_scrape.len(); - Self { - rooms_to_scrape, - rooms_cnt, - time_window, - scraping_start: Utc::now(), - } - } - - pub async fn scrape_to_db(&mut self, conn: &sqlx::PgPool) { - info!("Starting scraping calendar entries"); - let mut work_queue = FuturesUnordered::new(); - let start = self.scraping_start - self.time_window / 2; - while !self.rooms_to_scrape.is_empty() { - while work_queue.len() < CONCURRENT_REQUESTS { - if let Some(room) = self.rooms_to_scrape.pop() { - // sleep to not overload TUMonline. - // It is critical for successfully scraping that we are not blocked. - sleep(Duration::from_millis(50)).await; - - work_queue.push(scrape(conn, room, start.date_naive(), self.time_window)); - } - } - work_queue.next().await; - - let scraped_entries = self.rooms_cnt - self.rooms_to_scrape.len(); - if scraped_entries % 30 == 0 { - let elapsed = self.elapsed(); - info!("Scraped {progress:.2}% ({scraped_entries}/{rooms_cnt}) in {elapsed:.1?} (avg {time_per_key:.1?}/key)", - rooms_cnt=self.rooms_cnt, - progress = scraped_entries as f32 / self.rooms_cnt as f32 * 100.0, - time_per_key = elapsed / scraped_entries as u32); - } - } - - info!( - "Finished scraping calendar entrys. ({rooms_cnt} entries in {elapsed:?})", - rooms_cnt = self.rooms_cnt, - elapsed = self.elapsed(), - ); - } - - fn elapsed(&self) -> Duration { - (Utc::now() - self.scraping_start).to_std().unwrap() - } - - pub async fn delete_stale_results(&self, conn: &PgPool) { - let start_time = Instant::now(); - let scrape_interval = ( - self.scraping_start - self.time_window / 2, - self.scraping_start + self.time_window / 2, - ); - sqlx::query!( - "DELETE FROM calendar WHERE dtstart > $1 AND dtend < $2 AND last_scrape < $3", - scrape_interval.0.naive_local(), - scrape_interval.1.naive_local(), - self.scraping_start.naive_local() - ) - .execute(conn) - .await - .expect("Failed to delete calendar"); - - info!( - "Finished deleting stale results ({time_window} in {passed:?})", - time_window = self.time_window, - passed = start_time.elapsed(), - ); - } -} - -async fn scrape(conn: &PgPool, room: Room, from: NaiveDate, duration: chrono::Duration) { - let _timer = REQ_TIME_HISTOGRAM.start_timer(); // drop as observe - - // request and parse the xml file - let mut request_queue = vec![ScrapeRoomTask::new(room, from, duration)]; - let mut success_cnt = 0; - while !request_queue.is_empty() { - let mut new_request_queue = vec![]; - for task in request_queue { - let events = XMLEvents::request(task.clone()).await; - - //store the events in the database if successful, otherwise retry - match events { - Ok(events) => { - success_cnt += events.len(); - events.store_in_db(conn).await; - } - Err(retry) => match retry { - Strategy::NoRetry => {} - Strategy::RetrySmaller => { - if task.num_days() > 1 { - let (t1, t2) = task.split(); - new_request_queue.push(t1); - new_request_queue.push(t2); - } else { - warn!("The following ScrapeOrder cannot be fulfilled: {task:?}"); - } - } - }, - }; - - // sleep to not overload TUMonline. - // It is critical for successfully scraping that we are not blocked. - sleep(Duration::from_millis(50)).await; - } - request_queue = new_request_queue; - } - - REQ_SUCCESS_HISTOGRAM.observe(success_cnt as f64); - SCRAPED_CALENDAR_ENTRIES_COUNTER.inc_by(success_cnt as f64); -} diff --git a/server/calendar/src/scrape_task/scrape_room_task.rs b/server/calendar/src/scrape_task/scrape_room_task.rs deleted file mode 100644 index d4f010333..000000000 --- a/server/calendar/src/scrape_task/scrape_room_task.rs +++ /dev/null @@ -1,96 +0,0 @@ -use crate::scrape_task::main_api_connector::Room; -use chrono::NaiveDate; - -#[derive(Clone, Debug)] -pub(crate) struct ScrapeRoomTask { - pub(crate) room: Room, - pub(crate) from: NaiveDate, - pub(crate) to: NaiveDate, -} - -impl ScrapeRoomTask { - pub fn new(room: Room, from: NaiveDate, duration: chrono::Duration) -> Self { - let to = from + duration; - Self { room, from, to } - } - pub fn num_days(&self) -> u64 { - // we want to count from the morning of "from" to the evening of "to" => +1 - (self.to + chrono::Days::new(1)) - .signed_duration_since(self.from) - .num_days() as u64 - } - pub fn split(&self) -> (Self, Self) { - let mid_offset = self.num_days() / 2 - 1; - let lower_middle = self.from + chrono::Days::new(mid_offset); - ( - Self { - room: self.room.clone(), - from: self.from, - to: lower_middle, - }, - Self { - room: self.room.clone(), - from: lower_middle + chrono::Days::new(1), - to: self.to, - }, - ) - } -} - -#[cfg(test)] -mod test_scrape_task { - use super::ScrapeRoomTask; - use crate::scrape_task::main_api_connector::Room; - use chrono::NaiveDate; - use pretty_assertions::assert_eq; - - #[test] - fn test_split() { - let start = NaiveDate::from_ymd_opt(2020, 1, 1).unwrap(); - let task = ScrapeRoomTask::new(Room::default(), start, chrono::Duration::days(365)); - let (o1, o2) = task.split(); - assert_eq!(task.from, start); - assert_eq!(task.to, NaiveDate::from_ymd_opt(2020, 12, 31).unwrap()); - assert_eq!(o1.from, task.from); - assert_eq!(o2.to, task.to); - assert_eq!(o1.to + chrono::Duration::days(1), o2.from); - } - #[test] - fn test_split_small() { - let task = ScrapeRoomTask { - room: Room::default(), - from: NaiveDate::from_ymd_opt(2020, 1, 1).unwrap(), - to: NaiveDate::from_ymd_opt(2020, 1, 2).unwrap(), - }; - let (t1, t2) = task.split(); - assert_eq!(t1.to + chrono::Duration::days(1), t2.from); - assert_eq!(task.num_days(), 2); - assert_eq!(task.from, t1.from); - assert_eq!(task.to, t2.to); - assert_eq!(t1.num_days(), 1); - assert_eq!(t2.num_days(), 1); - assert_eq!(task.from, t1.to); - assert_eq!(task.to, t2.from); - } - #[test] - fn test_num_days() { - let mut task = ScrapeRoomTask { - room: Room::default(), - from: NaiveDate::from_ymd_opt(2020, 1, 1).unwrap(), - to: NaiveDate::from_ymd_opt(2020, 1, 1).unwrap(), - }; - assert_eq!(task.num_days(), 1); - task.to = NaiveDate::from_ymd_opt(2020, 1, 2).unwrap(); - assert_eq!(task.num_days(), 2); - task.to = NaiveDate::from_ymd_opt(2020, 12, 31).unwrap(); - assert_eq!(task.num_days(), 366); - } - #[test] - fn test_same_day() { - let start = NaiveDate::from_ymd_opt(2020, 1, 1).unwrap(); - let task = ScrapeRoomTask::new(Room::default(), start, chrono::Duration::days(0)); - assert_eq!(task.from, start); - assert_eq!(task.to, NaiveDate::from_ymd_opt(2020, 1, 1).unwrap()); - assert_eq!(task.num_days(), 1); - } -} diff --git a/server/calendar/src/scrape_task/tumonline_calendar_connector.rs b/server/calendar/src/scrape_task/tumonline_calendar_connector.rs deleted file mode 100644 index f322d5ad4..000000000 --- a/server/calendar/src/scrape_task/tumonline_calendar_connector.rs +++ /dev/null @@ -1,250 +0,0 @@ -use crate::models::XMLEvent; -use crate::scrape_task::main_api_connector::Room; -use crate::scrape_task::scrape_room_task::ScrapeRoomTask; -use chrono::{NaiveDateTime, Utc}; -use log::{debug, error, warn}; -use minidom::Element; -use rand::Rng; -use sqlx::PgPool; -use std::collections::HashMap; -use std::time::Duration; -use tokio::time::sleep; - -enum RequestStatus { - Success(String), - Timeout, - NotFound, - OneDayFaulty, - Error, -} - -async fn request_body(url: String) -> RequestStatus { - let request = reqwest::get(&url).await; - let body = match request { - Ok(response) => match response.status().as_u16() { - 200 => response.text().await, - 404 => return RequestStatus::NotFound, - _ => { - error!("Error sending request (invalid status code): {response:?}"); - return RequestStatus::Error; - } - }, - Err(e) => { - if e.is_timeout() { - return RequestStatus::Timeout; - } - error!("Error sending request: {e:?}"); - return RequestStatus::Error; - } - }; - let res_string = match body { - Ok(body) => body, - Err(e) => { - error!("Error converting body to string: {e:?}"); - return RequestStatus::Error; - } - }; - match res_string.as_str() { - "" => RequestStatus::OneDayFaulty, - _ => RequestStatus::Success(res_string), - } -} - -fn construct_hm(elem: &Element) -> HashMap { - let mut hm: HashMap = HashMap::new(); - let attrs = elem.children().filter(|e| e.is("attribute", NS)); - let readable_attrs = attrs.map(|e| (e.attr("cor:attrID").unwrap(), e.text())); - readable_attrs.for_each(|(attr, val)| { - hm.insert(attr.to_string(), val); - }); - - hm -} - -fn xml_event_from_hm(key: String, xml_hm: &HashMap) -> XMLEvent { - let other_keys = xml_hm - .keys() - .filter(|s| { - !matches!( - s.to_string().as_str(), - "dtstart" - | "dtend" - | "dtstamp" - | "duration" // ignored - | "eventID" - | "eventTitle" - | "singleEventID" - | "singleEventTypeID" - | "singleEventIDSuccessor" // ignored - | "singleEventTypeName" - | "eventTypeID" - | "eventTypeName" - | "courseTypeName" - | "courseType" - | "courseCode" - | "courseSemesterHours" - | "groupID" - | "group" - | "statusID" - | "status" - |"comment" - ) - }) - .map(|s| s.to_string()) - .collect::>(); - if !other_keys.is_empty() { - error!("found additional key(s) in hashmap: {other_keys:?}"); - } - XMLEvent { - key, - dtstart: extract_dt(xml_hm, "dtstart").unwrap(), - dtend: extract_dt(xml_hm, "dtend").unwrap(), - dtstamp: extract_dt(xml_hm, "dtstamp").unwrap(), - event_id: extract_i32(xml_hm, "eventID").unwrap(), - event_title: extract_str(xml_hm, "eventTitle") - .unwrap_or_else(|| "Title not available".to_string()), // some deleted entries are broken in this sens - single_event_id: extract_i32(xml_hm, "singleEventID").unwrap(), - single_event_type_id: extract_str(xml_hm, "singleEventTypeID").unwrap(), - single_event_type_name: extract_str(xml_hm, "singleEventTypeName").unwrap(), - event_type_id: extract_str(xml_hm, "eventTypeID").unwrap(), - event_type_name: extract_str(xml_hm, "eventTypeName"), - course_type_name: extract_str(xml_hm, "courseTypeName"), - course_type: extract_str(xml_hm, "courseType"), - course_code: extract_str(xml_hm, "courseCode"), - course_semester_hours: extract_i32(xml_hm, "courseSemesterHours"), - group_id: extract_str(xml_hm, "groupID"), - xgroup: extract_str(xml_hm, "group"), - status_id: extract_str(xml_hm, "statusID").unwrap(), - status: extract_str(xml_hm, "status").unwrap(), - comment: extract_str(xml_hm, "comment").unwrap_or_default(), - last_scrape: Utc::now().naive_utc(), - } -} - -fn extract_i32(hm: &HashMap, key: &str) -> Option { - let str = extract_str(hm, key)?; - str.parse::().ok() -} - -fn extract_dt(hm: &HashMap, key: &str) -> Option { - let str = extract_str(hm, key).unwrap(); - NaiveDateTime::parse_from_str(&str, "%Y%m%dT%H%M%S").ok() -} - -fn extract_str(hm: &HashMap, key: &str) -> Option { - hm.get(key).map(|s| s.trim().to_string()) -} - -pub struct XMLEvents { - events: Vec, -} - -const NS: &str = "http://rdm.campusonline.at/"; - -const CALENDAR_BASE_URL: &str = - "https://campus.tum.de/tumonlinej/ws/webservice_v1.0/rdm/room/schedule/xml"; - -impl XMLEvents { - pub(crate) fn len(&self) -> usize { - self.events.len() - } - pub(crate) async fn store_in_db(self, conn: &PgPool) { - for event in self.events { - if let Err(e) = sqlx::query!(r#" - INSERT INTO calendar(key, dtstart, dtend, dtstamp, event_id, event_title, single_event_id, single_event_type_id, single_event_type_name, event_type_id, event_type_name, course_type_name, course_type, course_code, course_semester_hours, group_id, xgroup, status_id, status, comment, last_scrape) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21)"#, - event.key, event.dtstart, event.dtend, event.dtstamp, event.event_id, event.event_title, event.single_event_id, event.single_event_type_id, event.single_event_type_name, event.event_type_id, event.event_type_name, event.course_type_name, event.course_type, event.course_code, event.course_semester_hours, event.group_id, event.xgroup, event.status_id, event.status, event.comment, event.last_scrape) - .execute(conn).await { - error!("Error inserting into database: {e:?}"); - } - } - } - fn new(requested_room: &Room, body: &str) -> Option { - let root = match body.parse::() { - Ok(root) => root, - Err(e) => { - error!("Error parsing body to xml: {e:?} body={body:?}"); - return None; - } - }; - let mut events: Vec = Vec::new(); - let res = root.get_child("resource", NS).unwrap(); - let desc = res.get_child("description", NS).unwrap(); - let rg = desc.get_child("resourceGroup", NS).unwrap(); - let xml_super_events = rg.get_child("description", NS).unwrap(); - let xml_events = xml_super_events - .children() - .filter_map(|e| e.get_child("description", NS)); - - for e in xml_events { - let hm = construct_hm(e); - - let valid_status = match hm.get("status") { - Some(s) => match s.as_str() { - "fix" | "geplant" => true, - "verschoben" | "gelöscht" | "abgesagt" | "abgelehnt" => false, - _ => { - error!("unknown status: {s:?}"); - false - } - }, - _ => false, - }; - if valid_status { - events.push(xml_event_from_hm(requested_room.sap_id.clone(), &hm)); - } - } - Some(XMLEvents { events }) - } - pub(crate) async fn request(task: ScrapeRoomTask) -> Result { - // The token being embedded here is not an issue, since the token has only access to - // the data this API is providing anyway... - // If people want to disrupt this API, they can just do it by abusing this TUMonline-endpoint. - // We (and TUMonline) monitor for this and will switch to a backup token not in this API - // We do not want to repeat the DOS-attack that happened to TUMonline in December of 2022. - let token = std::env::var("TUMONLINE_TOKEN") - .unwrap_or_else(|_| "yeIKcuCGSzUCosnPZcKXkGeyUYGTQqUw".to_string()); - - //get the xml file from TUMonline - //why this API uses the tumonline_room_id and not the tumonline_calendar_id like the URLs is unclear - //TUMonline apparently thinks this is sane - let url = format!( - "{CALENDAR_BASE_URL}?roomID={room_id}&timeMode=absolute&fromDate={from}&untilDate={to}&token={token}&buildingCode=", - room_id=task.room.tumonline_room_id, - from=task.from.format("%Y%m%d"), - to=task.to.format("%Y%m%d") - ); - debug!("url: {url}"); - for retry_cnt in 1..=5 { - let body = request_body(url.to_string()).await; - // randomized to avoid thundering herd phenomenon - let mut rng = rand::thread_rng(); - // Retry 1: 400..800ms - // Retry 5: 6.4s..12.8s - let backoff_ms = rng.gen_range(2_u64.pow(retry_cnt)..2_u64.pow(retry_cnt + 1)) * 200; - let backoff_duration = Duration::from_millis(backoff_ms); - match body { - RequestStatus::Success(body) => { - return XMLEvents::new(&task.room, &body).ok_or(Strategy::NoRetry); - } - // This consistently means, that there is no data for this room - RequestStatus::NotFound => return Err(Strategy::NoRetry), - // TUMonline sometimes returns an empty body due to one day being invalid - // => Retry smaller will get the other entries.. - RequestStatus::OneDayFaulty => return Err(Strategy::RetrySmaller), - RequestStatus::Timeout | RequestStatus::Error => { - warn!("Retry {retry_cnt}/5, retrying in {backoff_duration:?} url={url}"); - } - }; - sleep(backoff_duration).await; - } - // the entry may just be too large, as can be seen that we are getting enough Timeouts/Errors - // => retrying smaller may be able to get around the errors/timeouts - Err(Strategy::RetrySmaller) - } -} - -pub enum Strategy { - NoRetry, - RetrySmaller, -} diff --git a/server/calendar/src/scraper.rs b/server/calendar/src/scraper.rs deleted file mode 100644 index 595d757fc..000000000 --- a/server/calendar/src/scraper.rs +++ /dev/null @@ -1,78 +0,0 @@ -use crate::scrape_task::ScrapeTask; -use log::{error, info}; -use structured_logger::{async_json::new_writer, Builder}; - -use std::fmt; - -use prometheus::labels; -use sqlx::postgres::PgPoolOptions; - -mod models; -mod scrape_task; -mod utils; - -struct TimeWindow { - duration: chrono::Duration, -} - -impl TimeWindow { - fn init_from_env() -> Self { - let time_window_months = std::env::var("SCRAPED_TIME_WINDOW_MONTHS") - .expect("SCRAPED_TIME_WINDOW_MONTHS not set"); - let time_window_months = time_window_months - .parse::() - .expect("SCRAPED_TIME_WINDOW_MONTHS not a number"); - // 30 days/month is a simplification, but over-scraping by a few days probably does not matter - Self { - duration: chrono::Duration::days(time_window_months * 30), - } - } -} - -impl fmt::Debug for TimeWindow { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let num_years = self.duration.num_days() / 365; - let num_remaining_days = self.duration.num_days() - num_years * 365; - f.debug_struct("TimeWindow") - .field("years", &num_years) - .field("months", &(num_remaining_days / 30)) - .finish() - } -} - -#[tokio::main] -async fn main() { - Builder::with_level("info") - .with_target_writer("*", new_writer(tokio::io::stdout())) - .init(); - - let uri = utils::connection_string(); - let pool = PgPoolOptions::new() - .max_connections(20) - .connect(&uri) - .await - .expect("Failed to connect to database"); - let time_window = TimeWindow::init_from_env(); - info!("Scraping time window: {time_window:?}"); - let mut scraper = ScrapeTask::new(&pool, time_window.duration).await; - scraper.scrape_to_db(&pool).await; - scraper.delete_stale_results(&pool).await; - - info!("Pushing metrics to the pushgateway"); - tokio::task::spawn_blocking(move || { - let address = std::env::var("PUSHGATEWAY_URL") - .unwrap_or_else(|_| "pushgateway.monitoring.svc.cluster.local".to_string()); - prometheus::push_metrics( - "navigatum_calendarscraper", - labels! {"duration".to_owned() => format!("{time_window:?}"),}, - &address, - prometheus::gather(), - None, - ) - .unwrap_or_else(|e| { - error!("could not push metrics to the pushgateway, because: {e:?}"); - }); - }) - .await - .expect("Spawing a blocking task failed"); -} diff --git a/server/calendar/src/utils.rs b/server/calendar/src/utils.rs deleted file mode 100644 index 017a6612c..000000000 --- a/server/calendar/src/utils.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub fn connection_string() -> String { - let username = std::env::var("POSTGRES_USER").unwrap_or_else(|_| "postgres".to_string()); - let password = std::env::var("POSTGRES_PASSWORD").unwrap_or_else(|_| "password".to_string()); - let url = std::env::var("POSTGRES_URL").unwrap_or_else(|_| "localhost".to_string()); - let db = std::env::var("POSTGRES_DB").unwrap_or_else(|_| username.clone()); - format!("postgres://{username}:{password}@{url}/{db}") -} diff --git a/server/feedback/src/main.rs b/server/feedback/src/main.rs index 40eae54b1..4bd2c0bbf 100644 --- a/server/feedback/src/main.rs +++ b/server/feedback/src/main.rs @@ -1,6 +1,7 @@ use actix_cors::Cors; use actix_governor::{GlobalKeyExtractor, Governor, GovernorConfigBuilder}; use std::collections::HashMap; +use std::error::Error; use crate::tokens::RecordedTokens; use actix_web::{get, middleware, web, App, HttpResponse, HttpServer}; @@ -12,6 +13,7 @@ mod github; mod post_feedback; mod proposed_edits; mod tokens; +type BoxedError = Box; const MAX_JSON_PAYLOAD: usize = 1024 * 1024; // 1 MB diff --git a/server/feedback/src/proposed_edits/image.rs b/server/feedback/src/proposed_edits/image.rs index 0213e2ec8..1c217800a 100644 --- a/server/feedback/src/proposed_edits/image.rs +++ b/server/feedback/src/proposed_edits/image.rs @@ -6,7 +6,6 @@ use serde::Deserialize; use serde::Serialize; use std::cmp::max; use std::collections::BTreeMap; -use std::error; use std::fs::File; use std::path::{Path, PathBuf}; @@ -57,7 +56,7 @@ impl Image { } } } - fn save_metadata(&self, key: &str, image_dir: &Path) -> Result<(), Box> { + fn save_metadata(&self, key: &str, image_dir: &Path) -> Result<(), crate::BoxedError> { let file = File::open(image_dir.join("img-sources.yaml"))?; let mut image_sources = serde_yaml::from_reader::<_, BTreeMap>>(file)?; @@ -80,7 +79,7 @@ impl Image { + 1; image_dir.join(format!("{key}_{next_free_slot}.webp")) } - fn save_content(&self, target: &Path) -> Result<(), Box> { + fn save_content(&self, target: &Path) -> Result<(), crate::BoxedError> { let bytes = BASE64_STANDARD.decode(&self.content)?; let image = image::load_from_memory(&bytes)?; diff --git a/server/feedback/src/proposed_edits/mod.rs b/server/feedback/src/proposed_edits/mod.rs index 84e8fc8d2..11ecd5298 100644 --- a/server/feedback/src/proposed_edits/mod.rs +++ b/server/feedback/src/proposed_edits/mod.rs @@ -36,7 +36,7 @@ impl EditRequest { async fn apply_changes_and_generate_description( &self, branch_name: &str, - ) -> Result> { + ) -> Result { let repo = TempRepo::clone_and_checkout(GIT_URL, branch_name).await?; let desc = repo.apply_and_gen_description(self); repo.commit(&desc.title).await?; diff --git a/server/feedback/src/proposed_edits/tmp_repo.rs b/server/feedback/src/proposed_edits/tmp_repo.rs index 8d149fb95..1f40ace00 100644 --- a/server/feedback/src/proposed_edits/tmp_repo.rs +++ b/server/feedback/src/proposed_edits/tmp_repo.rs @@ -1,6 +1,5 @@ use crate::proposed_edits::EditRequest; use log::{debug, info}; -use std::error; use tokio::process::Command; use crate::proposed_edits::discription::Description; @@ -13,7 +12,7 @@ impl TempRepo { pub async fn clone_and_checkout( url: &'static str, branch_name: &str, - ) -> Result> { + ) -> Result { let dir = tempfile::tempdir()?; info!("Cloning {url} into {dir:?}"); @@ -60,7 +59,7 @@ impl TempRepo { description } - pub async fn commit(&self, title: &str) -> Result<(), Box> { + pub async fn commit(&self, title: &str) -> Result<(), crate::BoxedError> { let out = Command::new("git") .current_dir(&self.dir) .arg("add") @@ -82,7 +81,7 @@ impl TempRepo { _ => Err(format!("git commit failed with output: {out:?}").into()), } } - pub async fn push(&self) -> Result<(), Box> { + pub async fn push(&self) -> Result<(), crate::BoxedError> { let out = Command::new("git") .current_dir(&self.dir) .arg("status") diff --git a/server/main-api/Cargo.toml b/server/main-api/Cargo.toml index 11be7a9f4..fece13f4e 100644 --- a/server/main-api/Cargo.toml +++ b/server/main-api/Cargo.toml @@ -34,15 +34,19 @@ serde_yaml = "0.9" # database sqlx = { version = "0.7.3", features = ["postgres", "runtime-tokio-rustls", "migrate", "macros", "chrono"] } +chrono = { version = "0.4.31", default-features = false, features = ["serde"] } # search meilisearch-sdk = "0.24.3" -logos="0.13.0" +logos = "0.13.0" regex = "1.10.3" -# maps +# web access +oauth2 = { version = "4.4.2", default-features = false, features = ["rustls-tls", "reqwest"] } rustls = "0.22.2" -reqwest = { version= "0.11.23", default-features = false, features = ["tokio-rustls", "json", "gzip"] } +reqwest = { version = "0.11.23", default-features = false, features = ["tokio-rustls", "rustls-tls-webpki-roots", "rustls", "webpki-roots", "json", "gzip"] } + +# image production image = "0.24.8" imageproc = "0.23.0" rusttype = "0.9.3" @@ -50,3 +54,7 @@ lazy_static = "1.4.0" [dev-dependencies] pretty_assertions.workspace = true + +[features] +skip_db_setup = [] +skip_ms_setup = [] diff --git a/server/main-api/README.md b/server/main-api/README.md index 91ab3120d..10cb3e430 100644 --- a/server/main-api/README.md +++ b/server/main-api/README.md @@ -65,7 +65,8 @@ To get compiletime guarantees for our queries, we use sqlx. To add/edit a query, you will need to run the following command: ```bash -cargo sqlx prepare --database-url sqlite://main-api/api_data.db --workspace +sqlx migrate run --database-url postgres://postgres:password@localhost:5432/postgres +cargo sqlx prepare --database-url postgres://postgres:password@localhost:5432/postgres --workspace ``` ### How to Set up the tileserver (needed for the `preview` endpoint) diff --git a/server/main-api/migrations/20240101134507_migrate_to_tum_calendar.down.sql b/server/main-api/migrations/20240101134507_migrate_to_tum_calendar.down.sql new file mode 100644 index 000000000..aa5b4e439 --- /dev/null +++ b/server/main-api/migrations/20240101134507_migrate_to_tum_calendar.down.sql @@ -0,0 +1,30 @@ +-- This file should undo anything in `up.sql` +DROP TABLE IF EXISTS calendar; +DROP INDEX IF EXISTS calendar_lut; +DROP TYPE IF EXISTS EventType; + +CREATE TABLE calendar +( + key VARCHAR(30) NOT NULL, + dtstart timestamp NOT NULL, + dtend timestamp NOT NULL, + dtstamp timestamp NOT NULL, + event_id INTEGER NOT NULL, + event_title TEXT NOT NULL, + single_event_id INTEGER UNIQUE PRIMARY KEY NOT NULL, + single_event_type_id TEXT NOT NULL, + single_event_type_name TEXT NOT NULL, + event_type_id TEXT NOT NULL, + event_type_name TEXT, + course_type_name TEXT, + course_type TEXT, + course_code TEXT, + course_semester_hours INTEGER, + group_id TEXT, + xgroup TEXT, + status_id TEXT NOT NULL, + status TEXT NOT NULL, + comment TEXT NOT NULL, + last_scrape TIMESTAMP NOT NULL +); +CREATE INDEX IF NOT EXISTS calendar_lut ON calendar (key, dtstart, dtend); diff --git a/server/main-api/migrations/20240101134507_migrate_to_tum_calendar.up.sql b/server/main-api/migrations/20240101134507_migrate_to_tum_calendar.up.sql new file mode 100644 index 000000000..b991a5df7 --- /dev/null +++ b/server/main-api/migrations/20240101134507_migrate_to_tum_calendar.up.sql @@ -0,0 +1,18 @@ +-- Your SQL goes here +DROP TABLE IF EXISTS calendar; +DROP INDEX IF EXISTS calendar_lut; + +CREATE TYPE EventType AS ENUM ('lecture','exercise','exam','barred','other'); +CREATE TABLE calendar +( + id INTEGER UNIQUE PRIMARY KEY NOT NULL, + room_code VARCHAR(30) NOT NULL references en, + start_at TIMESTAMPTZ NOT NULL, + end_at TIMESTAMPTZ NOT NULL, + stp_title_de TEXT NOT NULL, + stp_title_en TEXT NOT NULL, + stp_type TEXT NOT NULL, + entry_type EventType NOT NULL, + detailed_entry_type TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS calendar_lut ON calendar (room_code, start_at, end_at); diff --git a/server/main-api/migrations/20240123175552_store_when_last_scrape_happened.down.sql b/server/main-api/migrations/20240123175552_store_when_last_scrape_happened.down.sql new file mode 100644 index 000000000..2d9659a39 --- /dev/null +++ b/server/main-api/migrations/20240123175552_store_when_last_scrape_happened.down.sql @@ -0,0 +1,3 @@ +-- Add down migration script here +alter table de drop column last_calendar_scrape_at; +alter table en drop column last_calendar_scrape_at; diff --git a/server/main-api/migrations/20240123175552_store_when_last_scrape_happened.up.sql b/server/main-api/migrations/20240123175552_store_when_last_scrape_happened.up.sql new file mode 100644 index 000000000..0433bd812 --- /dev/null +++ b/server/main-api/migrations/20240123175552_store_when_last_scrape_happened.up.sql @@ -0,0 +1,6 @@ +-- Add up migration script here +alter table de add last_calendar_scrape_at TIMESTAMPTZ default null; +comment on column de.last_calendar_scrape_at is 'the last time the calendar was scraped for this room'; + +alter table en add last_calendar_scrape_at TIMESTAMPTZ default null; +comment on column en.last_calendar_scrape_at is 'the last time the calendar was scraped for this room'; diff --git a/server/main-api/src/calendar/fetch/connectum.rs b/server/main-api/src/calendar/fetch/connectum.rs new file mode 100644 index 000000000..b75be1551 --- /dev/null +++ b/server/main-api/src/calendar/fetch/connectum.rs @@ -0,0 +1,142 @@ +use std::env; + +use cached::instant::Instant; +use chrono::{DateTime, Utc}; +use log::{debug, error, info, warn}; +use oauth2::basic::{BasicClient, BasicTokenResponse}; +use oauth2::reqwest::async_http_client; +use oauth2::url::Url; +use oauth2::{AuthUrl, ClientId, ClientSecret, Scope, TokenResponse, TokenUrl}; +use sqlx::PgPool; + +use crate::calendar::fetch::CalendarEntryFetcher; +use crate::calendar::models::Event; + +pub(super) struct APIRequestor { + client: reqwest::Client, + pool: PgPool, +} + +impl CalendarEntryFetcher for APIRequestor { + fn new(pool: &PgPool, _: &Option>) -> Self { + Self { + client: reqwest::Client::new(), + pool: pool.clone(), + } + } + async fn fetch( + &self, + id: &str, + start_after: &DateTime, + end_before: &DateTime, + ) -> Result { + let tumonline_id = id.replace('.', ""); + + let sync_start = Utc::now(); + let start = Instant::now(); + // Make OAuth2 secured request + let oauth_token = self + .fetch_oauth_token() + .await? + .access_token() + .secret() + .clone(); + let url = format!( + "https://review.campus.tum.de/RSYSTEM/co/connectum/api/rooms/{tumonline_id}/calendars" + ); + + let events: Vec = self + .client + .get(url) + .bearer_auth(oauth_token) + .send() + .await? + .json() + .await?; + info!( + "finished fetching for {id}: {cnt} calendar events in {elapsed:?}", + cnt = events.len(), + elapsed = start.elapsed() + ); + let events = events + .into_iter() + .map(|mut e| { + e.room_code = id.into(); + e + }) + .collect::>(); + self.store(&events, &sync_start, id).await?; + let events = events + .into_iter() + .filter(|e| *start_after <= e.start_at && *end_before >= e.end_at) + .collect(); + Ok((sync_start, events)) + } +} + +impl APIRequestor { + async fn store( + &self, + events: &[Event], + last_sync: &DateTime, + id: &str, + ) -> Result<(), crate::BoxedError> { + // insert into db + let mut tx = self.pool.begin().await?; + if let Err(e) = self.delete_events(id, &mut tx).await { + error!("could not delete existing events because {e:?}"); + tx.rollback().await?; + return Err(e.into()); + } + for (i, event) in events.iter().enumerate() { + // conflicts cannot occur because all values for said room were dropped + if let Err(e) = event.store(&mut tx).await { + warn!( + "ignoring insert {event:?} ({i}/{total}) because {e:?}", + total = events.len() + ); + } + } + sqlx::query!( + "UPDATE de SET last_calendar_scrape_at = $1 WHERE key=$2", + last_sync, + id + ) + .execute(&self.pool) + .await?; + tx.commit().await?; + debug!("finished inserting into the db for {id}"); + Ok(()) + } + async fn fetch_oauth_token(&self) -> Result { + let client_id = env::var("TUMONLINE_OAUTH_CLIENT_ID").expect( + "please configure the environment variable TUMONLINE_OAUTH_CLIENT_ID to use this endpoint", + ); + let client_secret = env::var("TUMONLINE_OAUTH_CLIENT_SECRET").expect("please configure the environment variable TUMONLINE_OAUTH_CLIENT_SECRET to use this endpoint"); + + // for urls see https://review.campus.tum.de/RSYSTEM/co/public/sec/auth/realms/CAMPUSonline/.well-known/openid-configuration + let auth_url = Url::parse("https://review.campus.tum.de/RSYSTEM/co/public/sec/auth/realms/CAMPUSonline/protocol/openid-connect/auth")?; + let token_url = Url::parse("https://review.campus.tum.de/RSYSTEM/co/public/sec/auth/realms/CAMPUSonline/protocol/openid-connect/token")?; + + let token = BasicClient::new( + ClientId::new(client_id), + Some(ClientSecret::new(client_secret)), + AuthUrl::from_url(auth_url), + Some(TokenUrl::from_url(token_url)), + ) + .exchange_client_credentials() + .add_scope(Scope::new("connectum-rooms.read".into())) + .request_async(async_http_client) + .await; + Ok(token?) // not directly returned for typing issues + } + async fn delete_events( + &self, + id: &str, + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + ) -> Result { + sqlx::query!(r#"DELETE FROM calendar WHERE room_code = $1"#, id) + .execute(&mut **tx) + .await + } +} diff --git a/server/main-api/src/calendar/fetch/db.rs b/server/main-api/src/calendar/fetch/db.rs new file mode 100644 index 000000000..c59d55427 --- /dev/null +++ b/server/main-api/src/calendar/fetch/db.rs @@ -0,0 +1,34 @@ +use crate::calendar::fetch::CalendarEntryFetcher; +use chrono::{DateTime, Utc}; +use sqlx::PgPool; + +pub(super) struct DbRequestor { + pool: PgPool, + last_calendar_scrape_at: Option>, +} + +impl CalendarEntryFetcher for DbRequestor { + fn new(pool: &PgPool, last_calendar_scrape_at: &Option>) -> Self { + Self { + pool: pool.clone(), + last_calendar_scrape_at: *last_calendar_scrape_at, + } + } + async fn fetch( + &self, + id: &str, + start_after: &DateTime, + end_before: &DateTime, + ) -> Result { + let events = sqlx::query_as!(crate::calendar::models::Event, r#"SELECT id,room_code,start_at,end_at,stp_title_de,stp_title_en,stp_type,entry_type AS "entry_type!:crate::calendar::models::EventType",detailed_entry_type + FROM calendar + WHERE room_code = $1 AND start_at >= $2 AND end_at <= $3"#, + id, start_after, end_before) + .fetch_all(&self.pool) + .await?; + let last_scrape = self + .last_calendar_scrape_at + .expect("an entry exists in the db, therefore the time of last scrape is known"); + Ok((last_scrape, events)) + } +} diff --git a/server/main-api/src/calendar/fetch/mod.rs b/server/main-api/src/calendar/fetch/mod.rs new file mode 100644 index 000000000..e84cd7db1 --- /dev/null +++ b/server/main-api/src/calendar/fetch/mod.rs @@ -0,0 +1,108 @@ +use std::ops::Sub; + +use actix_web::HttpResponse; +use chrono::{DateTime, FixedOffset, Utc}; +use log::error; +use sqlx::PgPool; + +use connectum::APIRequestor; +use db::DbRequestor; + +use crate::calendar::models::Event; + +mod connectum; +mod db; + +type CalendarEntries = (DateTime, Vec); + +trait CalendarEntryFetcher { + fn new(pool: &PgPool, last_calendar_scrape_at: &Option>) -> Self; + async fn fetch( + &self, + id: &str, + start_after: &DateTime, + end_before: &DateTime, + ) -> Result; +} + +pub struct StrategyExecutor { + pool: PgPool, + id: String, + start_after: DateTime, + end_before: DateTime, +} + +impl StrategyExecutor { + pub(super) fn new( + pool: &PgPool, + id: &str, + start_after: &DateTime, + end_before: &DateTime, + ) -> Self { + Self { + pool: pool.clone(), + id: id.into(), + start_after: *start_after, + end_before: *end_before, + } + } + async fn exec( + &self, + last_calendar_scrape_at: &Option>, + ) -> Result { + T::new(&self.pool, last_calendar_scrape_at) + .fetch(&self.id, &self.start_after, &self.end_before) + .await + } + + pub(super) async fn exec_with_retrying( + self, + last_calendar_scrape_at: &Option>, + ) -> Result { + let intial = match last_calendar_scrape_at { + Some(l) => { + if *l > Self::one_hour_ago() { + self.exec::(last_calendar_scrape_at).await + } else { + self.exec::(last_calendar_scrape_at).await + } + } + None => self.exec::(last_calendar_scrape_at).await, + }; + + match intial { + Ok(r) => Ok(r), + Err(e) => { + error!("could not fetch due to {e:?}"); + let last_scrape = last_calendar_scrape_at.unwrap_or_default(); + if last_scrape < Self::three_days_ago() { + match self.exec::(last_calendar_scrape_at).await { + Ok(res) => Ok(res), + Err(e) => { + error!("could not get substitute from db due to {e:?}"); + Err(HttpResponse::InternalServerError() + .body("could not get calendar entrys, please try again later")) + } + } + } else { + error!("cannot get substitute from db due to staleness"); + Err(HttpResponse::InternalServerError() + .body("could not get calendar entrys, please try again later")) + } + } + } + } + + fn one_hour_ago() -> DateTime { + let one_hour = FixedOffset::east_opt(60 * 60) + .expect("time travel is impossible and chronos is Y2K38-safe"); + Utc::now().sub(one_hour) + } + + fn three_days_ago() -> DateTime { + let three_days = chrono::Days::new(3); + Utc::now() + .checked_sub_days(three_days) + .expect("time travel is impossible and chronos is Y2K38-save") + } +} diff --git a/server/main-api/src/calendar/mod.rs b/server/main-api/src/calendar/mod.rs new file mode 100644 index 000000000..48679644b --- /dev/null +++ b/server/main-api/src/calendar/mod.rs @@ -0,0 +1,63 @@ +use actix_web::{get, web, HttpResponse}; +use chrono::{DateTime, Utc}; +use log::error; +use serde::Deserialize; +use sqlx::PgPool; + +use crate::models::Location; + +mod fetch; +mod models; + +async fn get_location(pool: &PgPool, id: &str) -> Result, sqlx::Error> { + sqlx::query_as!(Location, "SELECT * FROM de WHERE key = $1", id) + .fetch_optional(pool) + .await +} + +#[derive(Deserialize, Debug)] +pub struct QueryArguments { + /// eg. 2039-01-19T03:14:07+1 + start_after: DateTime, + /// eg. 2042-01-07T00:00:00 UTC + end_before: DateTime, +} + +#[get("/api/calendar/{id}")] +pub async fn calendar_handler( + params: web::Path, + web::Query(args): web::Query, + data: web::Data, +) -> HttpResponse { + let id = params.into_inner(); + let location = match get_location(&data.db, &id).await { + Err(e) => { + error!("could not refetch due to {e:?}"); + return HttpResponse::InternalServerError() + .body("could not get calendar entrys, please try again later"); + } + Ok(None) => { + return HttpResponse::NotFound() + .content_type("text/plain") + .body("Room not found"); + } + Ok(Some(loc)) => loc, + }; + let calendar_url = format!( + "https://campus.tum.de/tumonline/wbKalender.wbRessource?pResNr={id}", + id = 0 + ); // TODO: room.tumonline_calendar_id + let fetching_strategy = + fetch::StrategyExecutor::new(&data.db, &id, &args.start_after, &args.end_before); + match fetching_strategy + .exec_with_retrying(&location.last_calendar_scrape_at) + .await + { + Ok((last_sync, events)) => HttpResponse::Ok().json(models::Events { + events, + last_sync, + calendar_url, + }), + Err(resp) => resp, + } +} diff --git a/server/main-api/src/calendar/models.rs b/server/main-api/src/calendar/models.rs new file mode 100644 index 000000000..6efa77fe8 --- /dev/null +++ b/server/main-api/src/calendar/models.rs @@ -0,0 +1,72 @@ +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug)] +pub(super) struct Events { + pub(super) events: Vec, + pub(super) last_sync: DateTime, + pub(super) calendar_url: String, +} + +#[derive(Serialize, Deserialize, Debug, sqlx::Type)] +pub(super) struct Event { + pub(super) id: i32, + /// e.g. 5121.EG.003 + pub(super) room_code: String, + /// e.g. 2018-01-01T00:00:00 + pub(super) start_at: DateTime, + /// e.g. 2019-01-01T00:00:00 + pub(super) end_at: DateTime, + /// e.g. Quantenteleportation + pub(super) stp_title_de: String, + /// e.g. Quantum teleportation + pub(super) stp_title_en: String, + /// e.g. Vorlesung mit Zentralübung + pub(super) stp_type: String, + /// e.g. lecture + pub(super) entry_type: EventType, + /// e.g. Abhaltung + pub(super) detailed_entry_type: String, +} + +impl Event { + pub(crate) async fn store( + &self, + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + ) -> Result { + sqlx::query!( + r#"INSERT INTO calendar (id,room_code,start_at,end_at,stp_title_de,stp_title_en,stp_type,entry_type,detailed_entry_type) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (id) DO UPDATE SET + room_code = $2, + start_at = $3, + end_at = $4, + stp_title_de = $5, + stp_title_en = $6, + stp_type = $7, + entry_type = $8, + detailed_entry_type = $9"#, + self.id, + self.room_code, + self.start_at, + self.end_at, + self.stp_title_de, + self.stp_title_en, + self.stp_type, + self.entry_type.clone() as EventType, // see https://github.com/launchbadge/sqlx/issues/1004 => our type is not possible (?) + self.detailed_entry_type, + ).execute(&mut **tx).await + } +} + +#[derive(Serialize, Deserialize, Clone, Debug, sqlx::Type)] +#[sqlx(type_name = "EventType")] +#[sqlx(rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum EventType { + Lecture, + Exercise, + Exam, + Barred, + Other, +} diff --git a/server/main-api/src/entries/get.rs b/server/main-api/src/details.rs similarity index 98% rename from server/main-api/src/entries/get.rs rename to server/main-api/src/details.rs index c8d88ea88..1a4620979 100644 --- a/server/main-api/src/entries/get.rs +++ b/server/main-api/src/details.rs @@ -1,4 +1,4 @@ -use crate::models::DBRoomKeyAlias; +use crate::models::LocationKeyAlias; use crate::utils; use actix_web::{get, web, HttpResponse}; use log::error; @@ -52,7 +52,7 @@ pub async fn get_handler( async fn get_alias_and_redirect(conn: &PgPool, query: &str) -> Option<(String, String)> { let result = sqlx::query_as!( - DBRoomKeyAlias, + LocationKeyAlias, r#" SELECT DISTINCT key, visible_id, type FROM aliases diff --git a/server/main-api/src/entries/mod.rs b/server/main-api/src/entries/mod.rs deleted file mode 100644 index afa5f000c..000000000 --- a/server/main-api/src/entries/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod get; diff --git a/server/main-api/src/main.rs b/server/main-api/src/main.rs index 248ff0744..ed36155bf 100644 --- a/server/main-api/src/main.rs +++ b/server/main-api/src/main.rs @@ -1,3 +1,6 @@ +use std::collections::HashMap; +use std::error::Error; + use actix_cors::Cors; use actix_web::{get, middleware, web, App, HttpResponse, HttpServer}; use actix_web_prom::PrometheusMetricsBuilder; @@ -5,20 +8,22 @@ use log::{debug, error, info}; use sqlx::postgres::PgPoolOptions; use sqlx::prelude::*; use sqlx::PgPool; -use std::collections::HashMap; use structured_logger::async_json::new_writer; use structured_logger::Builder; -mod entries; +mod calendar; +mod details; mod maps; mod models; mod search; mod setup; mod utils; +type BoxedError = Box; + const MAX_JSON_PAYLOAD: usize = 1024 * 1024; // 1 MB -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct AppData { db: PgPool, } @@ -51,15 +56,15 @@ fn connection_string() -> String { } #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> Result<(), crate::BoxedError> { Builder::with_level("info") .with_target_writer("*", new_writer(tokio::io::stdout())) .init(); let uri = connection_string(); let pool = PgPoolOptions::new().connect(&uri).await?; - info!("setting up the database"); + #[cfg(not(feature = "skip_db_setup"))] setup::database::setup_database(&pool).await?; - info!("setting up meilisearch"); + #[cfg(not(feature = "skip_ms_setup"))] setup::meilisearch::setup_meilisearch().await?; debug!("setting up metrics"); @@ -89,8 +94,9 @@ async fn main() -> Result<(), Box> { .app_data(web::JsonConfig::default().limit(MAX_JSON_PAYLOAD)) .app_data(web::Data::new(AppData { db: pool.clone() })) .service(health_status_handler) + .service(calendar::calendar_handler) .service(web::scope("/api/preview").configure(maps::configure)) - .service(entries::get::get_handler) + .service(details::get_handler) .service(search::search_handler) }) .bind(std::env::var("BIND_ADDRESS").unwrap_or_else(|_| "0.0.0.0:3003".to_string()))? diff --git a/server/main-api/src/maps/fetch_tile.rs b/server/main-api/src/maps/fetch_tile.rs index efb680cd8..573288e92 100644 --- a/server/main-api/src/maps/fetch_tile.rs +++ b/server/main-api/src/maps/fetch_tile.rs @@ -109,7 +109,7 @@ impl FetchTileTask { async fn download_map_image( &self, file: &std::path::PathBuf, - ) -> Result> { + ) -> Result { let url = self.get_tileserver_url(); let res = reqwest::get(&url).await?.bytes().await?; diff --git a/server/main-api/src/maps/mod.rs b/server/main-api/src/maps/mod.rs index dd1f6628b..60e5d826f 100644 --- a/server/main-api/src/maps/mod.rs +++ b/server/main-api/src/maps/mod.rs @@ -4,7 +4,7 @@ mod overlay_text; use crate::maps::overlay_map::OverlayMapTask; use crate::maps::overlay_text::{OverlayText, CANTARELL_BOLD, CANTARELL_REGULAR}; -use crate::models::DBRoomEntry; +use crate::models::Location; use actix_web::{get, web, HttpResponse}; use cached::proc_macro::cached; use cached::SizedCache; @@ -31,13 +31,13 @@ async fn get_localised_data( conn: &PgPool, id: &str, should_use_english: bool, -) -> Result { +) -> Result { let result = if should_use_english { - sqlx::query_as!(DBRoomEntry, "SELECT * FROM en WHERE key = $1", id) + sqlx::query_as!(Location, "SELECT * FROM en WHERE key = $1", id) .fetch_all(conn) .await } else { - sqlx::query_as!(DBRoomEntry, "SELECT * FROM de WHERE key = $1", id) + sqlx::query_as!(Location, "SELECT * FROM de WHERE key = $1", id) .fetch_all(conn) .await }; @@ -66,7 +66,7 @@ async fn get_localised_data( option = true, convert = r#"{ _id.to_string() }"# )] -async fn construct_image_from_data(_id: &str, data: DBRoomEntry) -> Option> { +async fn construct_image_from_data(_id: &str, data: Location) -> Option> { let start_time = Instant::now(); let mut img = image::RgbaImage::new(1200, 630); @@ -87,7 +87,7 @@ fn wrap_image_in_response(img: &image::RgbaImage) -> Vec { w.into_inner() } -fn draw_bottom(data: &DBRoomEntry, img: &mut image::RgbaImage) { +fn draw_bottom(data: &Location, img: &mut image::RgbaImage) { // draw background white for x in 0..1200 { for y in 630 - 125..630 { diff --git a/server/main-api/src/maps/overlay_map.rs b/server/main-api/src/maps/overlay_map.rs index b51497ad0..c5c07fa5d 100644 --- a/server/main-api/src/maps/overlay_map.rs +++ b/server/main-api/src/maps/overlay_map.rs @@ -1,5 +1,5 @@ use crate::maps::fetch_tile::FetchTileTask; -use crate::models::DBRoomEntry; +use crate::models::Location; use futures::{stream::FuturesUnordered, StreamExt}; use log::warn; @@ -10,7 +10,7 @@ pub(crate) struct OverlayMapTask { } impl OverlayMapTask { - pub fn with(entry: &DBRoomEntry) -> Self { + pub fn with(entry: &Location) -> Self { let zoom = match entry.r#type.as_str() { "campus" => 14, "area" | "site" => 15, diff --git a/server/main-api/src/models.rs b/server/main-api/src/models.rs index 40b0b6239..2f099ae12 100644 --- a/server/main-api/src/models.rs +++ b/server/main-api/src/models.rs @@ -1,7 +1,10 @@ +use chrono::{DateTime, Utc}; + #[derive(Debug, Clone)] -pub struct DBRoomEntry { +pub struct Location { pub key: String, pub name: String, + pub last_calendar_scrape_at: Option>, pub tumonline_room_nr: Option, pub r#type: String, pub type_common_name: String, @@ -11,7 +14,7 @@ pub struct DBRoomEntry { } #[derive(Debug, Clone)] -pub struct DBRoomKeyAlias { +pub struct LocationKeyAlias { pub key: String, pub visible_id: String, pub r#type: String, diff --git a/server/main-api/src/setup/database/alias.rs b/server/main-api/src/setup/database/alias.rs index a546d6240..c7117d6ea 100644 --- a/server/main-api/src/setup/database/alias.rs +++ b/server/main-api/src/setup/database/alias.rs @@ -104,7 +104,7 @@ impl Alias { pub(crate) async fn load_all_to_db( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, -) -> Result<(), Box> { +) -> Result<(), crate::BoxedError> { let cdn_url = std::env::var("CDN_URL").unwrap_or_else(|_| "https://nav.tum.de/cdn".to_string()); let raw_aliase = reqwest::get(format!("{cdn_url}/api_data.json")) .await? diff --git a/server/main-api/src/setup/database/data.rs b/server/main-api/src/setup/database/data.rs index f95a98ca0..3c9c91b75 100644 --- a/server/main-api/src/setup/database/data.rs +++ b/server/main-api/src/setup/database/data.rs @@ -149,7 +149,7 @@ impl DelocalisedValues { } pub(crate) async fn load_all_to_db( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, -) -> Result<(), Box> { +) -> Result<(), crate::BoxedError> { let start = Instant::now(); let cdn_url = std::env::var("CDN_URL").unwrap_or_else(|_| "https://nav.tum.de/cdn".to_string()); let tasks = reqwest::get(format!("{cdn_url}/api_data.json")) diff --git a/server/main-api/src/setup/database/mod.rs b/server/main-api/src/setup/database/mod.rs index e197fdb43..223b21c3d 100644 --- a/server/main-api/src/setup/database/mod.rs +++ b/server/main-api/src/setup/database/mod.rs @@ -3,7 +3,8 @@ mod data; use log::info; -pub(crate) async fn setup_database(pool: &sqlx::PgPool) -> Result<(), Box> { +pub(crate) async fn setup_database(pool: &sqlx::PgPool) -> Result<(), crate::BoxedError> { + info!("setting up the database"); sqlx::migrate!("./migrations").run(pool).await?; info!("migrations complete"); @@ -14,7 +15,7 @@ pub(crate) async fn setup_database(pool: &sqlx::PgPool) -> Result<(), Box, -) -> Result<(), Box> { +) -> Result<(), crate::BoxedError> { info!("deleting old data"); sqlx::query!("DELETE FROM aliases") .execute(&mut **tx) diff --git a/server/main-api/src/setup/meilisearch.rs b/server/main-api/src/setup/meilisearch.rs index 55a4c47b1..c0dbeeda3 100644 --- a/server/main-api/src/setup/meilisearch.rs +++ b/server/main-api/src/setup/meilisearch.rs @@ -43,7 +43,8 @@ async fn wait_for_healthy(client: &Client) { } } -pub(crate) async fn setup_meilisearch() -> Result<(), Box> { +pub(crate) async fn setup_meilisearch() -> Result<(), crate::BoxedError> { + info!("setting up meilisearch"); let start = std::time::Instant::now(); let ms_url = std::env::var("MIELI_URL").unwrap_or_else(|_| "http://localhost:7700".to_string()); info!("connecting to Meilisearch at {ms_url}", ms_url = ms_url); diff --git a/server/main-api/src/setup/mod.rs b/server/main-api/src/setup/mod.rs index d3468b530..e77362fd1 100644 --- a/server/main-api/src/setup/mod.rs +++ b/server/main-api/src/setup/mod.rs @@ -1,2 +1,5 @@ +#[cfg(not(feature = "skip_db_setup"))] pub(crate) mod database; + +#[cfg(not(feature = "skip_ms_setup"))] pub(crate) mod meilisearch; diff --git a/webclient/Dockerfile b/webclient/Dockerfile index 81da14a9a..05f280598 100644 --- a/webclient/Dockerfile +++ b/webclient/Dockerfile @@ -19,4 +19,4 @@ COPY --from=build-stage /dist /app EXPOSE 3000 HEALTHCHECK CMD curl --fail localhost:3000/health || exit 1 -CMD sh -c "sed -i 's|TILESERVER_URL|${TILESERVER_URL}|g' /etc/nginx/nginx.conf && sed -i 's|CDN_URL|${CDN_URL}|g' /etc/nginx/nginx.conf && sed -i 's|MAIN_API_URL|${MAIN_API_URL}|g' /etc/nginx/nginx.conf && sed -i 's|FEEDBACK_API_URL|${FEEDBACK_API_URL}|g' /etc/nginx/nginx.conf && sed -i 's|CALENDAR_API_URL|${CALENDAR_API_URL}|g' /etc/nginx/nginx.conf && nginx -g 'daemon off;'" +CMD sh -c "sed -i 's|TILESERVER_URL|${TILESERVER_URL}|g' /etc/nginx/nginx.conf && sed -i 's|CDN_URL|${CDN_URL}|g' /etc/nginx/nginx.conf && sed -i 's|MAIN_API_URL|${MAIN_API_URL}|g' /etc/nginx/nginx.conf && sed -i 's|FEEDBACK_API_URL|${FEEDBACK_API_URL}|g' /etc/nginx/nginx.conf && nginx -g 'daemon off;'" diff --git a/webclient/README.md b/webclient/README.md index 61fec2352..8b88465e3 100644 --- a/webclient/README.md +++ b/webclient/README.md @@ -55,10 +55,11 @@ pnpm run dev pnpm run build ``` -### Linting with [ESLint](https://eslint.org/) +### Linting with [ESLint](https://eslint.org/) and formatting via prettier ```sh pnpm run lint +pnpm run format ``` ### Update the API's type definitions @@ -68,6 +69,7 @@ From the folder of this README, run: ```sh npx openapi-typescript ../openapi.yaml --output ./src/api_types/index.ts --export-type --immutable-types --support-array-length pnpm run lint +pnpm run format ``` ## Build files & Serving release build diff --git a/webclient/nginx.conf b/webclient/nginx.conf index 57ae3704b..7dfc5b0ec 100644 --- a/webclient/nginx.conf +++ b/webclient/nginx.conf @@ -70,9 +70,6 @@ http { location /api/feedback { proxy_pass FEEDBACK_API_URL; } - location /api/calendar { - proxy_pass CALENDAR_API_URL; - } location /cdn { proxy_pass CDN_URL; } diff --git a/webclient/src/api_types/index.ts b/webclient/src/api_types/index.ts index 0fc0dc19d..a21db36d1 100644 --- a/webclient/src/api_types/index.ts +++ b/webclient/src/api_types/index.ts @@ -31,19 +31,21 @@ export type paths = { }; "/api/calendar/{id}": { /** - * Get a entry-preview - * @description This returns the a 1200x630px preview for the entry (room/building/..). + * Get a + * @description Retrieves calendar entries for a specific `id` within the requested time span. + * The time span is defined by the `start_after` and `end_before` query parameters. + * Ensure to provide valid date-time formats for these parameters. * - * This is usefull for implementing custom OpenGraph images for detail previews. + * If successful, returns additional entries in the requested time span. */ get: operations["calendar"]; }; "/api/preview/{id}": { /** * Get a entry-preview - * @description This returns the a 1200x630px preview for the entry (room/building/..). + * @description This returns a 1200x630px preview for the location (room/building/..). * - * This is usefull for implementing custom OpenGraph images for detail previews. + * This is usefully for implementing custom OpenGraph images for detail previews. */ get: operations["previews"]; }; @@ -122,13 +124,6 @@ export type paths = { */ get: operations["feedback-health"]; }; - "/api/calendar/status": { - /** - * calendar-API healthcheck - * @description If this endpoint does not return 200, the API is experiencing a catastrophic outage. Should never happen. - */ - get: operations["calendar-health"]; - }; "/cdn/health": { /** * CDN healthcheck @@ -948,18 +943,20 @@ export type operations = { }; }; /** - * Get a entry-preview - * @description This returns the a 1200x630px preview for the entry (room/building/..). + * Get a + * @description Retrieves calendar entries for a specific `id` within the requested time span. + * The time span is defined by the `start_after` and `end_before` query parameters. + * Ensure to provide valid date-time formats for these parameters. * - * This is usefull for implementing custom OpenGraph images for detail previews. + * If successful, returns additional entries in the requested time span. */ calendar: { parameters: { query: { /** @description The first allowed time the calendar would like to display */ - start: string; + start_after: string; /** @description The last allowed time the calendar would like to display */ - end: string; + end_before: string; }; path: { /** @description string you want to search for */ @@ -989,9 +986,9 @@ export type operations = { }; /** * Get a entry-preview - * @description This returns the a 1200x630px preview for the entry (room/building/..). + * @description This returns a 1200x630px preview for the location (room/building/..). * - * This is usefull for implementing custom OpenGraph images for detail previews. + * This is usefully for implementing custom OpenGraph images for detail previews. */ previews: { parameters: { @@ -1352,24 +1349,6 @@ export type operations = { }; }; }; - /** - * calendar-API healthcheck - * @description If this endpoint does not return 200, the API is experiencing a catastrophic outage. Should never happen. - */ - "calendar-health": { - responses: { - /** @description Ok */ - 200: { - content: { - readonly "text/plain": string; - }; - }; - /** @description Service Unavailable */ - 503: { - content: never; - }; - }; - }; /** * CDN healthcheck * @description If this endpoint does not return 200, the CDN is experiencing a catastrophic outage. Should never happen.