Merge branch 'main' into connectum-based-scraper

Author: CommanderStorm

.editorconfig

@@ -10,3 +10,6 @@ insert_final_newline = true
 max_line_length = 120
 indent_size = 2
 indent_style = space
+
+[*.py]
+indent_size = 4

.env

@@ -12,3 +12,7 @@ JWT_KEY=CHANGE_ME
 
 # main api
 MEILI_MASTER_KEY=CHANGE_ME
+
+# calendar
+CONNECTUM_OAUTH_CLIENT_SECRET=CHANGE_ME
+CONNECTUM_OAUTH_CLIENT_ID=CHANGE_ME

.github/workflows/update-data.yml

@@ -30,8 +30,16 @@ jobs:
           export PYTHONPATH=$PYTHONPATH:..
           python3 main.py
           ls -lah results
+          sed -i 's/Bestelmeyer S\u00fcd/Zentralgeb\u00e4ude 2/g' results/buildings_roomfinder.json
+          sed -i 's/Bestelmeyer Nord/Zentralgeb\u00e4ude 7/g' results/buildings_roomfinder.json
+          sed -i 's/Bestelmeyer S\u00fcd/Zentralgeb\u00e4ude 2/g' results/maps_roomfinder.json
+          sed -i 's/Bestelmeyer Nord/Zentralgeb\u00e4ude 7/g' results/maps_roomfinder.json
+          sed -i 's/Bestelmeyer S\u00fcd/Zentralgeb\u00e4ude 2/g' results/rooms_roomfinder.json
+          sed -i 's/Bestelmeyer Nord/Zentralgeb\u00e4ude 7/g' results/rooms_roomfinder.json
         env:
           TQDM_MININTERVAL: 100
+          CONNECTUM_OAUTH_CLIENT_ID: ${{ secrets.CONECTUM_OAUTH_CLIENT_ID }}
+          CONNECTUM_OAUTH_CLIENT_SECRET: ${{ secrets.CONECTUM_OAUTH_CLIENT_SECRET }}
         working-directory: data/external
       - name: Fix linting errors
         continue-on-error: true

data/compile.py

@@ -18,7 +18,7 @@
     structure,
     tumonline,
 )
-from utils import DEBUG_MODE, setup_logging
+from utils import DEV_MODE, setup_logging
 
 
 # pylint: disable=too-many-locals,too-many-statements
@@ -130,7 +130,7 @@ def main() -> None:
 
 
 if __name__ == "__main__":
-    setup_logging(level=logging.DEBUG if DEBUG_MODE else logging.INFO)
+    setup_logging(level=logging.DEBUG if DEV_MODE else logging.INFO)
 
     # Pillow prints all imported modules to the debug stream
     logging.getLogger("PIL").setLevel(logging.INFO)

data/processors/images.py

@@ -9,13 +9,14 @@
 from typing import Any, NamedTuple, TypeVar
 
 import pydantic
-import utils
 import yaml
-from external.models.common import PydanticConfiguration
 from PIL import Image
 from pydantic import Field
 from pydantic.networks import HttpUrl
 
+import utils
+from external.models.common import PydanticConfiguration
+
 
 class UrlStr(PydanticConfiguration):
     text: str
@@ -219,9 +220,6 @@ def _extract_offsets(_id: str, _index: int, img_path: Path, img_sources: dict[st
 
 def _get_hash_lut() -> dict[str, str]:
     """Get a lookup table for the hash of the image files content and offset if present"""
-    if not DEV_MODE:
-        return {}
-    logging.info("Since GIT_COMMIT_SHA is unset, we assume this is acting in In Dev mode.")
     logging.info("Only files, with sha256(file-content)_sha256(offset) not present in the .hash_lut.json will be used")
     if HASH_LUT.is_file():
         with open(HASH_LUT, encoding="utf-8") as file:
@@ -268,13 +266,12 @@ def resize_and_crop() -> None:
         for img_path in IMAGE_SOURCE.glob("*.webp"):
             _id, _index = parse_image_filename(img_path.name)
             offsets = _extract_offsets(_id, _index, img_path, img_sources)
+            actual_hash = _gen_file_hash(img_path, offsets)
+            if actual_hash == expected_hashes_lut.get(img_path.name, ""):
+                continue  # skip this image, since it (and its offsets) have not changed
             if DEV_MODE:
-                actual_hash = _gen_file_hash(img_path, offsets)
-                if actual_hash == expected_hashes_lut.get(img_path.name, ""):
-                    continue  # skip this image, since it (and its offsets) have not changed
                 logging.debug(f"Image '{img_path.name}' has changed, resizing and cropping...")
             executor.submit(_refresh_for_all_resolutions, RefreshResolutionOrder(img_path, offsets))
-    if DEV_MODE:
-        _save_hash_lut(img_sources)
-        resize_and_crop_time = time.time() - start_time
-        logging.info(f"Resize and crop took {resize_and_crop_time:.2f}s")
+    _save_hash_lut(img_sources)
+    resize_and_crop_time = time.time() - start_time
+    logging.info(f"Resize and crop took {resize_and_crop_time:.2f}s")

data/processors/sitemap.py

@@ -9,8 +9,6 @@
 import requests
 from defusedxml import ElementTree as defusedET
 
-from utils import DEBUG_MODE
-
 OLD_DATA_URL = "https://nav.tum.de/cdn/api_data.json"
 
 
@@ -35,10 +33,6 @@ class SimplifiedSitemaps(TypedDict):
 
 def generate_sitemap() -> None:
     """Generate a sitemap that diffs changes since to the currently online data"""
-    if DEBUG_MODE:
-        logging.info("Skipping sitemap generation in Dev Mode (GIT_COMMIT_SHA is unset)")
-        return
-
     # Load exported data. This function is intentionally not using the data object
     # directly, but re-parsing the output file instead, because the export not
     # export all fields. This way we're also guaranteed to have the same types

data/utils.py

@@ -15,7 +15,7 @@
 with open(TRANSLATION_BUFFER_PATH, encoding="utf-8") as yaml_file:
     TRANSLATION_BUFFER = yaml.load(yaml_file)
 
-DEBUG_MODE = "GIT_COMMIT_SHA" not in os.environ
+DEV_MODE = "GIT_COMMIT_SHA" not in os.environ
 
 
 class TranslatableStr(dict):

deployment/README.md

@@ -24,7 +24,8 @@ On every of these nodes we run the following supporting services:
 
 - [traefik](https://traefik.io/) as a reverse proxy
 - [cert-manager](https://cert-manager.io/) to manage the https-certificate via [Let's Encrypt](https://letsencrypt.org/)
-- [prometeus](https://prometheus.io/), [allertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) and [grafana](https://grafana.com/) for monitoring purposes
+- [prometeus](https://prometheus.io/), [allertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/)
+  and [grafana](https://grafana.com/) for monitoring purposes
 - [argocd](https://argo-cd.readthedocs.io/) to manage the deployments
 - [uptime-kuma](https://github.com/louislam/uptime-kuma) to monitor the uptime of the services
 
@@ -46,11 +47,16 @@ Namely:
 
 - we don't publish our `GITHUB_TOKEN` to git. (used to pass feedback from the webclient to github)
 - we don't publish the `JWT_KEY` to git. (used to generate tokens to ratelimt feedback creation)
-- we don't publish the `MEILI_MASTER_KEY` to git. (used as aditional layer of network hardening between the webclient and the server)
+- we don't publish the `MEILI_MASTER_KEY` to git. (used as aditional layer of network hardening between the webclient
+  and the server)
+- we don't publish the `CONNECTUM_OAUTH_CLIENT_{SECRET,ID}` to git. (used to connect to the calendar and possibly
+  further apis in the future)
 
 Deployment happen automatically on push to main, or on push to a PR.
-For PRs we only execute this deployment request, if the autor is a member of the `@TUM-Dev/navigatum`-group or a member authorises this PR to run actions.
-The reasoning is, that we don't want strangers to be able to fork our project, change the deployment to something malicious and make us deploy it.
+For PRs we only execute this deployment request, if the autor is a member of the `@TUM-Dev/navigatum`-group or a member
+authorises this PR to run actions.
+The reasoning is, that we don't want strangers to be able to fork our project, change the deployment to something
+malicious and make us deploy it.
 
 ### ansible
 

deployment/k3s/templates/deployments/server-deployment.yaml

@@ -81,14 +81,14 @@ spec:
         - name: mieli-search
           image: {{ $.Values.mielisearch.image.repository }}:{{ $.Values.mielisearch.image.tag }}
           imagePullPolicy: IfNotPresent
-          {{ if $.Values.server.MEILI_MASTER_KEY }}
           envFrom:
             - secretRef:
-                name: server-api-keys # MEILI_MASTER_KEY
+                name: server-api-keys # GITHUB_TOKEN, JWT_KEY, CONNECTUM_OAUTH_CLIENT_{SECRET,ID}
+            - secretRef:
+                name: server-search-key # MEILI_MASTER_KEY
           env:
             - name: MEILI_ENV
               value: production
-          {{ end }}
           ports:
             - containerPort: 7700
               name: http
@@ -117,7 +117,20 @@ spec:
               port: http
             periodSeconds: 10
             failureThreshold: 2
-  {{ if $.Values.server.MEILI_MASTER_KEY }}
+{{ if $.Values.server.MEILI_MASTER_KEY }}
+---
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: server-search-key
+  namespace: {{ $.Values.namespace }}
+  labels:
+    app.kubernetes.io/part-of: navigatum
+    app.kubernetes.io/name: api
+data:
+  MEILI_MASTER_KEY: {{ $.Values.server.MEILI_MASTER_KEY }}
+{{ end }}
 ---
 apiVersion: v1
 kind: Secret
@@ -129,7 +142,8 @@ metadata:
     app.kubernetes.io/part-of: navigatum
     app.kubernetes.io/name: api
 data:
-  {{ if $.Values.server.MEILI_MASTER_KEY }}
   MEILI_MASTER_KEY: {{ $.Values.server.MEILI_MASTER_KEY }}
-  {{ end }}
-  {{ end }}
+  GITHUB_TOKEN: {{ $.Values.server.GITHUB_TOKEN }}
+  JWT_KEY: {{ $.Values.server.JWT_KEY }}
+  CONNECTUM_OAUTH_CLIENT_SECRET: {{ $.Values.server.CONNECTUM_OAUTH_CLIENT_SECRET }}
+  CONNECTUM_OAUTH_CLIENT_ID: {{ $.Values.server.CONNECTUM_OAUTH_CLIENT_ID }}

deployment/k3s/values.yaml

@@ -2,6 +2,8 @@ namespace: navigatum
 tag: main
 url: nav.tum.de
 server:
+  CONNECTUM_OAUTH_CLIENT_SECRET: ""
+  CONNECTUM_OAUTH_CLIENT_ID: ""
   MEILI_MASTER_KEY: ""
   GITHUB_TOKEN: ""
   JWT_KEY: ""

server/Cargo.lock

@@ -31,7 +31,7 @@ actix-web = { version = "4.5.1", default-features = false, features = ["macros",
 actix-cors = "0.7.0"
 rustls = { version = "0.23.5", default-features = false, features = ["ring"] } # the aws' fips complient libary has weird bingen issues which require deeper looking into
 
-cached = { version = "0.51.0", features = ["default", "async", "tokio", "disk_store"] }
+cached = { version = "0.51.1", features = ["default", "async", "tokio", "disk_store"] }
 futures = "0.3.30"
 unicode-truncate = "1.0.0"
 

server/main-api/Cargo.toml

@@ -42,7 +42,7 @@ impl CalendarEntryFetcher for APIRequestor {
             .secret()
             .clone();
         let url = format!(
-            "https://review.campus.tum.de/RSYSTEM/co/connectum/api/rooms/{tumonline_id}/calendars"
+            "https://campus.tum.de/tumonline/co/connectum/api/rooms/{tumonline_id}/calendars"
         );
 
         let events: Vec<Event> = self
@@ -109,20 +109,20 @@ impl APIRequestor {
         Ok(())
     }
     async fn fetch_oauth_token(&self) -> Result<BasicTokenResponse, crate::BoxedError> {
-        let client_id = env::var("TUMONLINE_OAUTH_CLIENT_ID")
+        let client_id = env::var("CONNECTUM_OAUTH_CLIENT_ID")
         .map_err(|e| {
-            error!("TUMONLINE_OAUTH_CLIENT_ID needs to be set: {e:?}");
-            io::Error::other("please configure the environment variable TUMONLINE_OAUTH_CLIENT_ID to use this endpoint")
+            error!("CONNECTUM_OAUTH_CLIENT_ID needs to be set: {e:?}");
+            io::Error::other("please configure the environment variable CONNECTUM_OAUTH_CLIENT_ID to use this endpoint")
         })?;
-        let client_secret = env::var("TUMONLINE_OAUTH_CLIENT_SECRET")
+        let client_secret = env::var("CONNECTUM_OAUTH_CLIENT_SECRET")
             .map_err(|e| {
-                error!("TUMONLINE_OAUTH_CLIENT_SECRET needs to be set: {e:?}");
-                io::Error::other("please configure the environment variable TUMONLINE_OAUTH_CLIENT_SECRET to use this endpoint")
+                error!("CONNECTUM_OAUTH_CLIENT_SECRET needs to be set: {e:?}");
+                io::Error::other("please configure the environment variable CONNECTUM_OAUTH_CLIENT_SECRET to use this endpoint")
             })?;
 
-        // for urls see https://review.campus.tum.de/RSYSTEM/co/public/sec/auth/realms/CAMPUSonline/.well-known/openid-configuration
-        let auth_url = Url::parse("https://review.campus.tum.de/RSYSTEM/co/public/sec/auth/realms/CAMPUSonline/protocol/openid-connect/auth")?;
-        let token_url = Url::parse("https://review.campus.tum.de/RSYSTEM/co/public/sec/auth/realms/CAMPUSonline/protocol/openid-connect/token")?;
+        // for urls see https://campus.tum.de/tumonline/co/public/sec/auth/realms/CAMPUSonline/.well-known/openid-configuration
+        let auth_url = Url::parse("https://campus.tum.de/tumonline/co/public/sec/auth/realms/CAMPUSonline/protocol/openid-connect/auth")?;
+        let token_url = Url::parse("https://campus.tum.de/tumonline/co/public/sec/auth/realms/CAMPUSonline/protocol/openid-connect/token")?;
 
         let token = BasicClient::new(
             ClientId::new(client_id),