[Cal-12] Replacement of internal API calls with the CDN

server/Cargo.toml

@@ -21,4 +21,4 @@ serde = { version = "1.0.163", features = ["derive"] }
 serde_json = "1.0.96"
 
 # testing
-pretty_assertions = "1.3.0"
+pretty_assertions = "1.3.0"

server/calendar/Cargo.toml

@@ -42,8 +42,9 @@ prometheus = { version = "0.13.3", features = ["default", "push"] }
 rand = "0.8.5"
 futures = "0.3.28"
 rustls = "0.21.1"
-reqwest = { version = "0.11.18", features = ["rustls"] }
+reqwest = { version = "0.11.18", features = ["rustls", "json"] }
 minidom = "0.15.2"
+regex = "1.8.3"
 
 [dev-dependencies]
-pretty_assertions.workspace = true
+pretty_assertions.workspace = true

server/calendar/migrations/2023-03-23-144107_create_calendar/up.sql

@@ -20,5 +20,5 @@ CREATE TABLE calendar (
     status_id               TEXT NOT NULL,
     status                  TEXT NOT NULL,
     comment                 TEXT NOT NULL,
-    last_scrape             timestamp NOT NULL
+    last_scrape             TIMESTAMP NOT NULL
 );

server/calendar/migrations/2023-06-03-173046_new_table_rooms/down.sql

@@ -0,0 +1,2 @@
+-- This file should undo anything in `down.sql`
+DROP TABLE IF EXISTS rooms;

server/calendar/migrations/2023-06-03-173046_new_table_rooms/up.sql

@@ -0,0 +1,8 @@
+-- Your SQL goes here
+CREATE TABLE IF NOT EXISTS rooms (
+    key                   TEXT PRIMARY KEY NOT NULL,
+    tumonline_org_id      INTEGER NOT NULL,
+    tumonline_calendar_id INTEGER NOT NULL,
+    tumonline_room_id     INTEGER NOT NULL,
+    last_scrape           TIMESTAMP NOT NULL
+);

server/calendar/migrations/2023-06-03-173522_removal_of_tumonline_id/down.sql

@@ -0,0 +1,3 @@
+-- Your SQL goes here
+ALTER TABLE calendar
+ADD COLUMN tumonline_id INTEGER NOT NULL DEFAULT 0;

server/calendar/migrations/2023-06-03-173522_removal_of_tumonline_id/up.sql

@@ -0,0 +1,3 @@
+-- This file should undo anything in `down.sql`
+ALTER TABLE calendar
+DROP COLUMN tumonline_id;

server/calendar/src/calendar.rs

@@ -17,33 +17,57 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(calendar_handler);
 }
 
+fn get_calendar_url(requested_key: &str, conn: &mut PgConnection) -> QueryResult<String> {
+    use crate::schema::rooms::dsl::*;
+    let room = rooms
+        .filter(key.eq(requested_key))
+        .first::<crate::models::Room>(conn)?;
+    Ok(format!(
+        "https://campus.tum.de/tumonline/wbKalender.wbRessource?pResNr={}",
+        room.tumonline_calendar_id
+    ))
+}
+
+fn get_entries(
+    requested_key: &str,
+    args: CalendarQueryArgs,
+    conn: &mut PgConnection,
+) -> QueryResult<Vec<XMLEvent>> {
+    use crate::schema::calendar::dsl::*;
+    calendar
+        .filter(key.eq(&requested_key))
+        .filter(dtstart.ge(&args.start))
+        .filter(dtend.le(&args.end))
+        .load::<XMLEvent>(conn)
+}
+
 #[get("/{id}")]
 pub async fn calendar_handler(
     params: web::Path<String>,
     web::Query(args): web::Query<CalendarQueryArgs>,
 ) -> HttpResponse {
     let id = params.into_inner();
     let conn = &mut utils::establish_connection();
-    use crate::schema::calendar::dsl::*;
-    let results = calendar
-        .filter(key.eq(&id))
-        .filter(dtstart.ge(&args.start))
-        .filter(dtend.le(&args.end))
-        .load::<XMLEvent>(conn);
-    match results {
-        Ok(results) => {
+    let results = get_entries(&id, args, conn);
+    let calendar_url = get_calendar_url(&id, conn);
+    match (results, calendar_url) {
+        (Ok(results), Ok(calendar_url)) => {
             let last_sync = results.iter().map(|e| e.last_scrape).min().unwrap();
-            let tumonline_room_number = results.iter().map(|e| e.tumonline_id).next().unwrap();
-            let calendar_url = format!("https://campus.tum.de/tumonline/wbKalender.wbRessource?pResNr={tumonline_room_number}");
             let events = results.into_iter().map(Event::from).collect();
             HttpResponse::Ok().json(Events {
                 events,
                 last_sync,
                 calendar_url,
             })
         }
-        Err(e) => {
-            error!("Error loading calendar: {e:?}");
+        (Err(e), _) => {
+            error!("Error loading calendar entries: {e:?}");
+            HttpResponse::InternalServerError()
+                .content_type("text/plain")
+                .body("Error loading calendar")
+        }
+        (_, Err(e)) => {
+            error!("Error loading calendar_url: {e:?}");
             HttpResponse::InternalServerError()
                 .content_type("text/plain")
                 .body("Error loading calendar")

server/calendar/src/models.rs

@@ -6,7 +6,6 @@ use diesel::Insertable;
 #[diesel(table_name = crate::schema::calendar)]
 pub struct XMLEvent {
     pub key: String,
-    pub tumonline_id: i32,
     pub dtstart: NaiveDateTime,
     pub dtend: NaiveDateTime,
     pub dtstamp: NaiveDateTime,
@@ -28,3 +27,13 @@ pub struct XMLEvent {
     pub comment: String,
     pub last_scrape: NaiveDateTime,
 }
+
+#[derive(Insertable, Queryable, AsChangeset, Clone)]
+#[diesel(table_name = crate::schema::rooms)]
+pub struct Room {
+    pub key: String,
+    pub tumonline_org_id: i32,
+    pub tumonline_calendar_id: i32,
+    pub tumonline_room_id: i32,
+    pub last_scrape: NaiveDateTime,
+}

server/calendar/src/schema.rs

@@ -3,7 +3,6 @@
 diesel::table! {
     calendar (single_event_id) {
         key -> Varchar,
-        tumonline_id -> Int4,
         dtstart -> Timestamp,
         dtend -> Timestamp,
         dtstamp -> Timestamp,
@@ -26,3 +25,15 @@ diesel::table! {
         last_scrape -> Timestamp,
     }
 }
+
+diesel::table! {
+    rooms (key) {
+        key -> Text,
+        tumonline_org_id -> Int4,
+        tumonline_calendar_id -> Int4,
+        tumonline_room_id -> Int4,
+        last_scrape -> Timestamp,
+    }
+}
+
+diesel::allow_tables_to_appear_in_same_query!(calendar, rooms,);

server/calendar/src/scrape_task/main_api_connector.rs

@@ -1,46 +1,105 @@
-use log::error;
+use chrono::{NaiveDateTime, Utc};
+use diesel::PgConnection;
+use log::{error, info};
+use regex::Regex;
+use serde::Deserialize;
+use std::collections::HashMap;
 
 fn api_url_from_env() -> Option<String> {
-    let main_api_addr = std::env::var("API_SVC_SERVICE_HOST").ok()?;
-    let main_api_port = std::env::var("API_SVC_SERVICE_PORT_HTTP").ok()?;
+    let main_api_addr = std::env::var("CDN_SVC_SERVICE_HOST").ok()?;
+    let main_api_port = std::env::var("CDN_SVC_SERVICE_PORT_HTTP").ok()?;
 
     Some(format!(
-        "http://{main_api_addr}:{main_api_port}/internal/list/ids_with_calendar"
+        "http://{main_api_addr}:{main_api_port}/cdn/api_data.json"
     ))
 }
 
+#[derive(Deserialize, Debug)]
 pub struct ReducedRoom {
-    pub key: String,
-    pub tumonline_room_nr: i32,
+    props: ReducedRoomProps,
 }
 
-pub async fn get_all_ids() -> Vec<ReducedRoom> {
-    // returns all (key, tumonline_room_nr) from the main-api
-    let url = api_url_from_env()
-        .unwrap_or_else(|| "https://nav.tum.de/internal/list/ids_with_calendar".to_string());
+#[derive(Deserialize, Debug)]
+pub struct ReducedRoomProps {
+    calendar_url: Option<String>, //tumonline_room_nr and calendar_url are sometimes not present, but only ever both
+    tumonline_room_nr: Option<i32>,
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct Room {
+    pub sap_id: String,
+    pub tumonline_org_id: i32,
+    pub tumonline_calendar_id: i32,
+    pub tumonline_room_id: i32,
+}
+
+impl Room {
+    fn from((key, room): (String, ReducedRoom)) -> Option<Room> {
+        let url = room.props.calendar_url?;
+        let regex = Regex::new(r".*cOrg=(?P<org>\d+)&cRes=(?P<cal>\d+)\D.*").unwrap();
+        let captures = regex.captures(&url)?;
+        Some(Room {
+            sap_id: key,
+            tumonline_org_id: captures.name("org")?.as_str().parse().ok()?,
+            tumonline_calendar_id: captures.name("cal")?.as_str().parse().ok()?,
+            tumonline_room_id: room.props.tumonline_room_nr?,
+        })
+    }
+}
+
+pub async fn get_all_ids() -> Vec<Room> {
+    let url =
+        api_url_from_env().unwrap_or_else(|| "https://nav.tum.de/cdn/api_data.json".to_string());
     let res = reqwest::get(&url).await;
-    let text = match res {
-        Ok(res) => res.text().await,
+    let rooms = match res {
+        Ok(res) => res.json::<HashMap<String, ReducedRoom>>().await,
         Err(e) => {
             error!("Failed to contact main-api at {url}: {e:#?}");
             return vec![];
         }
     };
-    match text {
-        Ok(ids) => {
-            serde_json::from_slice::<Vec<(String, i32)>>(ids.as_bytes()).unwrap_or_else(|_| {
-                panic!("JSON-parsing error, make sure the schema matches. Got {ids} from {url}")
-            })
-        }
-        Err(e) => {
-            error!("Failed to process text get all ids from api at {url}: {e:#?}");
-            vec![]
-        }
-    }
-    .into_iter()
-    .map(|(key, tumonline_room_nr)| ReducedRoom {
-        key,
-        tumonline_room_nr,
-    })
-    .collect()
+    let rooms: Vec<Room> = match rooms {
+        Ok(rooms) => rooms.into_iter().flat_map(Room::from).collect(),
+        Err(e) => panic!("Failed to parse main-api response: {e:#?}"),
+    };
+    let start_time = Utc::now().naive_utc();
+    let conn = &mut crate::utils::establish_connection();
+    store_in_db(conn, &rooms, &start_time);
+    delete_stale_results(conn, start_time);
+    rooms
+}
+
+fn store_in_db(conn: &mut PgConnection, rooms_to_store: &[Room], start_time: &NaiveDateTime) {
+    info!("Storing {} rooms in database", rooms_to_store.len());
+    use crate::schema::rooms::dsl::*;
+    use diesel::prelude::*;
+    rooms_to_store
+        .iter()
+        .map(|room| crate::models::Room {
+            key: room.sap_id.clone(),
+            tumonline_org_id: room.tumonline_org_id,
+            tumonline_calendar_id: room.tumonline_calendar_id,
+            tumonline_room_id: room.tumonline_room_id,
+            last_scrape: *start_time,
+        })
+        .for_each(|room| {
+            let res = diesel::insert_into(rooms)
+                .values(&room)
+                .on_conflict(key)
+                .do_update()
+                .set(&room)
+                .execute(conn);
+            if let Err(e) = res {
+                error!("Error inserting into database: {e:?}");
+            }
+        });
+}
+fn delete_stale_results(conn: &mut PgConnection, start_time: NaiveDateTime) {
+    info!("Deleting stale rooms from the database");
+    use crate::schema::rooms::dsl::*;
+    use diesel::prelude::*;
+    diesel::delete(rooms)
+        .filter(last_scrape.lt(start_time))
+        .execute(conn)
+        .expect("Failed to delete stale rooms");
 }