From 93e6a8046aefa7e5ed0f9c27041a6551768b45b0 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Tue, 7 May 2024 17:16:10 +0200 Subject: [PATCH 1/2] JSON based generated database fields instead of imports (#1155) migrated to using JSONB columns to store data instead of string columns --- ...f617a646c40ec0e2f614d035948f240c8b245.json | 15 ++ ...5860cfad2fc568d61ce6648ca1ba7770db617.json | 15 ++ ...b37f497aeb87f11eeafa13eadd4fefda706b5.json | 21 --- ...3d161757d7ce6fcee59c23c23abbf45e803cb.json | 36 +++-- ...0de7a221ce7d9bb8378028c695e1b665c55c5.json | 2 +- ...d9b32f08c86626db9f4d63249dfc2cd77b80c.json | 2 +- ...d1a495534817fb39a9e25f6720267c6025965.json | 36 +++-- ...9ffdcaa2465cb27b510f9e50682f15980c45f.json | 15 ++ ...1af9a99868a7dee15c2f32ccca3c9b1c8b2b6.json | 21 --- ...430_fix-calendar-id-not-extracted.down.sql | 43 ++++++ ...24430_fix-calendar-id-not-extracted.up.sql | 34 +++++ .../main-api/src/calendar/fetch/connectum.rs | 42 ++++-- server/main-api/src/calendar/fetch/db.rs | 3 +- server/main-api/src/calendar/mod.rs | 9 +- server/main-api/src/details.rs | 4 +- server/main-api/src/main.rs | 24 +++- server/main-api/src/models.rs | 4 +- server/main-api/src/setup/database/data.rs | 133 +++++------------- 18 files changed, 260 insertions(+), 199 deletions(-) create mode 100644 server/.sqlx/query-0129b39fb399d5d3d41b2053c9bf617a646c40ec0e2f614d035948f240c8b245.json create mode 100644 server/.sqlx/query-7a7a91b71318cae47f5aafceb365860cfad2fc568d61ce6648ca1ba7770db617.json delete mode 100644 server/.sqlx/query-866f41f41ff9e68e92736009d98b37f497aeb87f11eeafa13eadd4fefda706b5.json create mode 100644 server/.sqlx/query-acfccc2d96fcc85154dc3a6dd969ffdcaa2465cb27b510f9e50682f15980c45f.json delete mode 100644 server/.sqlx/query-e08d77584b88d04d53710630fc11af9a99868a7dee15c2f32ccca3c9b1c8b2b6.json create mode 100644 server/main-api/migrations/20240505224430_fix-calendar-id-not-extracted.down.sql create mode 100644 server/main-api/migrations/20240505224430_fix-calendar-id-not-extracted.up.sql diff --git a/server/.sqlx/query-0129b39fb399d5d3d41b2053c9bf617a646c40ec0e2f614d035948f240c8b245.json b/server/.sqlx/query-0129b39fb399d5d3d41b2053c9bf617a646c40ec0e2f614d035948f240c8b245.json new file mode 100644 index 000000000..661285ae5 --- /dev/null +++ b/server/.sqlx/query-0129b39fb399d5d3d41b2053c9bf617a646c40ec0e2f614d035948f240c8b245.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO en(key,data)VALUES ($1,$2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Json" + ] + }, + "nullable": [] + }, + "hash": "0129b39fb399d5d3d41b2053c9bf617a646c40ec0e2f614d035948f240c8b245" +} diff --git a/server/.sqlx/query-7a7a91b71318cae47f5aafceb365860cfad2fc568d61ce6648ca1ba7770db617.json b/server/.sqlx/query-7a7a91b71318cae47f5aafceb365860cfad2fc568d61ce6648ca1ba7770db617.json new file mode 100644 index 000000000..7783e44af --- /dev/null +++ b/server/.sqlx/query-7a7a91b71318cae47f5aafceb365860cfad2fc568d61ce6648ca1ba7770db617.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE en SET last_calendar_scrape_at = $1 WHERE key=$2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Timestamptz", + "Text" + ] + }, + "nullable": [] + }, + "hash": "7a7a91b71318cae47f5aafceb365860cfad2fc568d61ce6648ca1ba7770db617" +} diff --git a/server/.sqlx/query-866f41f41ff9e68e92736009d98b37f497aeb87f11eeafa13eadd4fefda706b5.json b/server/.sqlx/query-866f41f41ff9e68e92736009d98b37f497aeb87f11eeafa13eadd4fefda706b5.json deleted file mode 100644 index 25293b974..000000000 --- a/server/.sqlx/query-866f41f41ff9e68e92736009d98b37f497aeb87f11eeafa13eadd4fefda706b5.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO en(key,data,name,tumonline_room_nr,type,type_common_name,lat,lon)\n VALUES ($1,$2,$3,$4,$5,$6,$7,$8)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Int4", - "Text", - "Text", - "Float8", - "Float8" - ] - }, - "nullable": [] - }, - "hash": "866f41f41ff9e68e92736009d98b37f497aeb87f11eeafa13eadd4fefda706b5" -} diff --git a/server/.sqlx/query-874f7f834952dc9935690debfad3d161757d7ce6fcee59c23c23abbf45e803cb.json b/server/.sqlx/query-874f7f834952dc9935690debfad3d161757d7ce6fcee59c23c23abbf45e803cb.json index f1a625cd9..0bdf8ff0b 100644 --- a/server/.sqlx/query-874f7f834952dc9935690debfad3d161757d7ce6fcee59c23c23abbf45e803cb.json +++ b/server/.sqlx/query-874f7f834952dc9935690debfad3d161757d7ce6fcee59c23c23abbf45e803cb.json @@ -10,43 +10,48 @@ }, { "ordinal": 1, - "name": "name", - "type_info": "Text" + "name": "data", + "type_info": "Json" }, { "ordinal": 2, - "name": "tumonline_room_nr", - "type_info": "Int4" + "name": "last_calendar_scrape_at", + "type_info": "Timestamptz" }, { "ordinal": 3, - "name": "type", - "type_info": "Text" + "name": "lat", + "type_info": "Float8" }, { "ordinal": 4, - "name": "type_common_name", - "type_info": "Text" + "name": "lon", + "type_info": "Float8" }, { "ordinal": 5, - "name": "lat", - "type_info": "Float8" + "name": "name", + "type_info": "Text" }, { "ordinal": 6, - "name": "lon", - "type_info": "Float8" + "name": "type_common_name", + "type_info": "Text" }, { "ordinal": 7, - "name": "data", + "name": "type", "type_info": "Text" }, { "ordinal": 8, - "name": "last_calendar_scrape_at", - "type_info": "Timestamptz" + "name": "calendar_url", + "type_info": "Text" + }, + { + "ordinal": 9, + "name": "tumonline_room_nr", + "type_info": "Int4" } ], "parameters": { @@ -63,6 +68,7 @@ false, false, false, + true, true ] }, diff --git a/server/.sqlx/query-9f518183559171969ae448a0f3f0de7a221ce7d9bb8378028c695e1b665c55c5.json b/server/.sqlx/query-9f518183559171969ae448a0f3f0de7a221ce7d9bb8378028c695e1b665c55c5.json index ebe2808cc..f3747183a 100644 --- a/server/.sqlx/query-9f518183559171969ae448a0f3f0de7a221ce7d9bb8378028c695e1b665c55c5.json +++ b/server/.sqlx/query-9f518183559171969ae448a0f3f0de7a221ce7d9bb8378028c695e1b665c55c5.json @@ -6,7 +6,7 @@ { "ordinal": 0, "name": "data", - "type_info": "Text" + "type_info": "Json" } ], "parameters": { diff --git a/server/.sqlx/query-a16d239da728e396e283aea0782d9b32f08c86626db9f4d63249dfc2cd77b80c.json b/server/.sqlx/query-a16d239da728e396e283aea0782d9b32f08c86626db9f4d63249dfc2cd77b80c.json index 8b2ce6edc..653b8f972 100644 --- a/server/.sqlx/query-a16d239da728e396e283aea0782d9b32f08c86626db9f4d63249dfc2cd77b80c.json +++ b/server/.sqlx/query-a16d239da728e396e283aea0782d9b32f08c86626db9f4d63249dfc2cd77b80c.json @@ -6,7 +6,7 @@ { "ordinal": 0, "name": "data", - "type_info": "Text" + "type_info": "Json" } ], "parameters": { diff --git a/server/.sqlx/query-a87d33b55872d9135ab82391b36d1a495534817fb39a9e25f6720267c6025965.json b/server/.sqlx/query-a87d33b55872d9135ab82391b36d1a495534817fb39a9e25f6720267c6025965.json index 89d01e799..2c2083956 100644 --- a/server/.sqlx/query-a87d33b55872d9135ab82391b36d1a495534817fb39a9e25f6720267c6025965.json +++ b/server/.sqlx/query-a87d33b55872d9135ab82391b36d1a495534817fb39a9e25f6720267c6025965.json @@ -10,43 +10,48 @@ }, { "ordinal": 1, - "name": "name", - "type_info": "Text" + "name": "data", + "type_info": "Json" }, { "ordinal": 2, - "name": "tumonline_room_nr", - "type_info": "Int4" + "name": "last_calendar_scrape_at", + "type_info": "Timestamptz" }, { "ordinal": 3, - "name": "type", - "type_info": "Text" + "name": "lat", + "type_info": "Float8" }, { "ordinal": 4, - "name": "type_common_name", - "type_info": "Text" + "name": "lon", + "type_info": "Float8" }, { "ordinal": 5, - "name": "lat", - "type_info": "Float8" + "name": "name", + "type_info": "Text" }, { "ordinal": 6, - "name": "lon", - "type_info": "Float8" + "name": "type_common_name", + "type_info": "Text" }, { "ordinal": 7, - "name": "data", + "name": "type", "type_info": "Text" }, { "ordinal": 8, - "name": "last_calendar_scrape_at", - "type_info": "Timestamptz" + "name": "calendar_url", + "type_info": "Text" + }, + { + "ordinal": 9, + "name": "tumonline_room_nr", + "type_info": "Int4" } ], "parameters": { @@ -63,6 +68,7 @@ false, false, false, + true, true ] }, diff --git a/server/.sqlx/query-acfccc2d96fcc85154dc3a6dd969ffdcaa2465cb27b510f9e50682f15980c45f.json b/server/.sqlx/query-acfccc2d96fcc85154dc3a6dd969ffdcaa2465cb27b510f9e50682f15980c45f.json new file mode 100644 index 000000000..48d0d982a --- /dev/null +++ b/server/.sqlx/query-acfccc2d96fcc85154dc3a6dd969ffdcaa2465cb27b510f9e50682f15980c45f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO de(key,data)VALUES ($1,$2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Json" + ] + }, + "nullable": [] + }, + "hash": "acfccc2d96fcc85154dc3a6dd969ffdcaa2465cb27b510f9e50682f15980c45f" +} diff --git a/server/.sqlx/query-e08d77584b88d04d53710630fc11af9a99868a7dee15c2f32ccca3c9b1c8b2b6.json b/server/.sqlx/query-e08d77584b88d04d53710630fc11af9a99868a7dee15c2f32ccca3c9b1c8b2b6.json deleted file mode 100644 index c02d9fc75..000000000 --- a/server/.sqlx/query-e08d77584b88d04d53710630fc11af9a99868a7dee15c2f32ccca3c9b1c8b2b6.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO de(key,data,name,tumonline_room_nr,type,type_common_name,lat,lon)\n VALUES ($1,$2,$3,$4,$5,$6,$7,$8)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Int4", - "Text", - "Text", - "Float8", - "Float8" - ] - }, - "nullable": [] - }, - "hash": "e08d77584b88d04d53710630fc11af9a99868a7dee15c2f32ccca3c9b1c8b2b6" -} diff --git a/server/main-api/migrations/20240505224430_fix-calendar-id-not-extracted.down.sql b/server/main-api/migrations/20240505224430_fix-calendar-id-not-extracted.down.sql new file mode 100644 index 000000000..d5d5f29ff --- /dev/null +++ b/server/main-api/migrations/20240505224430_fix-calendar-id-not-extracted.down.sql @@ -0,0 +1,43 @@ +-- Add down migration script here +-- was never used +create table rooms +( + key text primary key not null, + tumonline_org_id integer not null, + tumonline_calendar_id integer not null, + tumonline_room_id integer not null, + last_scrape timestamp without time zone not null +); + +-- migrating to +DROP TABLE en; +create table en +( + key text not null + primary key + references de, + name text not null, + tumonline_room_nr integer, + type text not null, + type_common_name text not null, + lat double precision not null, + lon double precision not null, + data text not null, + last_calendar_scrape_at timestamp with time zone +); +comment on column en.last_calendar_scrape_at is 'the last time the calendar was scraped for this room'; + +DROP TABLE de; +create table de +( + key text not null primary key, + name text not null, + tumonline_room_nr integer, + type text not null, + type_common_name text not null, + lat double precision not null, + lon double precision not null, + data text not null, + last_calendar_scrape_at timestamp with time zone +); +comment on column de.last_calendar_scrape_at is 'the last time the calendar was scraped for this room'; diff --git a/server/main-api/migrations/20240505224430_fix-calendar-id-not-extracted.up.sql b/server/main-api/migrations/20240505224430_fix-calendar-id-not-extracted.up.sql new file mode 100644 index 000000000..9d37224d0 --- /dev/null +++ b/server/main-api/migrations/20240505224430_fix-calendar-id-not-extracted.up.sql @@ -0,0 +1,34 @@ +-- Add up migration script here +-- was never used +DROP TABLE rooms; + +-- migrating to using the json type instead of having elaborate insertion logic +alter table de alter column data type jsonb using data::jsonb; +alter table de drop column lat; +alter table de add column lat FLOAT NOT NULL GENERATED ALWAYS AS (CAST (data->'coords'->>'lat' AS FLOAT)) STORED; +alter table de drop column lon; +alter table de add column lon FLOAT NOT NULL GENERATED ALWAYS AS (CAST (data->'coords'->>'lon' AS FLOAT)) STORED; +alter table de drop column name; +alter table de add column name TEXT NOT NULL GENERATED ALWAYS AS (CAST (data->>'name' AS TEXT)) STORED; +alter table de drop column type_common_name; +alter table de add column type_common_name TEXT NOT NULL GENERATED ALWAYS AS (CAST (data->>'type_common_name' AS TEXT)) STORED; +alter table de drop column type; +alter table de add column type TEXT NOT NULL GENERATED ALWAYS AS (CAST (data->>'type' AS TEXT)) STORED; +alter table de add column calendar_url TEXT GENERATED ALWAYS AS (CAST (data->'props'->>'calendar_url' AS TEXT)) STORED; +alter table de drop column tumonline_room_nr; +alter table de add column tumonline_room_nr INTEGER GENERATED ALWAYS AS (CAST (data->'props'->>'tumonline_room_nr' AS INTEGER)) STORED; + +alter table en alter column data type jsonb using data::jsonb; +alter table en drop column lat; +alter table en add column lat FLOAT NOT NULL GENERATED ALWAYS AS (CAST (data->'coords'->>'lat' AS FLOAT)) STORED; +alter table en drop column lon; +alter table en add column lon FLOAT NOT NULL GENERATED ALWAYS AS (CAST (data->'coords'->>'lon' AS FLOAT)) STORED; +alter table en drop column name; +alter table en add column name TEXT NOT NULL GENERATED ALWAYS AS (CAST (data->>'name' AS TEXT)) STORED; +alter table en drop column type_common_name; +alter table en add column type_common_name TEXT NOT NULL GENERATED ALWAYS AS (CAST (data->>'type_common_name' AS TEXT)) STORED; +alter table en drop column type; +alter table en add column type TEXT NOT NULL GENERATED ALWAYS AS (CAST (data->>'type' AS TEXT)) STORED; +alter table en add column calendar_url TEXT GENERATED ALWAYS AS (CAST (data->'props'->>'calendar_url' AS TEXT)) STORED; +alter table en drop column tumonline_room_nr; +alter table en add column tumonline_room_nr INTEGER GENERATED ALWAYS AS (CAST (data->'props'->>'tumonline_room_nr' AS INTEGER)) STORED; diff --git a/server/main-api/src/calendar/fetch/connectum.rs b/server/main-api/src/calendar/fetch/connectum.rs index 489b920b8..e550b5fad 100644 --- a/server/main-api/src/calendar/fetch/connectum.rs +++ b/server/main-api/src/calendar/fetch/connectum.rs @@ -78,12 +78,12 @@ impl APIRequestor { async fn store( &self, events: &[Event], - last_sync: &DateTime, + last_calendar_scrape_at: &DateTime, id: &str, ) -> Result<(), crate::BoxedError> { // insert into db let mut tx = self.pool.begin().await?; - if let Err(e) = self.delete_events(id, &mut tx).await { + if let Err(e) = self.delete_events(&mut tx, id).await { error!("could not delete existing events because {e:?}"); tx.rollback().await?; return Err(e.into()); @@ -97,13 +97,14 @@ impl APIRequestor { ); } } - sqlx::query!( - "UPDATE de SET last_calendar_scrape_at = $1 WHERE key=$2", - last_sync, - id - ) - .execute(&self.pool) - .await?; + if let Err(e) = self + .update_last_calendar_scrape_at(&mut tx, id, last_calendar_scrape_at) + .await + { + error!("could not update last_calendar_scrape_at because {e:?}"); + tx.rollback().await?; + return Err(e.into()); + } tx.commit().await?; debug!("finished inserting into the db for {id}"); Ok(()) @@ -138,11 +139,32 @@ impl APIRequestor { } async fn delete_events( &self, - id: &str, tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + id: &str, ) -> Result { sqlx::query!(r#"DELETE FROM calendar WHERE room_code = $1"#, id) .execute(&mut **tx) .await } + async fn update_last_calendar_scrape_at( + &self, + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + id: &str, + last_calendar_scrape_at: &DateTime, + ) -> Result { + sqlx::query!( + "UPDATE en SET last_calendar_scrape_at = $1 WHERE key=$2", + last_calendar_scrape_at, + id + ) + .execute(&mut **tx) + .await?; + sqlx::query!( + "UPDATE de SET last_calendar_scrape_at = $1 WHERE key=$2", + last_calendar_scrape_at, + id + ) + .execute(&mut **tx) + .await + } } diff --git a/server/main-api/src/calendar/fetch/db.rs b/server/main-api/src/calendar/fetch/db.rs index c59d55427..62282bbbb 100644 --- a/server/main-api/src/calendar/fetch/db.rs +++ b/server/main-api/src/calendar/fetch/db.rs @@ -1,7 +1,8 @@ -use crate::calendar::fetch::CalendarEntryFetcher; use chrono::{DateTime, Utc}; use sqlx::PgPool; +use crate::calendar::fetch::CalendarEntryFetcher; + pub(super) struct DbRequestor { pool: PgPool, last_calendar_scrape_at: Option>, diff --git a/server/main-api/src/calendar/mod.rs b/server/main-api/src/calendar/mod.rs index d81284645..20009975b 100644 --- a/server/main-api/src/calendar/mod.rs +++ b/server/main-api/src/calendar/mod.rs @@ -45,10 +45,11 @@ pub async fn calendar_handler( } Ok(Some(loc)) => loc, }; - let calendar_url = format!( - "https://campus.tum.de/tumonline/wbKalender.wbRessource?pResNr={id}", - id = 0 - ); // TODO: room.tumonline_calendar_id + let Some(calendar_url) = location.calendar_url else { + return HttpResponse::NotFound() + .content_type("text/plain") + .body("Room does not have a calendar"); + }; let fetching_strategy = fetch::StrategyExecutor::new(&data.db, &id, &args.start_after, &args.end_before); match fetching_strategy diff --git a/server/main-api/src/details.rs b/server/main-api/src/details.rs index 6f5911a69..913f75cea 100644 --- a/server/main-api/src/details.rs +++ b/server/main-api/src/details.rs @@ -30,8 +30,8 @@ pub async fn get_handler( Ok(d) => match d { None => HttpResponse::NotFound().body("Not found"), Some(d) => { - let mut response_json = d.clone(); - // We don not want to serialise this data at any point in the server. + let mut response_json = serde_json::to_string(&d).unwrap(); + // We don't want to serialise this data at any point in the server. // This just flows through the server, but adding redirect_url to the response is necessary response_json.pop(); // remove last } response_json.push_str(&format!(",\"redirect_url\":\"{redirect_url}\"}}",)); diff --git a/server/main-api/src/main.rs b/server/main-api/src/main.rs index c73822d33..9bec74a36 100644 --- a/server/main-api/src/main.rs +++ b/server/main-api/src/main.rs @@ -57,7 +57,7 @@ fn connection_string() -> String { fn setup_logging() { #[cfg(debug_assertions)] { - let env = env_logger::Env::default().default_filter_or("trace"); + let env = env_logger::Env::default().default_filter_or("debug"); env_logger::Builder::from_env(env).init(); } #[cfg(not(debug_assertions))] @@ -72,12 +72,19 @@ fn setup_logging() { #[tokio::main] async fn main() -> Result<(), BoxedError> { setup_logging(); - let uri = connection_string(); - let pool = PgPoolOptions::new().connect(&uri).await?; - #[cfg(not(feature = "skip_db_setup"))] - setup::database::setup_database(&pool).await?; - #[cfg(not(feature = "skip_ms_setup"))] - setup::meilisearch::setup_meilisearch().await?; + #[cfg(any(not(feature = "skip_ms_setup"), not(feature = "skip_db_setup")))] + let setup_database = tokio::spawn(async move { + #[cfg(not(feature = "skip_db_setup"))] + { + let pool = PgPoolOptions::new() + .connect(&connection_string()) + .await + .unwrap(); + setup::database::setup_database(&pool).await.unwrap(); + } + #[cfg(not(feature = "skip_ms_setup"))] + setup::meilisearch::setup_meilisearch().await.unwrap(); + }); debug!("setting up metrics"); let labels = HashMap::from([( @@ -91,6 +98,7 @@ async fn main() -> Result<(), BoxedError> { .unwrap(); info!("running the server"); + let pool = PgPoolOptions::new().connect(&connection_string()).await?; HttpServer::new(move || { let cors = Cors::default() .allow_any_origin() @@ -115,5 +123,7 @@ async fn main() -> Result<(), BoxedError> { .bind(std::env::var("BIND_ADDRESS").unwrap_or_else(|_| "0.0.0.0:3003".to_string()))? .run() .await?; + #[cfg(any(not(feature = "skip_ms_setup"), not(feature = "skip_db_setup")))] + setup_database.await?; Ok(()) } diff --git a/server/main-api/src/models.rs b/server/main-api/src/models.rs index 2f099ae12..4b6a362cf 100644 --- a/server/main-api/src/models.rs +++ b/server/main-api/src/models.rs @@ -1,4 +1,5 @@ use chrono::{DateTime, Utc}; +use serde_json::Value; #[derive(Debug, Clone)] pub struct Location { @@ -6,11 +7,12 @@ pub struct Location { pub name: String, pub last_calendar_scrape_at: Option>, pub tumonline_room_nr: Option, + pub calendar_url: Option, pub r#type: String, pub type_common_name: String, pub lat: f64, pub lon: f64, - pub data: String, + pub data: Value, } #[derive(Debug, Clone)] diff --git a/server/main-api/src/setup/database/data.rs b/server/main-api/src/setup/database/data.rs index 03f67c1e8..2db9f6938 100644 --- a/server/main-api/src/setup/database/data.rs +++ b/server/main-api/src/setup/database/data.rs @@ -4,83 +4,10 @@ use std::time::Instant; use log::info; use serde_json::Value; -struct ExtractedFields { - name: String, - tumonline_room_nr: Option, - r#type: String, - type_common_name: String, - lat: f64, - lon: f64, -} - -impl From> for ExtractedFields { - fn from(obj: HashMap) -> Self { - let props = obj.get("props").unwrap().as_object().unwrap(); - let tumonline_room_nr = props - .get("tumonline_room_nr") - .map(|v| v.as_i64().unwrap() as i32); - let (lat, lon) = match obj.get("coords") { - Some(coords) => { - let lat = coords.get("lat").unwrap().as_f64(); - let lon = coords.get("lon").unwrap().as_f64(); - (lat, lon) - } - None => (None, None), - }; - ExtractedFields { - name: obj.get("name").unwrap().as_str().unwrap().to_string(), - tumonline_room_nr, - r#type: obj.get("type").unwrap().as_str().unwrap().to_string(), - type_common_name: obj - .get("type_common_name") - .unwrap() - .as_str() - .unwrap() - .to_string(), - lat: lat.unwrap_or(48.14903), - lon: lon.unwrap_or(11.56735), - } - } -} - -struct StorableValue; - -impl StorableValue { - fn from(value: HashMap) -> (String, ExtractedFields) { - let data = serde_json::to_string(&value).unwrap(); - (data, ExtractedFields::from(value)) - } -} - -fn delocalise(value: Value, language: &'static str) -> Value { - match value { - Value::Array(arr) => Value::Array( - arr.into_iter() - .map(|value| delocalise(value, language)) - .collect(), - ), - Value::Object(obj) => { - if obj.contains_key("de") || obj.contains_key("en") { - obj.get(language) - .cloned() - .unwrap_or(Value::String(String::new())) - } else { - Value::Object( - obj.into_iter() - .map(|(key, value)| (key, delocalise(value, language))) - .filter(|(key, _)| key != "de" && key != "en") - .collect(), - ) - } - } - a => a, - } -} - struct DelocalisedValues { key: String, - de: HashMap, - en: HashMap, + de: Value, + en: Value, } impl From> for DelocalisedValues { @@ -89,12 +16,12 @@ impl From> for DelocalisedValues { de: value .clone() .into_iter() - .map(|(k, v)| (k, delocalise(v.clone(), "de"))) + .map(|(k, v)| (k, Self::delocalise(v.clone(), "de"))) .collect(), en: value .clone() .into_iter() - .map(|(k, v)| (k, delocalise(v.clone(), "en"))) + .map(|(k, v)| (k, Self::delocalise(v.clone(), "en"))) .collect(), key: value .clone() @@ -106,41 +33,47 @@ impl From> for DelocalisedValues { } } } - impl DelocalisedValues { + fn delocalise(value: Value, language: &'static str) -> Value { + match value { + Value::Array(arr) => Value::Array( + arr.into_iter() + .map(|value| Self::delocalise(value, language)) + .collect(), + ), + Value::Object(obj) => { + if obj.contains_key("de") || obj.contains_key("en") { + obj.get(language) + .cloned() + .unwrap_or(Value::String(String::new())) + } else { + Value::Object( + obj.into_iter() + .map(|(key, value)| (key, Self::delocalise(value, language))) + .filter(|(key, _)| key != "de" && key != "en") + .collect(), + ) + } + } + a => a, + } + } async fn store( self, tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, ) -> Result<(), sqlx::Error> { - let key = self.key.clone(); // has to be here due to livetimes somehow - let (data, fields) = StorableValue::from(self.de); sqlx::query!( - r#"INSERT INTO de(key,data,name,tumonline_room_nr,type,type_common_name,lat,lon) - VALUES ($1,$2,$3,$4,$5,$6,$7,$8)"#, - key, - data, - fields.name, - fields.tumonline_room_nr, - fields.r#type, - fields.type_common_name, - fields.lat, - fields.lon, + r#"INSERT INTO de(key,data)VALUES ($1,$2)"#, + self.key, + self.de ) .execute(&mut **tx) .await?; - let (data, fields) = StorableValue::from(self.en); sqlx::query!( - r#"INSERT INTO en(key,data,name,tumonline_room_nr,type,type_common_name,lat,lon) - VALUES ($1,$2,$3,$4,$5,$6,$7,$8)"#, + r#"INSERT INTO en(key,data)VALUES ($1,$2)"#, self.key, - data, - fields.name, - fields.tumonline_room_nr, - fields.r#type, - fields.type_common_name, - fields.lat, - fields.lon, + self.en ) .execute(&mut **tx) .await?; From 61d9aa62c10ddda0a06a9653f16586d5db78e0f6 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Tue, 7 May 2024 17:26:10 +0200 Subject: [PATCH 2/2] fixed an invalid tumonline calendar link Resolves https://github.com/TUM-Dev/NavigaTUM/issues/1163 --- data/processors/sections.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data/processors/sections.py b/data/processors/sections.py index e5473a2db..6da0049af 100644 --- a/data/processors/sections.py +++ b/data/processors/sections.py @@ -10,7 +10,8 @@ def extract_tumonline_props(data: dict[str, dict[str, Any]]) -> None: """Extract some of the TUMonline data and provides it as `prop`.""" for entry in data.values(): if entry.get("tumonline_data", {}).get("calendar", None): - calendar_url = f"https://campus.tum.de/tumonline/{entry['tumonline_data']['calendar']}" + calendar_resource_id = entry["tumonline_data"]["calendar"] + calendar_url = f"https://campus.tum.de/tumonline/tvKalender.wSicht?cOrg=0&cRes={calendar_resource_id}" entry["props"]["calendar_url"] = calendar_url if entry.get("tumonline_data", {}).get("operator", None): entry["props"]["operator"] = {