From a29b06f670fe03071e43a5bca6a06f4e2b0947a0 Mon Sep 17 00:00:00 2001
From: Frank Elsinga <frank@elsinga.de>
Date: Mon, 29 Jul 2024 14:14:28 +0200
Subject: [PATCH] parquet

---
 server/main-api/src/setup/database/alias.rs |  1 +
 server/main-api/src/setup/database/data.rs  |  2 ++
 server/main-api/src/setup/database/mod.rs   | 11 -----------
 server/main-api/src/setup/meilisearch.rs    |  1 +
 server/main-api/src/setup/transportation.rs |  4 +++-
 5 files changed, 7 insertions(+), 12 deletions(-)

diff --git a/server/main-api/src/setup/database/alias.rs b/server/main-api/src/setup/database/alias.rs
index d605295ad..6927237e9 100644
--- a/server/main-api/src/setup/database/alias.rs
+++ b/server/main-api/src/setup/database/alias.rs
@@ -37,6 +37,7 @@ pub async fn download_updates() -> Result<LimitedVec<Alias>, crate::BoxedError>
     let cdn_url = std::env::var("CDN_URL").unwrap_or_else(|_| "https://nav.tum.de/cdn".to_string());
     let body = reqwest::get(format!("{cdn_url}/api_data.parquet"))
         .await?
+        .error_for_status()?
         .bytes()
         .await?;
     let mut aliase=Vec::<Alias>::new();
diff --git a/server/main-api/src/setup/database/data.rs b/server/main-api/src/setup/database/data.rs
index fcd8c7306..7fe5e12b0 100644
--- a/server/main-api/src/setup/database/data.rs
+++ b/server/main-api/src/setup/database/data.rs
@@ -129,6 +129,7 @@ pub async fn download_updates() -> Result<LimitedVec<DelocalisedValues>, crate::
     let cdn_url = std::env::var("CDN_URL").unwrap_or_else(|_| "https://nav.tum.de/cdn".to_string());
     let body = reqwest::get(format!("{cdn_url}/api_data.parquet"))
         .await?
+        .error_for_status()?
         .bytes()
         .await?;
     let mut file = tempfile()?;
@@ -162,6 +163,7 @@ pub async fn download_status() -> Result<LimitedVec<(String, i64)>, crate::Boxed
     let cdn_url = std::env::var("CDN_URL").unwrap_or_else(|_| "https://nav.tum.de/cdn".to_string());
     let body = reqwest::get(format!("{cdn_url}/status_data.parquet"))
         .await?
+        .error_for_status()?
         .bytes()
         .await?;
     let mut file = tempfile()?;
diff --git a/server/main-api/src/setup/database/mod.rs b/server/main-api/src/setup/database/mod.rs
index 6d40ccb27..5d92ebedb 100644
--- a/server/main-api/src/setup/database/mod.rs
+++ b/server/main-api/src/setup/database/mod.rs
@@ -14,17 +14,6 @@ pub async fn setup(pool: &sqlx::PgPool) -> Result<(), crate::BoxedError> {
 }
 #[tracing::instrument(skip(pool))]
 pub async fn load_data(pool: &sqlx::PgPool) -> Result<(), crate::BoxedError> {
-    let status = data::download_status().await?.0;
-    let new_keys = status
-        .clone()
-        .into_iter()
-        .map(|(k, _)| k)
-        .collect::<LimitedVec<String>>();
-    {
-        let mut tx = pool.begin().await?;
-        cleanup_deleted(&new_keys, &mut tx).await?;
-        tx.commit().await?;
-    }
     {
         let data = data::download_updates().await?;
         let mut tx = pool.begin().await?;
diff --git a/server/main-api/src/setup/meilisearch.rs b/server/main-api/src/setup/meilisearch.rs
index c5673998a..d46927b91 100644
--- a/server/main-api/src/setup/meilisearch.rs
+++ b/server/main-api/src/setup/meilisearch.rs
@@ -105,6 +105,7 @@ pub async fn load_data(client: &Client) -> Result<(), crate::BoxedError> {
     let cdn_url = std::env::var("CDN_URL").unwrap_or_else(|_| "https://nav.tum.de/cdn".to_string());
     let documents = reqwest::get(format!("{cdn_url}/search_data.json"))
         .await?
+        .error_for_status()?
         .json::<Vec<Value>>()
         .await?;
     let res = entries
diff --git a/server/main-api/src/setup/transportation.rs b/server/main-api/src/setup/transportation.rs
index a51c87cbd..147532f3a 100644
--- a/server/main-api/src/setup/transportation.rs
+++ b/server/main-api/src/setup/transportation.rs
@@ -54,7 +54,9 @@ impl DBStation {
 #[tracing::instrument(skip(pool))]
 pub async fn setup(pool: &sqlx::PgPool) -> Result<(), crate::BoxedError> {
     let url = "https://raw.githubusercontent.com/TUM-Dev/NavigaTUM/main/data/external/results/public_transport.json";
-    let transportations: Vec<Station> = reqwest::get(url).await?.json().await?;
+    let transportations = reqwest::get(url).await?
+        .error_for_status()?
+        .json::<Vec<Station>>().await?;
     let transportations = transportations.into_iter().flat_map(|s| {
         let id = s.station.station_id.clone();
         let mut stations = vec![DBStation::from_station(s.station, None)];