From 292cb91f4182abca84841eae7177fc9db18c9bf9 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Fri, 10 May 2024 05:54:49 +0200 Subject: [PATCH] tmp --- server/Cargo.lock | 248 +++++++++++++++++++++- server/main-api/Cargo.toml | 2 + server/main-api/src/main.rs | 16 +- server/main-api/src/setup/database/mod.rs | 25 ++- server/main-api/src/setup/meilisearch.rs | 26 +-- server/main-api/src/setup/mod.rs | 53 +++++ 6 files changed, 333 insertions(+), 37 deletions(-) diff --git a/server/Cargo.lock b/server/Cargo.lock index 47fcb14bf..833db3f82 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -559,6 +559,56 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bollard" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aed08d3adb6ebe0eff737115056652670ae290f177759aac19c30456135f94c" +dependencies = [ + "base64 0.22.1", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "home", + "http 1.1.0", + "http-body-util", + "hyper 1.3.1", + "hyper-named-pipe", + "hyper-rustls 0.26.0", + "hyper-util", + "hyperlocal-next", + "log", + "pin-project-lite", + "rustls 0.22.4", + "rustls-native-certs", + "rustls-pemfile 2.1.2", + "rustls-pki-types", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "serde_urlencoded", + "thiserror", + "tokio", + "tokio-util", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.44.0-rc.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709d9aa1c37abb89d40f19f5d0ad6f0d88cb1581264e571c9350fc5bb89cf1c5" +dependencies = [ + "serde", + "serde_repr", + "serde_with", +] + [[package]] name = "brotli" version = "6.0.0" @@ -637,7 +687,7 @@ dependencies = [ "cached_proc_macro_types", "directories", "futures", - "hashbrown", + "hashbrown 0.14.5", "instant", "once_cell", "rmp-serde", @@ -892,7 +942,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown", + "hashbrown 0.14.5", "lock_api", "once_cell", "parking_lot_core 0.9.10", @@ -965,6 +1015,15 @@ dependencies = [ "dirs-sys", ] +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + [[package]] name = "dirs-sys" version = "0.4.1" @@ -977,6 +1036,29 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "dns-lookup" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5766087c2235fec47fafa4cfecc81e494ee679d0fd4a59887ea0919bfb0e4fc" +dependencies = [ + "cfg-if", + "libc", + "socket2", + "windows-sys 0.48.0", +] + +[[package]] +name = "docker_credential" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31951f49556e34d90ed28342e1df7e1cb7a229c4cab0aecc627b5d91edd41d07" +dependencies = [ + "base64 0.21.7", + "serde", + "serde_json", +] + [[package]] name = "dotenvy" version = "0.15.7" @@ -1347,7 +1429,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap", + "indexmap 2.2.6", "slab", "tokio", "tokio-util", @@ -1366,7 +1448,7 @@ dependencies = [ "futures-sink", "futures-util", "http 1.1.0", - "indexmap", + "indexmap 2.2.6", "slab", "tokio", "tokio-util", @@ -1383,6 +1465,12 @@ dependencies = [ "crunchy", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" @@ -1399,7 +1487,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ - "hashbrown", + "hashbrown 0.14.5", ] [[package]] @@ -1630,6 +1718,21 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-named-pipe" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" +dependencies = [ + "hex", + "hyper 1.3.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", + "winapi", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -1702,6 +1805,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "hyperlocal-next" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acf569d43fa9848e510358c07b80f4adf34084ddc28c6a4a651ee8474c070dcc" +dependencies = [ + "hex", + "http-body-util", + "hyper 1.3.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -1808,6 +1926,17 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44feda355f4159a7c757171a77de25daf6411e217b4cabd03bd6650690468126" +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.2.6" @@ -1815,7 +1944,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.14.5", + "serde", ] [[package]] @@ -2255,6 +2385,8 @@ dependencies = [ "sqlx", "structured-logger", "tempfile", + "testcontainers", + "testcontainers-modules", "time", "tokio", "unicode-truncate", @@ -2552,6 +2684,31 @@ dependencies = [ "windows-targets 0.52.5", ] +[[package]] +name = "parse-display" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06af5f9333eb47bd9ba8462d612e37a8328a5cb80b13f0af4de4c3b89f52dee5" +dependencies = [ + "parse-display-derive", + "regex", + "regex-syntax", +] + +[[package]] +name = "parse-display-derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc9252f259500ee570c75adcc4e317fa6f57a1e47747d622e0bf838002a7b790" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "regex-syntax", + "structmeta", + "syn 2.0.61", +] + [[package]] name = "paste" version = "1.0.15" @@ -3441,6 +3598,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_repr" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.61", +] + [[package]] name = "serde_spanned" version = "0.6.5" @@ -3462,13 +3630,43 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad483d2ab0149d5a5ebcd9972a3852711e0153d863bf5a5d0391d28883c4a20" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.2.6", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65569b702f41443e8bc8bbb1c5779bd0450bbe723b56198980e80ec45780bce2" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.61", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap", + "indexmap 2.2.6", "itoa", "ryu", "serde", @@ -3692,7 +3890,7 @@ dependencies = [ "futures-util", "hashlink", "hex", - "indexmap", + "indexmap 2.2.6", "log", "memchr", "once_cell", @@ -4087,6 +4285,38 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "testcontainers" +version = "0.16.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69d47265a44d1035a322691cf0a6cc227d79b62ef86ffb0dbc204b394fee3d07" +dependencies = [ + "async-trait", + "bollard", + "bollard-stubs", + "dirs", + "dns-lookup", + "docker_credential", + "futures", + "log", + "parse-display", + "serde", + "serde_json", + "serde_with", + "tokio", + "tokio-util", + "url", +] + +[[package]] +name = "testcontainers-modules" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8da66a6ebd55684c8e3c58c7374dc2b2d0d1884ac688987e7ffb2d02103a3ee" +dependencies = [ + "testcontainers", +] + [[package]] name = "thiserror" version = "1.0.60" @@ -4277,7 +4507,7 @@ version = "0.22.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3328d4f68a705b2a4498da1d580585d39a6510f98318a2cec3018a7ec61ddef" dependencies = [ - "indexmap", + "indexmap 2.2.6", "serde", "serde_spanned", "toml_datetime", diff --git a/server/main-api/Cargo.toml b/server/main-api/Cargo.toml index 375fda053..653ce1aa3 100644 --- a/server/main-api/Cargo.toml +++ b/server/main-api/Cargo.toml @@ -69,6 +69,8 @@ time = "0.3.36" [dev-dependencies] pretty_assertions = "1.4.0" +testcontainers = "0.16.7" +testcontainers-modules = {version = "0.4.2",features = ["postgres"] } [features] skip_db_setup = [] diff --git a/server/main-api/src/main.rs b/server/main-api/src/main.rs index f8db7c867..a0a85dffc 100644 --- a/server/main-api/src/main.rs +++ b/server/main-api/src/main.rs @@ -5,6 +5,7 @@ use actix_cors::Cors; use actix_web::{get, middleware, web, App, HttpResponse, HttpServer}; use actix_web_prom::PrometheusMetricsBuilder; use log::{debug, error, info}; +use meilisearch_sdk::client::Client; use sqlx::postgres::PgPoolOptions; use sqlx::prelude::*; use sqlx::PgPool; @@ -83,9 +84,18 @@ async fn main() -> Result<(), BoxedError> { .await .unwrap(); #[cfg(not(feature = "skip_db_setup"))] - setup::database::setup(&pool).await.unwrap(); + { + setup::database::setup(&pool).await.unwrap(); + setup::database::load_data(&pool).await.unwrap(); + } #[cfg(not(feature = "skip_ms_setup"))] - setup::meilisearch::setup().await.unwrap(); + { + let ms_url = + std::env::var("MIELI_URL").unwrap_or_else(|_| "http://localhost:7700".to_string()); + let client = Client::new(ms_url, std::env::var("MEILI_MASTER_KEY").ok()).unwrap(); + setup::meilisearch::setup(&client).await.unwrap(); + setup::meilisearch::load_data(&client).await.unwrap(); + } calendar::refresh::all_entries(&pool).await; }); @@ -102,6 +112,7 @@ async fn main() -> Result<(), BoxedError> { info!("running the server"); let pool = PgPoolOptions::new().connect(&connection_string()).await?; + let shutdown_pool_clone = pool.clone(); HttpServer::new(move || { let cors = Cors::default() .allow_any_origin() @@ -127,5 +138,6 @@ async fn main() -> Result<(), BoxedError> { .run() .await?; maintenance_thread.abort(); + shutdown_pool_clone.close().await; Ok(()) } diff --git a/server/main-api/src/setup/database/mod.rs b/server/main-api/src/setup/database/mod.rs index 83f411322..f834c2ca8 100644 --- a/server/main-api/src/setup/database/mod.rs +++ b/server/main-api/src/setup/database/mod.rs @@ -5,27 +5,26 @@ mod data; pub async fn setup(pool: &sqlx::PgPool) -> Result<(), crate::BoxedError> { info!("setting up the database"); - sqlx::migrate!("./migrations").run(pool).await?; - info!("migrations complete"); - + Ok(()) +} +pub async fn load_data(pool: &sqlx::PgPool) -> Result<(), crate::BoxedError> { let mut tx = pool.begin().await?; - load_data(&mut tx).await?; + + info!("deleting old data"); + cleanup(&mut tx).await?; + info!("loading new data"); + data::load_all_to_db(&mut tx).await?; + info!("loading new aliases"); + alias::load_all_to_db(&mut tx).await?; tx.commit().await?; Ok(()) } -async fn load_data( - tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, -) -> Result<(), crate::BoxedError> { - info!("deleting old data"); + +async fn cleanup(tx: &mut sqlx::Transaction<'_, sqlx::Postgres>) -> Result<(), crate::BoxedError> { sqlx::query!("DELETE FROM aliases") .execute(&mut **tx) .await?; sqlx::query!("DELETE FROM en").execute(&mut **tx).await?; sqlx::query!("DELETE FROM de").execute(&mut **tx).await?; - - info!("loading new data"); - data::load_all_to_db(tx).await?; - info!("loading new aliases"); - alias::load_all_to_db(tx).await?; Ok(()) } diff --git a/server/main-api/src/setup/meilisearch.rs b/server/main-api/src/setup/meilisearch.rs index f1ba4e114..a55a6a90b 100644 --- a/server/main-api/src/setup/meilisearch.rs +++ b/server/main-api/src/setup/meilisearch.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::time::Duration; -use log::{error, info}; +use log::{debug, error, info}; use meilisearch_sdk::client::Client; use meilisearch_sdk::settings::Settings; use meilisearch_sdk::tasks::Task; @@ -44,20 +44,15 @@ async fn wait_for_healthy(client: &Client) { } } -pub async fn setup() -> Result<(), crate::BoxedError> { - info!("setting up meilisearch"); - let start = std::time::Instant::now(); - let ms_url = std::env::var("MIELI_URL").unwrap_or_else(|_| "http://localhost:7700".to_string()); - info!("connecting to Meilisearch at {ms_url}", ms_url = ms_url); - let client = Client::new(ms_url, std::env::var("MEILI_MASTER_KEY").ok())?; - info!("waiting for Meilisearch to be healthy"); - wait_for_healthy(&client).await; +pub async fn setup(client: &Client) -> Result<(), crate::BoxedError> { + debug!("waiting for Meilisearch to be healthy"); + wait_for_healthy(client).await; info!("Meilisearch is healthy"); client .create_index("entries", Some("ms_id")) .await? - .wait_for_completion(&client, POLLING_RATE, TIMEOUT) + .wait_for_completion(client, POLLING_RATE, TIMEOUT) .await?; let entries = client.index("entries"); @@ -97,12 +92,17 @@ pub async fn setup() -> Result<(), crate::BoxedError> { let res = entries .set_settings(&settings) .await? - .wait_for_completion(&client, POLLING_RATE, TIMEOUT) + .wait_for_completion(client, POLLING_RATE, TIMEOUT) .await?; if let Task::Failed { content } = res { - panic!("Failed to add documents to Meilisearch: {content:#?}"); + panic!("Failed to add settings to Meilisearch: {content:#?}"); } + Ok(()) +} +pub async fn load_data(client: &Client) -> Result<(), crate::BoxedError> { + let start = std::time::Instant::now(); + let entries = client.index("entries"); let cdn_url = std::env::var("CDN_URL").unwrap_or_else(|_| "https://nav.tum.de/cdn".to_string()); let documents = reqwest::get(format!("{cdn_url}/search_data.json")) .await? @@ -111,7 +111,7 @@ pub async fn setup() -> Result<(), crate::BoxedError> { let res = entries .add_documents(&documents, Some("ms_id")) .await? - .wait_for_completion(&client, POLLING_RATE, TIMEOUT) + .wait_for_completion(client, POLLING_RATE, TIMEOUT) .await?; if let Task::Failed { content } = res { panic!("Failed to add documents to Meilisearch: {content:#?}"); diff --git a/server/main-api/src/setup/mod.rs b/server/main-api/src/setup/mod.rs index 80c6dc003..4c3387cbf 100644 --- a/server/main-api/src/setup/mod.rs +++ b/server/main-api/src/setup/mod.rs @@ -3,3 +3,56 @@ pub mod database; #[cfg(not(feature = "skip_ms_setup"))] pub mod meilisearch; + +#[cfg(all( + test, + not(all(not(feature = "skip_ms_setup"), not(feature = "skip_db_setup"))) +))] +pub mod tests { + use meilisearch_sdk::client::Client; + use testcontainers::{core::WaitFor, runners::AsyncRunner, GenericImage}; + use testcontainers_modules::{postgres::Postgres, testcontainers::RunnableImage}; + + /// Create a postgres instance for testing against + pub fn create_postgres() -> RunnableImage { + let container = RunnableImage::from(Postgres::default()) + .with_tag("16") + .with_env_var(("PGDATA", "/var/lib/postgresql/data/pgdata")) + .with_env_var(("POSTGRES_PASSWORD", "")) + .with_env_var(("POSTGRES_USER", "")) + .with_env_var(("POSTGRES_DB", "")); + + let host_ip = container.get_host(); + let host_port = container.get_host_port_ipv4(5432); + let url = format!("http://{ip}:{port}"); + sqlx::migrate!("./migrations").run(pool).await.unwrap(); + (container, pool) + } + async fn create_meilisearch() -> (GenericImage, Client) { + let container = GenericImage::new("getmeili/meilisearch", "v1.8.0") + .with_exposed_port(7700) + .with_wait_for(WaitFor::message_on_stdout( + "Actix runtime found; starting in Actix runtime", + )) + .start() + .await; + let ports = container.ports().await; + let port = ports.map_to_host_port_ipv4(7700).unwrap(); + let ip = container.get_bridge_ip_address().await; + let meili_url = format!("http://{ip}:{port}"); + + let client = Client::new(meili_url.clone(), None::).unwrap(); + super::meilisearch::setup(&client).await.unwrap(); + (container, client) + } + #[tokio::test] + async fn test_meilisearch_setup() { + let (container, client) = create_meilisearch(); + super::meilisearch::load_data(&client).await.unwrap(); + } + #[tokio::test] + async fn test_db_setup() { + let (container, pool) = create_postgres(); + super::meilisearch::load_data(pool).await.unwrap(); + } +}