Testcontainer based integration tests (#1244)

* implemented the basic infrastructure for integration-testing via testcontainers

* made sure that `setup_logging` can accept multiple initialisations

server/main-api/Cargo.toml

@@ -69,6 +69,8 @@ time = "0.3.36"
 
 [dev-dependencies]
 pretty_assertions = "1.4.0"
+testcontainers = "0.18.0"
+testcontainers-modules = {version =  "0.6.1",features = ["postgres"] }
 
 [features]
 skip_db_setup = []

server/main-api/migrations/20240505224430_fix-calendar-id-not-extracted.up.sql

@@ -1,5 +1,4 @@
 -- Add up migration script here
--- was never used
 DROP TABLE rooms;
 
 -- migrating to using the json type instead of having elaborate insertion logic

server/main-api/src/main.rs

@@ -5,6 +5,7 @@ use actix_cors::Cors;
 use actix_web::{get, middleware, web, App, HttpResponse, HttpServer};
 use actix_web_prom::PrometheusMetricsBuilder;
 use log::{debug, error, info};
+use meilisearch_sdk::client::Client;
 use sqlx::postgres::PgPoolOptions;
 use sqlx::prelude::*;
 use sqlx::PgPool;
@@ -54,23 +55,25 @@ fn connection_string() -> String {
     format!("postgres://{username}:{password}@{url}/{db}")
 }
 
-fn setup_logging() {
-    #[cfg(debug_assertions)]
+pub fn setup_logging() {
+    #[cfg(any(debug_assertions, test))]
     {
         let log_level = std::env::var("LOG_LEVEL").unwrap_or_else(|_| "debug".to_string());
         let filter = format!("{log_level},hyper=info,rustls=info,h2=info,sqlx=info,hickory_resolver=info,hickory_proto=info");
         let env = env_logger::Env::default().default_filter_or(&filter);
-        env_logger::Builder::from_env(env).init();
+        let _ = env_logger::Builder::from_env(env)
+            .is_test(cfg!(test))
+            .try_init();
     }
-    #[cfg(not(debug_assertions))]
+    #[cfg(not(any(debug_assertions, test)))]
     {
         let log_level = std::env::var("LOG_LEVEL").unwrap_or_else(|_| "info".to_string());
-        structured_logger::Builder::with_level(&log_level)
+        let _ = structured_logger::Builder::with_level(&log_level)
             .with_target_writer(
                 "*",
                 structured_logger::async_json::new_writer(tokio::io::stdout()),
             )
-            .init();
+            .try_init();
     }
 }
 
@@ -83,9 +86,18 @@ async fn main() -> Result<(), BoxedError> {
             .await
             .unwrap();
         #[cfg(not(feature = "skip_db_setup"))]
-        setup::database::setup(&pool).await.unwrap();
+        {
+            setup::database::setup(&pool).await.unwrap();
+            setup::database::load_data(&pool).await.unwrap();
+        }
         #[cfg(not(feature = "skip_ms_setup"))]
-        setup::meilisearch::setup().await.unwrap();
+        {
+            let ms_url =
+                std::env::var("MIELI_URL").unwrap_or_else(|_| "http://localhost:7700".to_string());
+            let client = Client::new(ms_url, std::env::var("MEILI_MASTER_KEY").ok()).unwrap();
+            setup::meilisearch::setup(&client).await.unwrap();
+            setup::meilisearch::load_data(&client).await.unwrap();
+        }
         calendar::refresh::all_entries(&pool).await;
     });
 
@@ -102,6 +114,7 @@ async fn main() -> Result<(), BoxedError> {
 
     info!("running the server");
     let pool = PgPoolOptions::new().connect(&connection_string()).await?;
+    let shutdown_pool_clone = pool.clone();
     HttpServer::new(move || {
         let cors = Cors::default()
             .allow_any_origin()
@@ -127,5 +140,6 @@ async fn main() -> Result<(), BoxedError> {
     .run()
     .await?;
     maintenance_thread.abort();
+    shutdown_pool_clone.close().await;
     Ok(())
 }

server/main-api/src/setup/database/mod.rs

@@ -7,25 +7,26 @@ pub async fn setup(pool: &sqlx::PgPool) -> Result<(), crate::BoxedError> {
     info!("setting up the database");
     sqlx::migrate!("./migrations").run(pool).await?;
     info!("migrations complete");
-
+    Ok(())
+}
+pub async fn load_data(pool: &sqlx::PgPool) -> Result<(), crate::BoxedError> {
     let mut tx = pool.begin().await?;
-    load_data(&mut tx).await?;
+
+    info!("deleting old data");
+    cleanup(&mut tx).await?;
+    info!("loading new data");
+    data::load_all_to_db(&mut tx).await?;
+    info!("loading new aliases");
+    alias::load_all_to_db(&mut tx).await?;
     tx.commit().await?;
     Ok(())
 }
-async fn load_data(
-    tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
-) -> Result<(), crate::BoxedError> {
-    info!("deleting old data");
+
+async fn cleanup(tx: &mut sqlx::Transaction<'_, sqlx::Postgres>) -> Result<(), crate::BoxedError> {
     sqlx::query!("DELETE FROM aliases")
         .execute(&mut **tx)
         .await?;
     sqlx::query!("DELETE FROM en").execute(&mut **tx).await?;
     sqlx::query!("DELETE FROM de").execute(&mut **tx).await?;
-
-    info!("loading new data");
-    data::load_all_to_db(tx).await?;
-    info!("loading new aliases");
-    alias::load_all_to_db(tx).await?;
     Ok(())
 }

server/main-api/src/setup/meilisearch.rs

@@ -1,7 +1,7 @@
 use std::collections::HashMap;
 use std::time::Duration;
 
-use log::{error, info};
+use log::{debug, error, info};
 use meilisearch_sdk::client::Client;
 use meilisearch_sdk::settings::Settings;
 use meilisearch_sdk::tasks::Task;
@@ -44,20 +44,15 @@ async fn wait_for_healthy(client: &Client) {
     }
 }
 
-pub async fn setup() -> Result<(), crate::BoxedError> {
-    info!("setting up meilisearch");
-    let start = std::time::Instant::now();
-    let ms_url = std::env::var("MIELI_URL").unwrap_or_else(|_| "http://localhost:7700".to_string());
-    info!("connecting to Meilisearch at {ms_url}", ms_url = ms_url);
-    let client = Client::new(ms_url, std::env::var("MEILI_MASTER_KEY").ok())?;
-    info!("waiting for Meilisearch to be healthy");
-    wait_for_healthy(&client).await;
+pub async fn setup(client: &Client) -> Result<(), crate::BoxedError> {
+    debug!("waiting for Meilisearch to be healthy");
+    wait_for_healthy(client).await;
     info!("Meilisearch is healthy");
 
     client
         .create_index("entries", Some("ms_id"))
         .await?
-        .wait_for_completion(&client, POLLING_RATE, TIMEOUT)
+        .wait_for_completion(client, POLLING_RATE, TIMEOUT)
         .await?;
     let entries = client.index("entries");
 
@@ -97,12 +92,17 @@ pub async fn setup() -> Result<(), crate::BoxedError> {
     let res = entries
         .set_settings(&settings)
         .await?
-        .wait_for_completion(&client, POLLING_RATE, TIMEOUT)
+        .wait_for_completion(client, POLLING_RATE, TIMEOUT)
         .await?;
     if let Task::Failed { content } = res {
-        panic!("Failed to add documents to Meilisearch: {content:#?}");
+        panic!("Failed to add settings to Meilisearch: {content:#?}");
     }
+    Ok(())
+}
 
+pub async fn load_data(client: &Client) -> Result<(), crate::BoxedError> {
+    let start = std::time::Instant::now();
+    let entries = client.index("entries");
     let cdn_url = std::env::var("CDN_URL").unwrap_or_else(|_| "https://nav.tum.de/cdn".to_string());
     let documents = reqwest::get(format!("{cdn_url}/search_data.json"))
         .await?
@@ -111,7 +111,7 @@ pub async fn setup() -> Result<(), crate::BoxedError> {
     let res = entries
         .add_documents(&documents, Some("ms_id"))
         .await?
-        .wait_for_completion(&client, POLLING_RATE, TIMEOUT)
+        .wait_for_completion(client, POLLING_RATE, TIMEOUT)
         .await?;
     if let Task::Failed { content } = res {
         panic!("Failed to add documents to Meilisearch: {content:#?}");

server/main-api/src/setup/mod.rs

@@ -3,3 +3,5 @@ pub mod database;
 
 #[cfg(not(feature = "skip_ms_setup"))]
 pub mod meilisearch;
+#[cfg(test)]
+pub mod tests;

server/main-api/src/setup/tests.rs

@@ -0,0 +1,89 @@
+use meilisearch_sdk::client::Client;
+use sqlx::postgres::PgPoolOptions;
+use sqlx::{Pool, Postgres};
+use testcontainers::core::{ContainerPort, WaitFor};
+use testcontainers::{ContainerAsync, GenericImage, ImageExt};
+use testcontainers_modules::{postgres, testcontainers::runners::AsyncRunner};
+
+#[cfg(not(feature = "skip_db_setup"))]
+pub struct PostgresTestContainer {
+    _container: ContainerAsync<postgres::Postgres>,
+    pub pool: Pool<Postgres>,
+}
+
+#[cfg(not(feature = "skip_db_setup"))]
+impl PostgresTestContainer {
+    /// Create a postgres instance for testing against
+    pub async fn new() -> Self {
+        let container = postgres::Postgres::default()
+            .with_tag("16")
+            .start()
+            .await
+            .unwrap();
+        let connection_string = format!(
+            "postgres://postgres:postgres@{host}:{port}/postgres",
+            host = container.get_host().await.unwrap(),
+            port = container.get_host_port_ipv4(5432).await.unwrap(),
+        );
+        let pool = PgPoolOptions::new()
+            .connect(&connection_string)
+            .await
+            .unwrap();
+        crate::setup::database::setup(&pool).await.unwrap();
+        Self {
+            _container: container,
+            pool,
+        }
+    }
+}
+
+#[cfg(not(feature = "skip_ms_setup"))]
+pub struct MeiliSearchTestContainer {
+    _container: ContainerAsync<GenericImage>,
+    pub client: Client,
+}
+
+#[cfg(not(feature = "skip_ms_setup"))]
+impl MeiliSearchTestContainer {
+    /// Create a meilisearch instance for testing against
+    pub async fn new() -> Self {
+        let container = GenericImage::new("getmeili/meilisearch", "v1.8.0")
+            .with_exposed_port(ContainerPort::Tcp(7700))
+            .with_wait_for(WaitFor::message_on_stderr(
+                "Actix runtime found; starting in Actix runtime",
+            ))
+            .start()
+            .await
+            .unwrap();
+        let meili_url = format!(
+            "http://{host}:{port}",
+            host = container.get_host().await.unwrap(),
+            port = container.get_host_port_ipv4(7700).await.unwrap(),
+        );
+
+        let client = Client::new(meili_url.clone(), None::<String>).unwrap();
+        super::meilisearch::setup(&client).await.unwrap();
+        Self {
+            _container: container,
+            client,
+        }
+    }
+}
+
+#[tokio::test]
+#[cfg(not(feature = "skip_db_setup"))]
+async fn test_db_setup() {
+    crate::setup_logging();
+    let pg = PostgresTestContainer::new().await;
+    crate::setup::database::load_data(&pg.pool).await.unwrap();
+}
+
+#[tokio::test]
+#[cfg(not(feature = "skip_ms_setup"))]
+async fn test_meilisearch_setup() {
+    crate::setup_logging();
+    let ms = MeiliSearchTestContainer::new().await;
+    crate::setup::meilisearch::load_data(&ms.client)
+        .await
+        .unwrap();
+}