From 32c6a98bcdd1b5ebc522438e8aba42256ee5a2d6 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 12 Nov 2023 21:31:10 +0100 Subject: [PATCH 1/7] merged the main api to also utilise postgres --- .../deployments/server-deployment.yaml | 14 ++- docker-compose.yml | 4 + ...a5f5c40a99660e0413e0a26853ecb5c21462e.json | 4 +- ...84af18f85a63d62f7d7497359d8e5fc2d4078.json | 12 --- ...b0be2fec596fdc768167a81082d0b5966fce4.json | 17 +++ ...7eea452d645c26f2d3c44460e669a835ca815.json | 12 --- ...b37f497aeb87f11eeafa13eadd4fefda706b5.json | 21 ++++ ...d161757d7ce6fcee59c23c23abbf45e803cb.json} | 32 +++--- ...a8d4ec920f8b46eaeaa37205c307b27b7332.json} | 16 +-- ...de7a221ce7d9bb8378028c695e1b665c55c5.json} | 12 ++- ...9b32f08c86626db9f4d63249dfc2cd77b80c.json} | 12 ++- ...1a495534817fb39a9e25f6720267c6025965.json} | 32 +++--- ...7b1473d24ffea9d69a8a96ca7bedf6aa2c6e4.json | 4 +- ...2d5a84a0774d47e1957da8eac576bedcfebcc.json | 12 --- ...583c9a0917f5203ccdc4ed01e4272fb68d1d8.json | 4 +- ...1af9a99868a7dee15c2f32ccca3c9b1c8b2b6.json | 21 ++++ server/Cargo.lock | 101 +++--------------- server/calendar/Cargo.toml | 1 - server/calendar/README.md | 5 +- server/calendar/diesel.toml | 5 +- .../down.sql | 6 -- .../up.sql | 36 ------- server/calendar/src/main.rs | 13 --- server/calendar/src/utils.rs | 9 +- server/main-api/Cargo.toml | 3 +- server/main-api/README.md | 6 ++ .../20230323144107_create_calendar.down.sql} | 0 .../20230323144107_create_calendar.up.sql} | 0 .../20230323144504_add_index.down.sql} | 0 .../20230323144504_add_index.up.sql} | 0 .../20230421143407_add_tumonline_id.down.sql} | 0 .../20230421143407_add_tumonline_id.up.sql} | 0 .../20230603173046_new_table_rooms.down.sql} | 0 .../20230603173046_new_table_rooms.up.sql} | 0 ...03173522_removal_of_tumonline_id.down.sql} | 0 ...0603173522_removal_of_tumonline_id.up.sql} | 0 .../20230902115254_base-schema.up.sql | 4 +- .../migrations/20230902115306_aliases.up.sql | 2 +- server/main-api/src/entries/get.rs | 12 +-- server/main-api/src/main.rs | 18 +++- server/main-api/src/maps/mod.rs | 8 +- server/main-api/src/models.rs | 2 +- server/main-api/src/setup/database/alias.rs | 19 ++-- server/main-api/src/setup/database/data.rs | 18 ++-- server/main-api/src/setup/database/mod.rs | 4 +- 45 files changed, 215 insertions(+), 286 deletions(-) delete mode 100644 server/.sqlx/query-3de656b156e0d049f115ec5260f84af18f85a63d62f7d7497359d8e5fc2d4078.json create mode 100644 server/.sqlx/query-5d4abea3319fe17022f23ee48f7b0be2fec596fdc768167a81082d0b5966fce4.json delete mode 100644 server/.sqlx/query-644bce09e28dd856187ca56660f7eea452d645c26f2d3c44460e669a835ca815.json create mode 100644 server/.sqlx/query-866f41f41ff9e68e92736009d98b37f497aeb87f11eeafa13eadd4fefda706b5.json rename server/.sqlx/{query-4824bfa1ca8109255aa81c5b9744c4222a914ce2d09a112c4dda259c425b3587.json => query-874f7f834952dc9935690debfad3d161757d7ce6fcee59c23c23abbf45e803cb.json} (75%) rename server/.sqlx/{query-c407c4d9fe1ce6bd90db434750df722d58068d3187a54cdf95dd8361db2d4c26.json => query-8874c6d22d9ef278efa392191441a8d4ec920f8b46eaeaa37205c307b27b7332.json} (71%) rename server/.sqlx/{query-ca4c11b14efa6564387df4dbc475539734b181798e2dba867f40bc460c7b9a3a.json => query-9f518183559171969ae448a0f3f0de7a221ce7d9bb8378028c695e1b665c55c5.json} (51%) rename server/.sqlx/{query-fe271671edc17aafa97e4da7ea8d7097d4186c3eb97649a45abc378fc8a2a83b.json => query-a16d239da728e396e283aea0782d9b32f08c86626db9f4d63249dfc2cd77b80c.json} (51%) rename server/.sqlx/{query-84ee8b091b1e17632363a2c7e4efdc195d4716f7857a99494918905ee5bfb868.json => query-a87d33b55872d9135ab82391b36d1a495534817fb39a9e25f6720267c6025965.json} (75%) delete mode 100644 server/.sqlx/query-b689ec21c7bcb38c8853d5146852d5a84a0774d47e1957da8eac576bedcfebcc.json create mode 100644 server/.sqlx/query-e08d77584b88d04d53710630fc11af9a99868a7dee15c2f32ccca3c9b1c8b2b6.json delete mode 100644 server/calendar/migrations/00000000000000_diesel_initial_setup/down.sql delete mode 100644 server/calendar/migrations/00000000000000_diesel_initial_setup/up.sql rename server/{calendar/migrations/2023-03-23-144107_create_calendar/down.sql => main-api/migrations/20230323144107_create_calendar.down.sql} (100%) rename server/{calendar/migrations/2023-03-23-144107_create_calendar/up.sql => main-api/migrations/20230323144107_create_calendar.up.sql} (100%) rename server/{calendar/migrations/2023-03-23-144504_add_index/down.sql => main-api/migrations/20230323144504_add_index.down.sql} (100%) rename server/{calendar/migrations/2023-03-23-144504_add_index/up.sql => main-api/migrations/20230323144504_add_index.up.sql} (100%) rename server/{calendar/migrations/2023-04-21-143407_add_tumonline_id/down.sql => main-api/migrations/20230421143407_add_tumonline_id.down.sql} (100%) rename server/{calendar/migrations/2023-04-21-143407_add_tumonline_id/up.sql => main-api/migrations/20230421143407_add_tumonline_id.up.sql} (100%) rename server/{calendar/migrations/2023-06-03-173046_new_table_rooms/down.sql => main-api/migrations/20230603173046_new_table_rooms.down.sql} (100%) rename server/{calendar/migrations/2023-06-03-173046_new_table_rooms/up.sql => main-api/migrations/20230603173046_new_table_rooms.up.sql} (100%) rename server/{calendar/migrations/2023-06-03-173522_removal_of_tumonline_id/down.sql => main-api/migrations/20230603173522_removal_of_tumonline_id.down.sql} (100%) rename server/{calendar/migrations/2023-06-03-173522_removal_of_tumonline_id/up.sql => main-api/migrations/20230603173522_removal_of_tumonline_id.up.sql} (100%) diff --git a/deployment/k3s/templates/deployments/server-deployment.yaml b/deployment/k3s/templates/deployments/server-deployment.yaml index 7aca8ca5e..706bccd6a 100644 --- a/deployment/k3s/templates/deployments/server-deployment.yaml +++ b/deployment/k3s/templates/deployments/server-deployment.yaml @@ -42,8 +42,18 @@ spec: name: server-api-keys # MEILI_MASTER_KEY {{ end }} env: - - name: DB_LOCATION - value: api_data.db + - name: POSTGRES_URL + value: postgres-rw.navigatum.svc.cluster.local + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: postgres-db-secret + key: user + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-db-secret + key: password ports: - containerPort: 3003 name: api diff --git a/docker-compose.yml b/docker-compose.yml index 993128aaa..1ed6f38a3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,6 +79,10 @@ services: user: 1000:3000 environment: MIELI_URL: http://meilisearch:7700 + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_URL: db:5432 depends_on: meilisearch: condition: service_healthy diff --git a/server/.sqlx/query-1883c95212e7119034b648e74c6a5f5c40a99660e0413e0a26853ecb5c21462e.json b/server/.sqlx/query-1883c95212e7119034b648e74c6a5f5c40a99660e0413e0a26853ecb5c21462e.json index 20d57b825..714a9f23d 100644 --- a/server/.sqlx/query-1883c95212e7119034b648e74c6a5f5c40a99660e0413e0a26853ecb5c21462e.json +++ b/server/.sqlx/query-1883c95212e7119034b648e74c6a5f5c40a99660e0413e0a26853ecb5c21462e.json @@ -1,10 +1,10 @@ { - "db_name": "SQLite", + "db_name": "PostgreSQL", "query": "DELETE FROM aliases", "describe": { "columns": [], "parameters": { - "Right": 0 + "Left": [] }, "nullable": [] }, diff --git a/server/.sqlx/query-3de656b156e0d049f115ec5260f84af18f85a63d62f7d7497359d8e5fc2d4078.json b/server/.sqlx/query-3de656b156e0d049f115ec5260f84af18f85a63d62f7d7497359d8e5fc2d4078.json deleted file mode 100644 index 7ad42436c..000000000 --- a/server/.sqlx/query-3de656b156e0d049f115ec5260f84af18f85a63d62f7d7497359d8e5fc2d4078.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "INSERT INTO de(key,data,name,tumonline_room_nr,type,type_common_name,lat,lon)\n VALUES (?,?,?,?,?,?,?,?)", - "describe": { - "columns": [], - "parameters": { - "Right": 8 - }, - "nullable": [] - }, - "hash": "3de656b156e0d049f115ec5260f84af18f85a63d62f7d7497359d8e5fc2d4078" -} diff --git a/server/.sqlx/query-5d4abea3319fe17022f23ee48f7b0be2fec596fdc768167a81082d0b5966fce4.json b/server/.sqlx/query-5d4abea3319fe17022f23ee48f7b0be2fec596fdc768167a81082d0b5966fce4.json new file mode 100644 index 000000000..63a671697 --- /dev/null +++ b/server/.sqlx/query-5d4abea3319fe17022f23ee48f7b0be2fec596fdc768167a81082d0b5966fce4.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO aliases (alias, key, type, visible_id)\n VALUES ($1, $2, $3, $4)\n ON CONFLICT (alias) DO UPDATE\n SET\n key = $2,\n type = $3,\n visible_id = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "5d4abea3319fe17022f23ee48f7b0be2fec596fdc768167a81082d0b5966fce4" +} diff --git a/server/.sqlx/query-644bce09e28dd856187ca56660f7eea452d645c26f2d3c44460e669a835ca815.json b/server/.sqlx/query-644bce09e28dd856187ca56660f7eea452d645c26f2d3c44460e669a835ca815.json deleted file mode 100644 index 75672349f..000000000 --- a/server/.sqlx/query-644bce09e28dd856187ca56660f7eea452d645c26f2d3c44460e669a835ca815.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "INSERT OR REPLACE INTO aliases (alias, key, type, visible_id)\n VALUES (?, ?, ?, ?)", - "describe": { - "columns": [], - "parameters": { - "Right": 4 - }, - "nullable": [] - }, - "hash": "644bce09e28dd856187ca56660f7eea452d645c26f2d3c44460e669a835ca815" -} diff --git a/server/.sqlx/query-866f41f41ff9e68e92736009d98b37f497aeb87f11eeafa13eadd4fefda706b5.json b/server/.sqlx/query-866f41f41ff9e68e92736009d98b37f497aeb87f11eeafa13eadd4fefda706b5.json new file mode 100644 index 000000000..25293b974 --- /dev/null +++ b/server/.sqlx/query-866f41f41ff9e68e92736009d98b37f497aeb87f11eeafa13eadd4fefda706b5.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO en(key,data,name,tumonline_room_nr,type,type_common_name,lat,lon)\n VALUES ($1,$2,$3,$4,$5,$6,$7,$8)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Int4", + "Text", + "Text", + "Float8", + "Float8" + ] + }, + "nullable": [] + }, + "hash": "866f41f41ff9e68e92736009d98b37f497aeb87f11eeafa13eadd4fefda706b5" +} diff --git a/server/.sqlx/query-4824bfa1ca8109255aa81c5b9744c4222a914ce2d09a112c4dda259c425b3587.json b/server/.sqlx/query-874f7f834952dc9935690debfad3d161757d7ce6fcee59c23c23abbf45e803cb.json similarity index 75% rename from server/.sqlx/query-4824bfa1ca8109255aa81c5b9744c4222a914ce2d09a112c4dda259c425b3587.json rename to server/.sqlx/query-874f7f834952dc9935690debfad3d161757d7ce6fcee59c23c23abbf45e803cb.json index 0db008ca7..37694e0dc 100644 --- a/server/.sqlx/query-4824bfa1ca8109255aa81c5b9744c4222a914ce2d09a112c4dda259c425b3587.json +++ b/server/.sqlx/query-874f7f834952dc9935690debfad3d161757d7ce6fcee59c23c23abbf45e803cb.json @@ -1,51 +1,53 @@ { - "db_name": "SQLite", - "query": "SELECT * FROM de WHERE key = ?", + "db_name": "PostgreSQL", + "query": "SELECT * FROM en WHERE key = $1", "describe": { "columns": [ { - "name": "key", "ordinal": 0, + "name": "key", "type_info": "Text" }, { - "name": "name", "ordinal": 1, + "name": "name", "type_info": "Text" }, { - "name": "tumonline_room_nr", "ordinal": 2, - "type_info": "Int64" + "name": "tumonline_room_nr", + "type_info": "Int4" }, { - "name": "type", "ordinal": 3, + "name": "type", "type_info": "Text" }, { - "name": "type_common_name", "ordinal": 4, + "name": "type_common_name", "type_info": "Text" }, { - "name": "lat", "ordinal": 5, - "type_info": "Float" + "name": "lat", + "type_info": "Float8" }, { - "name": "lon", "ordinal": 6, - "type_info": "Float" + "name": "lon", + "type_info": "Float8" }, { - "name": "data", "ordinal": 7, + "name": "data", "type_info": "Text" } ], "parameters": { - "Right": 1 + "Left": [ + "Text" + ] }, "nullable": [ false, @@ -58,5 +60,5 @@ false ] }, - "hash": "4824bfa1ca8109255aa81c5b9744c4222a914ce2d09a112c4dda259c425b3587" + "hash": "874f7f834952dc9935690debfad3d161757d7ce6fcee59c23c23abbf45e803cb" } diff --git a/server/.sqlx/query-c407c4d9fe1ce6bd90db434750df722d58068d3187a54cdf95dd8361db2d4c26.json b/server/.sqlx/query-8874c6d22d9ef278efa392191441a8d4ec920f8b46eaeaa37205c307b27b7332.json similarity index 71% rename from server/.sqlx/query-c407c4d9fe1ce6bd90db434750df722d58068d3187a54cdf95dd8361db2d4c26.json rename to server/.sqlx/query-8874c6d22d9ef278efa392191441a8d4ec920f8b46eaeaa37205c307b27b7332.json index 8dd5185d2..582868ed8 100644 --- a/server/.sqlx/query-c407c4d9fe1ce6bd90db434750df722d58068d3187a54cdf95dd8361db2d4c26.json +++ b/server/.sqlx/query-8874c6d22d9ef278efa392191441a8d4ec920f8b46eaeaa37205c307b27b7332.json @@ -1,26 +1,28 @@ { - "db_name": "SQLite", - "query": "\n SELECT key, visible_id, type\n FROM aliases\n WHERE key = ? OR key = ?\n ", + "db_name": "PostgreSQL", + "query": "\n SELECT key, visible_id, type\n FROM aliases\n WHERE key = $1 OR key = $1 ", "describe": { "columns": [ { - "name": "key", "ordinal": 0, + "name": "key", "type_info": "Text" }, { - "name": "visible_id", "ordinal": 1, + "name": "visible_id", "type_info": "Text" }, { - "name": "type", "ordinal": 2, + "name": "type", "type_info": "Text" } ], "parameters": { - "Right": 2 + "Left": [ + "Text" + ] }, "nullable": [ false, @@ -28,5 +30,5 @@ false ] }, - "hash": "c407c4d9fe1ce6bd90db434750df722d58068d3187a54cdf95dd8361db2d4c26" + "hash": "8874c6d22d9ef278efa392191441a8d4ec920f8b46eaeaa37205c307b27b7332" } diff --git a/server/.sqlx/query-ca4c11b14efa6564387df4dbc475539734b181798e2dba867f40bc460c7b9a3a.json b/server/.sqlx/query-9f518183559171969ae448a0f3f0de7a221ce7d9bb8378028c695e1b665c55c5.json similarity index 51% rename from server/.sqlx/query-ca4c11b14efa6564387df4dbc475539734b181798e2dba867f40bc460c7b9a3a.json rename to server/.sqlx/query-9f518183559171969ae448a0f3f0de7a221ce7d9bb8378028c695e1b665c55c5.json index 13d4ee508..ebe2808cc 100644 --- a/server/.sqlx/query-ca4c11b14efa6564387df4dbc475539734b181798e2dba867f40bc460c7b9a3a.json +++ b/server/.sqlx/query-9f518183559171969ae448a0f3f0de7a221ce7d9bb8378028c695e1b665c55c5.json @@ -1,20 +1,22 @@ { - "db_name": "SQLite", - "query": "SELECT data FROM en WHERE key = ?", + "db_name": "PostgreSQL", + "query": "SELECT data FROM de WHERE key = $1", "describe": { "columns": [ { - "name": "data", "ordinal": 0, + "name": "data", "type_info": "Text" } ], "parameters": { - "Right": 1 + "Left": [ + "Text" + ] }, "nullable": [ false ] }, - "hash": "ca4c11b14efa6564387df4dbc475539734b181798e2dba867f40bc460c7b9a3a" + "hash": "9f518183559171969ae448a0f3f0de7a221ce7d9bb8378028c695e1b665c55c5" } diff --git a/server/.sqlx/query-fe271671edc17aafa97e4da7ea8d7097d4186c3eb97649a45abc378fc8a2a83b.json b/server/.sqlx/query-a16d239da728e396e283aea0782d9b32f08c86626db9f4d63249dfc2cd77b80c.json similarity index 51% rename from server/.sqlx/query-fe271671edc17aafa97e4da7ea8d7097d4186c3eb97649a45abc378fc8a2a83b.json rename to server/.sqlx/query-a16d239da728e396e283aea0782d9b32f08c86626db9f4d63249dfc2cd77b80c.json index 3809e3cac..8b2ce6edc 100644 --- a/server/.sqlx/query-fe271671edc17aafa97e4da7ea8d7097d4186c3eb97649a45abc378fc8a2a83b.json +++ b/server/.sqlx/query-a16d239da728e396e283aea0782d9b32f08c86626db9f4d63249dfc2cd77b80c.json @@ -1,20 +1,22 @@ { - "db_name": "SQLite", - "query": "SELECT data FROM de WHERE key = ?", + "db_name": "PostgreSQL", + "query": "SELECT data FROM en WHERE key = $1", "describe": { "columns": [ { - "name": "data", "ordinal": 0, + "name": "data", "type_info": "Text" } ], "parameters": { - "Right": 1 + "Left": [ + "Text" + ] }, "nullable": [ false ] }, - "hash": "fe271671edc17aafa97e4da7ea8d7097d4186c3eb97649a45abc378fc8a2a83b" + "hash": "a16d239da728e396e283aea0782d9b32f08c86626db9f4d63249dfc2cd77b80c" } diff --git a/server/.sqlx/query-84ee8b091b1e17632363a2c7e4efdc195d4716f7857a99494918905ee5bfb868.json b/server/.sqlx/query-a87d33b55872d9135ab82391b36d1a495534817fb39a9e25f6720267c6025965.json similarity index 75% rename from server/.sqlx/query-84ee8b091b1e17632363a2c7e4efdc195d4716f7857a99494918905ee5bfb868.json rename to server/.sqlx/query-a87d33b55872d9135ab82391b36d1a495534817fb39a9e25f6720267c6025965.json index 7863b66c5..9357fbca4 100644 --- a/server/.sqlx/query-84ee8b091b1e17632363a2c7e4efdc195d4716f7857a99494918905ee5bfb868.json +++ b/server/.sqlx/query-a87d33b55872d9135ab82391b36d1a495534817fb39a9e25f6720267c6025965.json @@ -1,51 +1,53 @@ { - "db_name": "SQLite", - "query": "SELECT * FROM en WHERE key = ?", + "db_name": "PostgreSQL", + "query": "SELECT * FROM de WHERE key = $1", "describe": { "columns": [ { - "name": "key", "ordinal": 0, + "name": "key", "type_info": "Text" }, { - "name": "name", "ordinal": 1, + "name": "name", "type_info": "Text" }, { - "name": "tumonline_room_nr", "ordinal": 2, - "type_info": "Int64" + "name": "tumonline_room_nr", + "type_info": "Int4" }, { - "name": "type", "ordinal": 3, + "name": "type", "type_info": "Text" }, { - "name": "type_common_name", "ordinal": 4, + "name": "type_common_name", "type_info": "Text" }, { - "name": "lat", "ordinal": 5, - "type_info": "Float" + "name": "lat", + "type_info": "Float8" }, { - "name": "lon", "ordinal": 6, - "type_info": "Float" + "name": "lon", + "type_info": "Float8" }, { - "name": "data", "ordinal": 7, + "name": "data", "type_info": "Text" } ], "parameters": { - "Right": 1 + "Left": [ + "Text" + ] }, "nullable": [ false, @@ -58,5 +60,5 @@ false ] }, - "hash": "84ee8b091b1e17632363a2c7e4efdc195d4716f7857a99494918905ee5bfb868" + "hash": "a87d33b55872d9135ab82391b36d1a495534817fb39a9e25f6720267c6025965" } diff --git a/server/.sqlx/query-b59dbe8a5c3b6f7fdb9543508ae7b1473d24ffea9d69a8a96ca7bedf6aa2c6e4.json b/server/.sqlx/query-b59dbe8a5c3b6f7fdb9543508ae7b1473d24ffea9d69a8a96ca7bedf6aa2c6e4.json index 094bfa433..e925b25fc 100644 --- a/server/.sqlx/query-b59dbe8a5c3b6f7fdb9543508ae7b1473d24ffea9d69a8a96ca7bedf6aa2c6e4.json +++ b/server/.sqlx/query-b59dbe8a5c3b6f7fdb9543508ae7b1473d24ffea9d69a8a96ca7bedf6aa2c6e4.json @@ -1,10 +1,10 @@ { - "db_name": "SQLite", + "db_name": "PostgreSQL", "query": "DELETE FROM en", "describe": { "columns": [], "parameters": { - "Right": 0 + "Left": [] }, "nullable": [] }, diff --git a/server/.sqlx/query-b689ec21c7bcb38c8853d5146852d5a84a0774d47e1957da8eac576bedcfebcc.json b/server/.sqlx/query-b689ec21c7bcb38c8853d5146852d5a84a0774d47e1957da8eac576bedcfebcc.json deleted file mode 100644 index bd64941cb..000000000 --- a/server/.sqlx/query-b689ec21c7bcb38c8853d5146852d5a84a0774d47e1957da8eac576bedcfebcc.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "INSERT INTO en(key,data,name,tumonline_room_nr,type,type_common_name,lat,lon)\n VALUES (?,?,?,?,?,?,?,?)", - "describe": { - "columns": [], - "parameters": { - "Right": 8 - }, - "nullable": [] - }, - "hash": "b689ec21c7bcb38c8853d5146852d5a84a0774d47e1957da8eac576bedcfebcc" -} diff --git a/server/.sqlx/query-b85ef040a74d51bf6b539703028583c9a0917f5203ccdc4ed01e4272fb68d1d8.json b/server/.sqlx/query-b85ef040a74d51bf6b539703028583c9a0917f5203ccdc4ed01e4272fb68d1d8.json index 96281fc03..6de4edc5e 100644 --- a/server/.sqlx/query-b85ef040a74d51bf6b539703028583c9a0917f5203ccdc4ed01e4272fb68d1d8.json +++ b/server/.sqlx/query-b85ef040a74d51bf6b539703028583c9a0917f5203ccdc4ed01e4272fb68d1d8.json @@ -1,10 +1,10 @@ { - "db_name": "SQLite", + "db_name": "PostgreSQL", "query": "DELETE FROM de", "describe": { "columns": [], "parameters": { - "Right": 0 + "Left": [] }, "nullable": [] }, diff --git a/server/.sqlx/query-e08d77584b88d04d53710630fc11af9a99868a7dee15c2f32ccca3c9b1c8b2b6.json b/server/.sqlx/query-e08d77584b88d04d53710630fc11af9a99868a7dee15c2f32ccca3c9b1c8b2b6.json new file mode 100644 index 000000000..c02d9fc75 --- /dev/null +++ b/server/.sqlx/query-e08d77584b88d04d53710630fc11af9a99868a7dee15c2f32ccca3c9b1c8b2b6.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO de(key,data,name,tumonline_room_nr,type,type_common_name,lat,lon)\n VALUES ($1,$2,$3,$4,$5,$6,$7,$8)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Int4", + "Text", + "Text", + "Float8", + "Float8" + ] + }, + "nullable": [] + }, + "hash": "e08d77584b88d04d53710630fc11af9a99868a7dee15c2f32ccca3c9b1c8b2b6" +} diff --git a/server/Cargo.lock b/server/Cargo.lock index e0a9ce7b1..2854c99c1 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -871,17 +871,6 @@ dependencies = [ "syn 2.0.31", ] -[[package]] -name = "diesel_migrations" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6036b3f0120c5961381b570ee20a02432d7e2d27ea60de9578799cf9156914ac" -dependencies = [ - "diesel", - "migrations_internals", - "migrations_macros", -] - [[package]] name = "diesel_table_macro_syntax" version = "0.1.0" @@ -1957,27 +1946,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "migrations_internals" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f23f71580015254b020e856feac3df5878c2c7a8812297edd6c0a485ac9dada" -dependencies = [ - "serde", - "toml", -] - -[[package]] -name = "migrations_macros" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cce3325ac70e67bbab5bd837a31cae01f1a6db64e0e744a33cb03a543469ef08" -dependencies = [ - "migrations_internals", - "proc-macro2", - "quote", -] - [[package]] name = "mime" version = "0.3.17" @@ -2072,7 +2040,6 @@ dependencies = [ "actix-web-prom", "chrono", "diesel", - "diesel_migrations", "futures", "lazy_static", "log", @@ -2127,7 +2094,6 @@ dependencies = [ "image", "imageproc", "lazy_static", - "libsqlite3-sys", "log", "logos", "meilisearch-sdk", @@ -2826,7 +2792,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots", + "webpki-roots 0.25.2", "winreg", ] @@ -3107,15 +3073,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_spanned" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" -dependencies = [ - "serde", -] - [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3370,6 +3327,8 @@ dependencies = [ "once_cell", "paste", "percent-encoding", + "rustls", + "rustls-pemfile", "serde", "serde_json", "sha2", @@ -3380,6 +3339,7 @@ dependencies = [ "tokio-stream", "tracing", "url", + "webpki-roots 0.24.0", ] [[package]] @@ -3413,6 +3373,7 @@ dependencies = [ "sha2", "sqlx-core", "sqlx-mysql", + "sqlx-postgres", "sqlx-sqlite", "syn 1.0.109", "tempfile", @@ -3847,40 +3808,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "toml" -version = "0.7.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de0a3ab2091e52d7299a39d098e200114a972df0a7724add02a273aa9aada592" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - -[[package]] -name = "toml_datetime" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.19.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" -dependencies = [ - "indexmap 2.0.0", - "serde", - "serde_spanned", - "toml_datetime", - "winnow", -] - [[package]] name = "tower" version = "0.4.13" @@ -4239,6 +4166,15 @@ dependencies = [ "libwebp-sys", ] +[[package]] +name = "webpki-roots" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b291546d5d9d1eab74f069c77749f2cb8504a12caa20f0f2de93ddbf6f411888" +dependencies = [ + "rustls-webpki", +] + [[package]] name = "webpki-roots" version = "0.25.2" @@ -4364,15 +4300,6 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" -[[package]] -name = "winnow" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" -dependencies = [ - "memchr", -] - [[package]] name = "winreg" version = "0.50.0" diff --git a/server/calendar/Cargo.toml b/server/calendar/Cargo.toml index e6106fdf5..8b68fbe0e 100644 --- a/server/calendar/Cargo.toml +++ b/server/calendar/Cargo.toml @@ -32,7 +32,6 @@ chrono = { version = "0.4.31", default-features = false, features = ["serde"] } # database diesel = { version = "2.1.3", features = ["default", "chrono", "postgres"] } -diesel_migrations = "2.1.0" # metrics lazy_static = "1.4.0" diff --git a/server/calendar/README.md b/server/calendar/README.md index 62c460cec..f452904b8 100644 --- a/server/calendar/README.md +++ b/server/calendar/README.md @@ -22,10 +22,7 @@ docker run -p 5432:5432 -e POSTGRES_PASSWORD=password postgres Any migrations this database needs are applied on first run of the server. In order to change the connection parameters that the calendar server uses you can use the environment variables: -- `POSTGRES_USER` (default `postgres`) -- `POSTGRES_PASSWORD` (default `password`) -- `POSTGRES_URL` (default `localhost`, specify the port if different from 5432) -- `POSTGRES_DB` (default: same as `POSTGRES_USER`) +- `DATABASE_URL` (default `postgres`) ### Starting the server diff --git a/server/calendar/diesel.toml b/server/calendar/diesel.toml index 35a12ff0d..9f7d63dad 100644 --- a/server/calendar/diesel.toml +++ b/server/calendar/diesel.toml @@ -2,7 +2,4 @@ # see https://diesel.rs/guides/configuring-diesel-cli [print_schema] -file = "src/schema.rs" - -[migrations_directory] -dir = "migrations" +file = "src/schema.rs" \ No newline at end of file diff --git a/server/calendar/migrations/00000000000000_diesel_initial_setup/down.sql b/server/calendar/migrations/00000000000000_diesel_initial_setup/down.sql deleted file mode 100644 index a9f526091..000000000 --- a/server/calendar/migrations/00000000000000_diesel_initial_setup/down.sql +++ /dev/null @@ -1,6 +0,0 @@ --- This file was automatically created by Diesel to setup helper functions --- and other internal bookkeeping. This file is safe to edit, any future --- changes will be added to existing projects as new migrations. - -DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass); -DROP FUNCTION IF EXISTS diesel_set_updated_at(); diff --git a/server/calendar/migrations/00000000000000_diesel_initial_setup/up.sql b/server/calendar/migrations/00000000000000_diesel_initial_setup/up.sql deleted file mode 100644 index d68895b1a..000000000 --- a/server/calendar/migrations/00000000000000_diesel_initial_setup/up.sql +++ /dev/null @@ -1,36 +0,0 @@ --- This file was automatically created by Diesel to setup helper functions --- and other internal bookkeeping. This file is safe to edit, any future --- changes will be added to existing projects as new migrations. - - - - --- Sets up a trigger for the given table to automatically set a column called --- `updated_at` whenever the row is modified (unless `updated_at` was included --- in the modified columns) --- --- # Example --- --- ```sql --- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW()); --- --- SELECT diesel_manage_updated_at('users'); --- ``` -CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$ -BEGIN - EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s - FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl); -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$ -BEGIN - IF ( - NEW IS DISTINCT FROM OLD AND - NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at - ) THEN - NEW.updated_at := current_timestamp; - END IF; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; diff --git a/server/calendar/src/main.rs b/server/calendar/src/main.rs index f26aa1bae..d9d978584 100644 --- a/server/calendar/src/main.rs +++ b/server/calendar/src/main.rs @@ -6,8 +6,6 @@ mod utils; use actix_cors::Cors; use actix_web::{get, middleware, web, App, HttpResponse, HttpServer}; use actix_web_prom::PrometheusMetricsBuilder; -use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; -use log::info; use std::collections::HashMap; use std::error::Error; use structured_logger::async_json::new_writer; @@ -26,22 +24,11 @@ async fn health_status_handler() -> HttpResponse { .body(format!("healthy\nsource_code: {github_link}")) } -pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations"); - -fn apply_db_migrations() -> Result<(), Box> { - info!("Applying database migrations"); - let con = &mut utils::establish_connection(); - con.run_pending_migrations(MIGRATIONS)?; - info!("database migrations applied"); - Ok(()) -} - #[tokio::main] async fn main() -> Result<(), Box> { Builder::with_level("info") .with_target_writer("*", new_writer(tokio::io::stdout())) .init(); - apply_db_migrations()?; // metrics let labels = HashMap::from([( diff --git a/server/calendar/src/utils.rs b/server/calendar/src/utils.rs index 5a20aaf3a..3ca0a4b3c 100644 --- a/server/calendar/src/utils.rs +++ b/server/calendar/src/utils.rs @@ -1,10 +1,11 @@ use diesel::{Connection, PgConnection}; - -pub fn establish_connection() -> PgConnection { +fn connection_string() -> String { let username = std::env::var("POSTGRES_USER").unwrap_or_else(|_| "postgres".to_string()); let password = std::env::var("POSTGRES_PASSWORD").unwrap_or_else(|_| "password".to_string()); let url = std::env::var("POSTGRES_URL").unwrap_or_else(|_| "localhost".to_string()); let db = std::env::var("POSTGRES_DB").unwrap_or_else(|_| username.clone()); - let connection_string = format!("postgres://{username}:{password}@{url}/{db}"); - PgConnection::establish(&connection_string).expect("Cannot open database") + format!("postgres://{username}:{password}@{url}/{db}") +} +pub fn establish_connection() -> PgConnection { + PgConnection::establish(&connection_string()).expect("Cannot open database") } diff --git a/server/main-api/Cargo.toml b/server/main-api/Cargo.toml index 216b6351a..304e901d5 100644 --- a/server/main-api/Cargo.toml +++ b/server/main-api/Cargo.toml @@ -33,8 +33,7 @@ unicode-truncate = "0.2.0" serde_yaml = "0.9" # database -libsqlite3-sys = { version = "*", features = ["bundled"] } -sqlx = { version = "0.7.2", features = ["sqlite", "runtime-tokio", "migrate", "macros"] } +sqlx = { version = "0.7.2", features = ["postgres", "runtime-tokio-rustls", "migrate", "macros"] } # search meilisearch-sdk = "0.24.3" diff --git a/server/main-api/README.md b/server/main-api/README.md index 138305d5d..7aeeebb70 100644 --- a/server/main-api/README.md +++ b/server/main-api/README.md @@ -30,6 +30,12 @@ docker run -it --rm -p 7700:7700 getmeili/meilisearch:latest MeiliSearch provides an interactive interface at . +To set up the Postgres, run the following command: + +```bash +docker run --name postgres -e POSTGRES_PASSWORD=password -p 5432:5432 postgres +``` + ### Starting the server Run `cargo run` to start the server. diff --git a/server/calendar/migrations/2023-03-23-144107_create_calendar/down.sql b/server/main-api/migrations/20230323144107_create_calendar.down.sql similarity index 100% rename from server/calendar/migrations/2023-03-23-144107_create_calendar/down.sql rename to server/main-api/migrations/20230323144107_create_calendar.down.sql diff --git a/server/calendar/migrations/2023-03-23-144107_create_calendar/up.sql b/server/main-api/migrations/20230323144107_create_calendar.up.sql similarity index 100% rename from server/calendar/migrations/2023-03-23-144107_create_calendar/up.sql rename to server/main-api/migrations/20230323144107_create_calendar.up.sql diff --git a/server/calendar/migrations/2023-03-23-144504_add_index/down.sql b/server/main-api/migrations/20230323144504_add_index.down.sql similarity index 100% rename from server/calendar/migrations/2023-03-23-144504_add_index/down.sql rename to server/main-api/migrations/20230323144504_add_index.down.sql diff --git a/server/calendar/migrations/2023-03-23-144504_add_index/up.sql b/server/main-api/migrations/20230323144504_add_index.up.sql similarity index 100% rename from server/calendar/migrations/2023-03-23-144504_add_index/up.sql rename to server/main-api/migrations/20230323144504_add_index.up.sql diff --git a/server/calendar/migrations/2023-04-21-143407_add_tumonline_id/down.sql b/server/main-api/migrations/20230421143407_add_tumonline_id.down.sql similarity index 100% rename from server/calendar/migrations/2023-04-21-143407_add_tumonline_id/down.sql rename to server/main-api/migrations/20230421143407_add_tumonline_id.down.sql diff --git a/server/calendar/migrations/2023-04-21-143407_add_tumonline_id/up.sql b/server/main-api/migrations/20230421143407_add_tumonline_id.up.sql similarity index 100% rename from server/calendar/migrations/2023-04-21-143407_add_tumonline_id/up.sql rename to server/main-api/migrations/20230421143407_add_tumonline_id.up.sql diff --git a/server/calendar/migrations/2023-06-03-173046_new_table_rooms/down.sql b/server/main-api/migrations/20230603173046_new_table_rooms.down.sql similarity index 100% rename from server/calendar/migrations/2023-06-03-173046_new_table_rooms/down.sql rename to server/main-api/migrations/20230603173046_new_table_rooms.down.sql diff --git a/server/calendar/migrations/2023-06-03-173046_new_table_rooms/up.sql b/server/main-api/migrations/20230603173046_new_table_rooms.up.sql similarity index 100% rename from server/calendar/migrations/2023-06-03-173046_new_table_rooms/up.sql rename to server/main-api/migrations/20230603173046_new_table_rooms.up.sql diff --git a/server/calendar/migrations/2023-06-03-173522_removal_of_tumonline_id/down.sql b/server/main-api/migrations/20230603173522_removal_of_tumonline_id.down.sql similarity index 100% rename from server/calendar/migrations/2023-06-03-173522_removal_of_tumonline_id/down.sql rename to server/main-api/migrations/20230603173522_removal_of_tumonline_id.down.sql diff --git a/server/calendar/migrations/2023-06-03-173522_removal_of_tumonline_id/up.sql b/server/main-api/migrations/20230603173522_removal_of_tumonline_id.up.sql similarity index 100% rename from server/calendar/migrations/2023-06-03-173522_removal_of_tumonline_id/up.sql rename to server/main-api/migrations/20230603173522_removal_of_tumonline_id.up.sql diff --git a/server/main-api/migrations/20230902115254_base-schema.up.sql b/server/main-api/migrations/20230902115254_base-schema.up.sql index 9cf9685c7..2bf8869c9 100644 --- a/server/main-api/migrations/20230902115254_base-schema.up.sql +++ b/server/main-api/migrations/20230902115254_base-schema.up.sql @@ -3,7 +3,7 @@ CREATE TABLE de ( key TEXT UNIQUE PRIMARY KEY NOT NULL, name TEXT NOT NULL, - tumonline_room_nr INTEGER NULLABLE, -- used for calendars + tumonline_room_nr INTEGER, -- used for calendars type TEXT NOT NULL, type_common_name TEXT NOT NULL, lat FLOAT NOT NULL, @@ -14,7 +14,7 @@ CREATE TABLE en ( key TEXT UNIQUE PRIMARY KEY NOT NULL, name TEXT NOT NULL, - tumonline_room_nr INTEGER NULLABLE, -- used for calendars + tumonline_room_nr INTEGER, -- used for calendars type TEXT NOT NULL, type_common_name TEXT NOT NULL, lat FLOAT NOT NULL, diff --git a/server/main-api/migrations/20230902115306_aliases.up.sql b/server/main-api/migrations/20230902115306_aliases.up.sql index d2e67795b..79e69fdf0 100644 --- a/server/main-api/migrations/20230902115306_aliases.up.sql +++ b/server/main-api/migrations/20230902115306_aliases.up.sql @@ -1,7 +1,7 @@ -- Add up migration script here CREATE TABLE aliases ( - id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + id SERIAL PRIMARY KEY NOT NULL, alias TEXT NOT NULL, key TEXT NOT NULL, visible_id TEXT NOT NULL, diff --git a/server/main-api/src/entries/get.rs b/server/main-api/src/entries/get.rs index 7d884329e..945338ffa 100644 --- a/server/main-api/src/entries/get.rs +++ b/server/main-api/src/entries/get.rs @@ -2,7 +2,7 @@ use crate::models::DBRoomKeyAlias; use crate::utils; use actix_web::{get, web, HttpResponse}; use log::error; -use sqlx::SqlitePool; +use sqlx::PgPool; #[get("/api/get/{id}")] pub async fn get_handler( @@ -17,12 +17,12 @@ pub async fn get_handler( }; let result = match args.should_use_english() { true => { - sqlx::query_scalar!("SELECT data FROM en WHERE key = ?", probable_id) + sqlx::query_scalar!("SELECT data FROM en WHERE key = $1", probable_id) .fetch_optional(&data.db) .await } false => { - sqlx::query_scalar!("SELECT data FROM de WHERE key = ?", probable_id) + sqlx::query_scalar!("SELECT data FROM de WHERE key = $1", probable_id) .fetch_optional(&data.db) .await } @@ -50,15 +50,13 @@ pub async fn get_handler( } } -async fn get_alias_and_redirect(conn: &SqlitePool, query: &str) -> Option<(String, String)> { +async fn get_alias_and_redirect(conn: &PgPool, query: &str) -> Option<(String, String)> { let result = sqlx::query_as!( DBRoomKeyAlias, r#" SELECT key, visible_id, type FROM aliases - WHERE key = ? OR key = ? - "#, - query, + WHERE key = $1 OR key = $1 "#, query ) .fetch_all(conn) diff --git a/server/main-api/src/main.rs b/server/main-api/src/main.rs index 63a4ebf44..7ccc2be10 100644 --- a/server/main-api/src/main.rs +++ b/server/main-api/src/main.rs @@ -3,8 +3,8 @@ use actix_web::{get, middleware, web, App, HttpResponse, HttpServer}; use actix_web_prom::PrometheusMetricsBuilder; use log::{debug, error, info}; use sqlx::prelude::*; -use sqlx::sqlite::SqlitePoolOptions; -use sqlx::SqlitePool; +use sqlx::postgres::PgPoolOptions; +use sqlx::PgPool; use std::collections::HashMap; use structured_logger::async_json::new_writer; use structured_logger::Builder; @@ -20,7 +20,7 @@ const MAX_JSON_PAYLOAD: usize = 1024 * 1024; // 1 MB #[derive(Clone, Debug)] pub struct AppData { - db: SqlitePool, + db: PgPool, } #[get("/api/status")] @@ -42,13 +42,21 @@ async fn health_status_handler(data: web::Data) -> HttpResponse { }; } +fn connection_string() -> String { + let username = std::env::var("POSTGRES_USER").unwrap_or_else(|_| "postgres".to_string()); + let password = std::env::var("POSTGRES_PASSWORD").unwrap_or_else(|_| "password".to_string()); + let url = std::env::var("POSTGRES_URL").unwrap_or_else(|_| "localhost".to_string()); + let db = std::env::var("POSTGRES_DB").unwrap_or_else(|_| username.clone()); + format!("postgres://{username}:{password}@{url}/{db}") +} + #[tokio::main] async fn main() -> Result<(), Box> { Builder::with_level("info") .with_target_writer("*", new_writer(tokio::io::stdout())) .init(); - let uri = std::env::var("DB_LOCATION").unwrap_or_else(|_| "api_data.db".to_string()); - let pool = SqlitePoolOptions::new().connect(&uri).await?; + let uri=connection_string(); + let pool = PgPoolOptions::new().connect(&uri).await?; info!("setting up the database"); setup::database::setup_database(&pool).await?; info!("setting up meilisearch"); diff --git a/server/main-api/src/maps/mod.rs b/server/main-api/src/maps/mod.rs index f234db364..152192e61 100644 --- a/server/main-api/src/maps/mod.rs +++ b/server/main-api/src/maps/mod.rs @@ -12,7 +12,7 @@ use image::Rgba; use std::io::Cursor; use log::{debug, error, warn}; -use sqlx::SqlitePool; +use sqlx::PgPool; use tokio::time::Instant; use unicode_truncate::UnicodeTruncateStr; @@ -28,16 +28,16 @@ pub fn configure(cfg: &mut web::ServiceConfig) { } async fn get_localised_data( - conn: &SqlitePool, + conn: &PgPool, id: &str, should_use_english: bool, ) -> Result { let result = if should_use_english { - sqlx::query_as!(DBRoomEntry, "SELECT * FROM en WHERE key = ?", id) + sqlx::query_as!(DBRoomEntry, "SELECT * FROM en WHERE key = $1", id) .fetch_all(conn) .await } else { - sqlx::query_as!(DBRoomEntry, "SELECT * FROM de WHERE key = ?", id) + sqlx::query_as!(DBRoomEntry, "SELECT * FROM de WHERE key = $1", id) .fetch_all(conn) .await }; diff --git a/server/main-api/src/models.rs b/server/main-api/src/models.rs index af8ad25ff..40b0b6239 100644 --- a/server/main-api/src/models.rs +++ b/server/main-api/src/models.rs @@ -2,7 +2,7 @@ pub struct DBRoomEntry { pub key: String, pub name: String, - pub tumonline_room_nr: Option, + pub tumonline_room_nr: Option, pub r#type: String, pub type_common_name: String, pub lat: f64, diff --git a/server/main-api/src/setup/database/alias.rs b/server/main-api/src/setup/database/alias.rs index e9270d876..034846705 100644 --- a/server/main-api/src/setup/database/alias.rs +++ b/server/main-api/src/setup/database/alias.rs @@ -1,6 +1,6 @@ use log::info; use serde::Deserialize; -use sqlx::SqlitePool; +use sqlx::PgPool; use std::time::Instant; #[derive(Debug)] @@ -84,22 +84,27 @@ impl Iterator for AliasIterator { impl Alias { async fn store( self, - tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, - ) -> Result { + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + ) -> Result { sqlx::query!( - r#"INSERT OR REPLACE INTO aliases (alias, key, type, visible_id) - VALUES (?, ?, ?, ?)"#, + r#"INSERT INTO aliases (alias, key, type, visible_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT (alias) DO UPDATE + SET + key = $2, + type = $3, + visible_id = $4"#, self.alias, self.key, self.r#type, - self.visible_id + self.visible_id, ) .execute(&mut **tx) .await } } -pub(crate) async fn load_all_to_db(pool: &SqlitePool) -> Result<(), Box> { +pub(crate) async fn load_all_to_db(pool: &PgPool) -> Result<(), Box> { let cdn_url = std::env::var("CDN_URL").unwrap_or_else(|_| "https://nav.tum.de/cdn".to_string()); let raw_aliase = reqwest::get(format!("{cdn_url}/api_data.json")) .await? diff --git a/server/main-api/src/setup/database/data.rs b/server/main-api/src/setup/database/data.rs index 9562b5c16..222300986 100644 --- a/server/main-api/src/setup/database/data.rs +++ b/server/main-api/src/setup/database/data.rs @@ -1,6 +1,6 @@ use log::info; use serde_json::Value; -use sqlx::SqlitePool; +use sqlx::PgPool; use std::collections::HashMap; use std::time::Instant; @@ -9,8 +9,8 @@ struct ExtractedFields { tumonline_room_nr: Option, r#type: String, type_common_name: String, - lat: f32, - lon: f32, + lat: f64, + lon: f64, } impl From> for ExtractedFields { fn from(obj: HashMap) -> Self { @@ -37,8 +37,8 @@ impl From> for ExtractedFields { .as_str() .unwrap() .to_string(), - lat: lat.unwrap_or(48.14903) as f32, - lon: lon.unwrap_or(11.56735) as f32, + lat: lat.unwrap_or(48.14903) as f64, + lon: lon.unwrap_or(11.56735) as f64, } } } @@ -108,12 +108,12 @@ impl From> for DelocalisedValues { } impl DelocalisedValues { - async fn store(self, tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>) -> Result<(), sqlx::Error> { + async fn store(self, tx: &mut sqlx::Transaction<'_, sqlx::Postgres>) -> Result<(), sqlx::Error> { let key = self.key.clone(); // has to be here due to livetimes somehow let (data, fields) = StorableValue::from(self.de); sqlx::query!( r#"INSERT INTO de(key,data,name,tumonline_room_nr,type,type_common_name,lat,lon) - VALUES (?,?,?,?,?,?,?,?)"#, + VALUES ($1,$2,$3,$4,$5,$6,$7,$8)"#, key, data, fields.name, @@ -129,7 +129,7 @@ impl DelocalisedValues { let (data, fields) = StorableValue::from(self.en); sqlx::query!( r#"INSERT INTO en(key,data,name,tumonline_room_nr,type,type_common_name,lat,lon) - VALUES (?,?,?,?,?,?,?,?)"#, + VALUES ($1,$2,$3,$4,$5,$6,$7,$8)"#, self.key, data, fields.name, @@ -145,7 +145,7 @@ impl DelocalisedValues { Ok(()) } } -pub(crate) async fn load_all_to_db(pool: &SqlitePool) -> Result<(), Box> { +pub(crate) async fn load_all_to_db(pool: &PgPool) -> Result<(), Box> { let start = Instant::now(); let cdn_url = std::env::var("CDN_URL").unwrap_or_else(|_| "https://nav.tum.de/cdn".to_string()); let tasks = reqwest::get(format!("{cdn_url}/api_data.json")) diff --git a/server/main-api/src/setup/database/mod.rs b/server/main-api/src/setup/database/mod.rs index 0d91e6667..5ba84fdfb 100644 --- a/server/main-api/src/setup/database/mod.rs +++ b/server/main-api/src/setup/database/mod.rs @@ -1,9 +1,9 @@ mod alias; mod data; -use sqlx::{Executor, SqlitePool}; +use sqlx::{Executor, PgPool}; -pub(crate) async fn setup_database(pool: &SqlitePool) -> Result<(), Box> { +pub(crate) async fn setup_database(pool: &PgPool) -> Result<(), Box> { sqlx::migrate!("./migrations").run(pool).await?; // this is to setup the database faster // we don't want to use an acid compliant database for this step ;) From eb63444f5f5438580f5e849c1cc1ab9465f98c4f Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 12 Nov 2023 22:33:13 +0100 Subject: [PATCH 2/7] fixed the dockerfile --- server/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/server/Dockerfile b/server/Dockerfile index b71e97038..b9d7eac07 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -29,7 +29,6 @@ RUN cargo build --release --workspace \ # second run of the image build (including our code) COPY calendar/diesel.toml ./calendar/diesel.toml -COPY calendar/migrations calendar/migrations COPY .sqlx .sqlx COPY feedback/src ./feedback/src From d47bf6374da744ab85bae7dc33fd0d151e5fd6d9 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 12 Nov 2023 22:44:21 +0100 Subject: [PATCH 3/7] formatting fixes --- server/calendar/README.md | 2 +- server/calendar/diesel.toml | 2 +- server/main-api/README.md | 2 +- server/main-api/src/main.rs | 4 ++-- server/main-api/src/setup/database/data.rs | 5 ++++- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/server/calendar/README.md b/server/calendar/README.md index f452904b8..446dd1063 100644 --- a/server/calendar/README.md +++ b/server/calendar/README.md @@ -16,7 +16,7 @@ Please follow the [system dependencys docs](resources/documentation/Dependencys. To set up the database, you will need to run a `postgres` instance. We recommend this configuration: ```bash -docker run -p 5432:5432 -e POSTGRES_PASSWORD=password postgres +docker run -it --rm -e POSTGRES_PASSWORD=password -p 5432:5432 postgres:latest ``` Any migrations this database needs are applied on first run of the server. diff --git a/server/calendar/diesel.toml b/server/calendar/diesel.toml index 9f7d63dad..925e2ed30 100644 --- a/server/calendar/diesel.toml +++ b/server/calendar/diesel.toml @@ -2,4 +2,4 @@ # see https://diesel.rs/guides/configuring-diesel-cli [print_schema] -file = "src/schema.rs" \ No newline at end of file +file = "src/schema.rs" diff --git a/server/main-api/README.md b/server/main-api/README.md index 7aeeebb70..91ab3120d 100644 --- a/server/main-api/README.md +++ b/server/main-api/README.md @@ -33,7 +33,7 @@ MeiliSearch provides an interactive interface at . To set up the Postgres, run the following command: ```bash -docker run --name postgres -e POSTGRES_PASSWORD=password -p 5432:5432 postgres +docker run -it --rm -e POSTGRES_PASSWORD=password -p 5432:5432 postgres:latest ``` ### Starting the server diff --git a/server/main-api/src/main.rs b/server/main-api/src/main.rs index 7ccc2be10..248ff0744 100644 --- a/server/main-api/src/main.rs +++ b/server/main-api/src/main.rs @@ -2,8 +2,8 @@ use actix_cors::Cors; use actix_web::{get, middleware, web, App, HttpResponse, HttpServer}; use actix_web_prom::PrometheusMetricsBuilder; use log::{debug, error, info}; -use sqlx::prelude::*; use sqlx::postgres::PgPoolOptions; +use sqlx::prelude::*; use sqlx::PgPool; use std::collections::HashMap; use structured_logger::async_json::new_writer; @@ -55,7 +55,7 @@ async fn main() -> Result<(), Box> { Builder::with_level("info") .with_target_writer("*", new_writer(tokio::io::stdout())) .init(); - let uri=connection_string(); + let uri = connection_string(); let pool = PgPoolOptions::new().connect(&uri).await?; info!("setting up the database"); setup::database::setup_database(&pool).await?; diff --git a/server/main-api/src/setup/database/data.rs b/server/main-api/src/setup/database/data.rs index 222300986..dfb809896 100644 --- a/server/main-api/src/setup/database/data.rs +++ b/server/main-api/src/setup/database/data.rs @@ -108,7 +108,10 @@ impl From> for DelocalisedValues { } impl DelocalisedValues { - async fn store(self, tx: &mut sqlx::Transaction<'_, sqlx::Postgres>) -> Result<(), sqlx::Error> { + async fn store( + self, + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + ) -> Result<(), sqlx::Error> { let key = self.key.clone(); // has to be here due to livetimes somehow let (data, fields) = StorableValue::from(self.de); sqlx::query!( From 391837d8e03842acae1fae752bb2d8ec6dd0aabe Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 12 Nov 2023 22:45:49 +0100 Subject: [PATCH 4/7] linting fixes --- server/main-api/src/setup/database/data.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/main-api/src/setup/database/data.rs b/server/main-api/src/setup/database/data.rs index dfb809896..b5def1214 100644 --- a/server/main-api/src/setup/database/data.rs +++ b/server/main-api/src/setup/database/data.rs @@ -37,8 +37,8 @@ impl From> for ExtractedFields { .as_str() .unwrap() .to_string(), - lat: lat.unwrap_or(48.14903) as f64, - lon: lon.unwrap_or(11.56735) as f64, + lat: lat.unwrap_or(48.14903), + lon: lon.unwrap_or(11.56735), } } } From e8ac4a517ee8775a673dffd1016e570fb17760dd Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 12 Nov 2023 23:32:13 +0100 Subject: [PATCH 5/7] fixed a few db setup issues --- docker-compose.yml | 2 ++ .../migrations/20230323144107_create_calendar.down.sql | 1 + .../migrations/20230323144107_create_calendar.up.sql | 2 ++ .../main-api/migrations/20230323144504_add_index.down.sql | 2 -- server/main-api/migrations/20230323144504_add_index.up.sql | 2 -- .../migrations/20230421143407_add_tumonline_id.down.sql | 3 --- .../migrations/20230421143407_add_tumonline_id.up.sql | 3 --- .../20230603173522_removal_of_tumonline_id.down.sql | 3 --- .../20230603173522_removal_of_tumonline_id.up.sql | 3 --- server/main-api/migrations/20230902115306_aliases.up.sql | 7 ++----- 10 files changed, 7 insertions(+), 21 deletions(-) delete mode 100644 server/main-api/migrations/20230323144504_add_index.down.sql delete mode 100644 server/main-api/migrations/20230323144504_add_index.up.sql delete mode 100644 server/main-api/migrations/20230421143407_add_tumonline_id.down.sql delete mode 100644 server/main-api/migrations/20230421143407_add_tumonline_id.up.sql delete mode 100644 server/main-api/migrations/20230603173522_removal_of_tumonline_id.down.sql delete mode 100644 server/main-api/migrations/20230603173522_removal_of_tumonline_id.up.sql diff --git a/docker-compose.yml b/docker-compose.yml index 1ed6f38a3..287d22cf6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -86,6 +86,8 @@ services: depends_on: meilisearch: condition: service_healthy + db: + condition: service_healthy healthcheck: test: wget -q --spider http://localhost:3003/api/status retries: 5 diff --git a/server/main-api/migrations/20230323144107_create_calendar.down.sql b/server/main-api/migrations/20230323144107_create_calendar.down.sql index f043144e0..6b6d42830 100644 --- a/server/main-api/migrations/20230323144107_create_calendar.down.sql +++ b/server/main-api/migrations/20230323144107_create_calendar.down.sql @@ -1,2 +1,3 @@ -- This file should undo anything in `up.sql` DROP TABLE IF EXISTS calendar +DROP INDEX IF EXISTS calendar_lut diff --git a/server/main-api/migrations/20230323144107_create_calendar.up.sql b/server/main-api/migrations/20230323144107_create_calendar.up.sql index b678ebee6..47178e641 100644 --- a/server/main-api/migrations/20230323144107_create_calendar.up.sql +++ b/server/main-api/migrations/20230323144107_create_calendar.up.sql @@ -22,3 +22,5 @@ CREATE TABLE calendar ( comment TEXT NOT NULL, last_scrape TIMESTAMP NOT NULL ); + +CREATE INDEX IF NOT EXISTS calendar_lut ON calendar(key, dtstart, dtend) \ No newline at end of file diff --git a/server/main-api/migrations/20230323144504_add_index.down.sql b/server/main-api/migrations/20230323144504_add_index.down.sql deleted file mode 100644 index 0b9b0a603..000000000 --- a/server/main-api/migrations/20230323144504_add_index.down.sql +++ /dev/null @@ -1,2 +0,0 @@ --- This file should undo anything in `up.sql` -DROP INDEX IF EXISTS calendar_lut diff --git a/server/main-api/migrations/20230323144504_add_index.up.sql b/server/main-api/migrations/20230323144504_add_index.up.sql deleted file mode 100644 index ff9dfefeb..000000000 --- a/server/main-api/migrations/20230323144504_add_index.up.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Your SQL goes here -CREATE INDEX IF NOT EXISTS calendar_lut ON calendar(key, dtstart, dtend) diff --git a/server/main-api/migrations/20230421143407_add_tumonline_id.down.sql b/server/main-api/migrations/20230421143407_add_tumonline_id.down.sql deleted file mode 100644 index addf2c404..000000000 --- a/server/main-api/migrations/20230421143407_add_tumonline_id.down.sql +++ /dev/null @@ -1,3 +0,0 @@ --- This file should undo anything in `up.sql` -ALTER TABLE calendar -DROP COLUMN tumonline_id; diff --git a/server/main-api/migrations/20230421143407_add_tumonline_id.up.sql b/server/main-api/migrations/20230421143407_add_tumonline_id.up.sql deleted file mode 100644 index 8eeefe951..000000000 --- a/server/main-api/migrations/20230421143407_add_tumonline_id.up.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Your SQL goes here -ALTER TABLE calendar -ADD COLUMN tumonline_id INTEGER NOT NULL DEFAULT 0; diff --git a/server/main-api/migrations/20230603173522_removal_of_tumonline_id.down.sql b/server/main-api/migrations/20230603173522_removal_of_tumonline_id.down.sql deleted file mode 100644 index 8eeefe951..000000000 --- a/server/main-api/migrations/20230603173522_removal_of_tumonline_id.down.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Your SQL goes here -ALTER TABLE calendar -ADD COLUMN tumonline_id INTEGER NOT NULL DEFAULT 0; diff --git a/server/main-api/migrations/20230603173522_removal_of_tumonline_id.up.sql b/server/main-api/migrations/20230603173522_removal_of_tumonline_id.up.sql deleted file mode 100644 index 07e5178c2..000000000 --- a/server/main-api/migrations/20230603173522_removal_of_tumonline_id.up.sql +++ /dev/null @@ -1,3 +0,0 @@ --- This file should undo anything in `down.sql` -ALTER TABLE calendar -DROP COLUMN tumonline_id; diff --git a/server/main-api/migrations/20230902115306_aliases.up.sql b/server/main-api/migrations/20230902115306_aliases.up.sql index 79e69fdf0..ff6e81ea6 100644 --- a/server/main-api/migrations/20230902115306_aliases.up.sql +++ b/server/main-api/migrations/20230902115306_aliases.up.sql @@ -2,12 +2,9 @@ CREATE TABLE aliases ( id SERIAL PRIMARY KEY NOT NULL, - alias TEXT NOT NULL, + alias TEXT UNIQUE NOT NULL, key TEXT NOT NULL, visible_id TEXT NOT NULL, type TEXT NOT NULL, FOREIGN KEY(key) REFERENCES de(key) -); - --- prevent duplicate aliases -CREATE UNIQUE INDEX alias_key ON aliases (alias, key); +); \ No newline at end of file From 79a183b0452e30847ec388880e7327d2ae2690e0 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 12 Nov 2023 23:36:43 +0100 Subject: [PATCH 6/7] fixed a few db setup issues --- .../migrations/20230323144107_create_calendar.down.sql | 4 ++-- .../main-api/migrations/20230323144107_create_calendar.up.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/main-api/migrations/20230323144107_create_calendar.down.sql b/server/main-api/migrations/20230323144107_create_calendar.down.sql index 6b6d42830..5a94e7b1f 100644 --- a/server/main-api/migrations/20230323144107_create_calendar.down.sql +++ b/server/main-api/migrations/20230323144107_create_calendar.down.sql @@ -1,3 +1,3 @@ -- This file should undo anything in `up.sql` -DROP TABLE IF EXISTS calendar -DROP INDEX IF EXISTS calendar_lut +DROP TABLE IF EXISTS calendar; +DROP INDEX IF EXISTS calendar_lut; diff --git a/server/main-api/migrations/20230323144107_create_calendar.up.sql b/server/main-api/migrations/20230323144107_create_calendar.up.sql index 47178e641..95f4f7a45 100644 --- a/server/main-api/migrations/20230323144107_create_calendar.up.sql +++ b/server/main-api/migrations/20230323144107_create_calendar.up.sql @@ -23,4 +23,4 @@ CREATE TABLE calendar ( last_scrape TIMESTAMP NOT NULL ); -CREATE INDEX IF NOT EXISTS calendar_lut ON calendar(key, dtstart, dtend) \ No newline at end of file +CREATE INDEX IF NOT EXISTS calendar_lut ON calendar(key, dtstart, dtend); \ No newline at end of file From bad75bb7aa1b32d2455833be6e8df9ce6a939cb2 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 12 Nov 2023 23:44:15 +0100 Subject: [PATCH 7/7] linting fixes --- .../main-api/migrations/20230323144107_create_calendar.up.sql | 2 +- server/main-api/migrations/20230902115306_aliases.up.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/main-api/migrations/20230323144107_create_calendar.up.sql b/server/main-api/migrations/20230323144107_create_calendar.up.sql index 95f4f7a45..729488a5f 100644 --- a/server/main-api/migrations/20230323144107_create_calendar.up.sql +++ b/server/main-api/migrations/20230323144107_create_calendar.up.sql @@ -23,4 +23,4 @@ CREATE TABLE calendar ( last_scrape TIMESTAMP NOT NULL ); -CREATE INDEX IF NOT EXISTS calendar_lut ON calendar(key, dtstart, dtend); \ No newline at end of file +CREATE INDEX IF NOT EXISTS calendar_lut ON calendar(key, dtstart, dtend); diff --git a/server/main-api/migrations/20230902115306_aliases.up.sql b/server/main-api/migrations/20230902115306_aliases.up.sql index ff6e81ea6..46fb174ee 100644 --- a/server/main-api/migrations/20230902115306_aliases.up.sql +++ b/server/main-api/migrations/20230902115306_aliases.up.sql @@ -7,4 +7,4 @@ CREATE TABLE aliases visible_id TEXT NOT NULL, type TEXT NOT NULL, FOREIGN KEY(key) REFERENCES de(key) -); \ No newline at end of file +);