Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Do not serialize missing configs #999

Merged
merged 1 commit into from
Nov 13, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion justfile
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ restart:

# Stop the test database
stop:
docker-compose down
docker-compose down --remove-orphans

# Run benchmark tests
bench: start
Expand Down
4 changes: 2 additions & 2 deletions martin/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ pub struct Config {
#[serde(flatten)]
pub srv: SrvConfig,

#[serde(default)]
#[serde(default, skip_serializing_if = "OptOneMany::is_none")]
pub postgres: OptOneMany<PgConfig>,

#[serde(default, skip_serializing_if = "FileConfigEnum::is_none")]
Expand All @@ -45,7 +45,7 @@ pub struct Config {
#[serde(default, skip_serializing_if = "FileConfigEnum::is_none")]
pub sprites: FileConfigEnum,

#[serde(default)]
#[serde(default, skip_serializing_if = "OptOneMany::is_none")]
pub fonts: OptOneMany<PathBuf>,

#[serde(flatten)]
Expand Down
8 changes: 4 additions & 4 deletions martin/src/pg/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ pub struct PgConfig {

#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct PgCfgPublish {
#[serde(default, skip_serializing_if = "OptOneMany::is_none")]
#[serde(alias = "from_schema")]
#[serde(default, skip_serializing_if = "OptOneMany::is_none")]
pub from_schemas: OptOneMany<String>,
#[serde(default, skip_serializing_if = "OptBoolObj::is_none")]
pub tables: OptBoolObj<PgCfgPublishTables>,
Expand All @@ -66,16 +66,16 @@ pub struct PgCfgPublish {
#[serde_with::skip_serializing_none]
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct PgCfgPublishTables {
#[serde(default, skip_serializing_if = "OptOneMany::is_none")]
#[serde(alias = "from_schema")]
#[serde(default, skip_serializing_if = "OptOneMany::is_none")]
pub from_schemas: OptOneMany<String>,
#[serde(alias = "id_format")]
pub source_id_format: Option<String>,
/// A table column to use as the feature ID
/// If a table has no column with this name, `id_column` will not be set for that table.
/// If a list of strings is given, the first found column will be treated as a feature ID.
#[serde(default, skip_serializing_if = "OptOneMany::is_none")]
#[serde(alias = "id_column")]
#[serde(default, skip_serializing_if = "OptOneMany::is_none")]
pub id_columns: OptOneMany<String>,
pub clip_geom: Option<bool>,
pub buffer: Option<u32>,
Expand All @@ -85,8 +85,8 @@ pub struct PgCfgPublishTables {
#[serde_with::skip_serializing_none]
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct PgCfgPublishFuncs {
#[serde(default, skip_serializing_if = "OptOneMany::is_none")]
#[serde(alias = "from_schema")]
#[serde(default, skip_serializing_if = "OptOneMany::is_none")]
pub from_schemas: OptOneMany<String>,
#[serde(alias = "id_format")]
pub source_id_format: Option<String>,
Expand Down
10 changes: 10 additions & 0 deletions tests/expected/auto2/catalog_auto.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"tiles": {
"webp2": {
"content_type": "image/webp",
"name": "ne2sr"
}
},
"sprites": {},
"fonts": {}
}
6 changes: 6 additions & 0 deletions tests/expected/generated_config2.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
listen_addresses: localhost:3111
pmtiles:
paths: tests/fixtures/pmtiles2
sources:
webp2: tests/fixtures/pmtiles2/webp2.pmtiles
mbtiles: tests/fixtures/pmtiles2
54 changes: 44 additions & 10 deletions tests/test.sh
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
#!/usr/bin/env bash
set -euo pipefail

MARTIN_DATABASE_URL="${DATABASE_URL:-postgres://postgres@localhost/db}"
unset DATABASE_URL

# TODO: use --fail-with-body to get the response body on failure
CURL=${CURL:-curl --silent --show-error --fail --compressed}
DATABASE_URL="${DATABASE_URL:-postgres://postgres@localhost/db}"

MARTIN_BUILD="${MARTIN_BUILD:-cargo build}"
MARTIN_PORT="${MARTIN_PORT:-3111}"
MARTIN_URL="http://localhost:${MARTIN_PORT}"
Expand Down Expand Up @@ -146,10 +149,6 @@ validate_log()
remove_line "$LOG_FILE" 'Margin parameter in ST_TileEnvelope is not supported'
remove_line "$LOG_FILE" 'Source IDs must be unique'

# Make sure the log has just the expected warnings, remove them, and test that there are no other ones
test_log_has_str "$LOG_FILE" 'WARN martin::pg::table_source] Table public.table_source has no spatial index on column geom'
test_log_has_str "$LOG_FILE" 'WARN martin::fonts] Ignoring duplicate font Overpass Mono Regular from tests'

echo "Checking for no other warnings or errors in the log"
if grep -e ' ERROR ' -e ' WARN ' "$LOG_FILE"; then
echo "Log file $LOG_FILE has unexpected warnings or errors"
Expand All @@ -172,17 +171,20 @@ fi
echo "------------------------------------------------------------------------------------------------------------------------"
echo "Test auto configured Martin"

LOG_FILE="${LOG_DIR}/test_log_1.txt"
TEST_OUT_DIR="$(dirname "$0")/output/auto"
mkdir -p "$TEST_OUT_DIR"

ARG=(--default-srid 900913 --auto-bounds calc --save-config "$(dirname "$0")/output/generated_config.yaml" tests/fixtures/mbtiles tests/fixtures/pmtiles tests/fixtures/pmtiles2 --sprite tests/fixtures/sprites/src1 --font tests/fixtures/fonts/overpass-mono-regular.ttf --font tests/fixtures/fonts)
export DATABASE_URL="$MARTIN_DATABASE_URL"

set -x
$MARTIN_BIN "${ARG[@]}" 2>&1 | tee "${LOG_DIR}/test_log_1.txt" &
$MARTIN_BIN "${ARG[@]}" 2>&1 | tee "$LOG_FILE" &
MARTIN_PROC_ID=`jobs -p | tail -n 1`

{ set +x; } 2> /dev/null
trap "echo 'Stopping Martin server $MARTIN_PROC_ID...'; kill -9 $MARTIN_PROC_ID 2> /dev/null || true; echo 'Stopped Martin server $MARTIN_PROC_ID';" EXIT HUP INT TERM
wait_for $MARTIN_PROC_ID Martin "$MARTIN_URL/health"
unset DATABASE_URL

>&2 echo "Test catalog"
test_jsn catalog_auto catalog
Expand Down Expand Up @@ -253,21 +255,51 @@ test_pbf mb_mvt_2_3_1 world_cities/2/3/1
test_pbf points_empty_srid_0_0_0 points_empty_srid/0/0/0

kill_process $MARTIN_PROC_ID Martin
validate_log "${LOG_DIR}/test_log_1.txt"

test_log_has_str "$LOG_FILE" 'WARN martin::pg::table_source] Table public.table_source has no spatial index on column geom'
test_log_has_str "$LOG_FILE" 'WARN martin::fonts] Ignoring duplicate font Overpass Mono Regular from tests'
validate_log "$LOG_FILE"


echo "------------------------------------------------------------------------------------------------------------------------"
echo "Test minimum auto configured Martin"

LOG_FILE="${LOG_DIR}/test_log_2.txt"
TEST_OUT_DIR="$(dirname "$0")/output/auto2"
mkdir -p "$TEST_OUT_DIR"

ARG=(--save-config "$(dirname "$0")/output/generated_config2.yaml" tests/fixtures/pmtiles2)
set -x
$MARTIN_BIN "${ARG[@]}" 2>&1 | tee "$LOG_FILE" &
MARTIN_PROC_ID=`jobs -p | tail -n 1`

{ set +x; } 2> /dev/null
trap "echo 'Stopping Martin server $MARTIN_PROC_ID...'; kill -9 $MARTIN_PROC_ID 2> /dev/null || true; echo 'Stopped Martin server $MARTIN_PROC_ID';" EXIT HUP INT TERM
wait_for $MARTIN_PROC_ID Martin "$MARTIN_URL/health"

>&2 echo "Test catalog"
test_jsn catalog_auto catalog

kill_process $MARTIN_PROC_ID Martin
validate_log "$LOG_FILE"


echo "------------------------------------------------------------------------------------------------------------------------"
echo "Test pre-configured Martin"

LOG_FILE="${LOG_DIR}/test_log_3.txt"
TEST_OUT_DIR="$(dirname "$0")/output/configured"
mkdir -p "$TEST_OUT_DIR"

ARG=(--config tests/config.yaml --max-feature-count 1000 --save-config "$(dirname "$0")/output/given_config.yaml" -W 1)
export DATABASE_URL="$MARTIN_DATABASE_URL"
set -x
$MARTIN_BIN "${ARG[@]}" 2>&1 | tee "${LOG_DIR}/test_log_2.txt" &
$MARTIN_BIN "${ARG[@]}" 2>&1 | tee "$LOG_FILE" &
MARTIN_PROC_ID=`jobs -p | tail -n 1`
{ set +x; } 2> /dev/null
trap "echo 'Stopping Martin server $MARTIN_PROC_ID...'; kill -9 $MARTIN_PROC_ID 2> /dev/null || true; echo 'Stopped Martin server $MARTIN_PROC_ID';" EXIT HUP INT TERM
wait_for $MARTIN_PROC_ID Martin "$MARTIN_URL/health"
unset DATABASE_URL

>&2 echo "Test catalog"
test_jsn catalog_cfg catalog
Expand Down Expand Up @@ -297,7 +329,9 @@ test_font font_2 font/Overpass%20Mono%20Regular/0-255
test_font font_3 font/Overpass%20Mono%20Regular,Overpass%20Mono%20Light/0-255

kill_process $MARTIN_PROC_ID Martin
validate_log "${LOG_DIR}/test_log_2.txt"
test_log_has_str "$LOG_FILE" 'WARN martin::pg::table_source] Table public.table_source has no spatial index on column geom'
test_log_has_str "$LOG_FILE" 'WARN martin::fonts] Ignoring duplicate font Overpass Mono Regular from tests'
validate_log "$LOG_FILE"

remove_line "$(dirname "$0")/output/given_config.yaml" " connection_string: "
remove_line "$(dirname "$0")/output/generated_config.yaml" " connection_string: "
Expand Down
Loading