diff --git a/CMakeLists.txt b/CMakeLists.txt index 533c88429..855760da2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,7 +51,7 @@ find_package(re2 REQUIRED) # Includes # --------------------------------------------------------------------------- -include_directories( +include_directories(SYSTEM ${CMAKE_SOURCE_DIR}/include ) diff --git a/include/config/config_metadata.h b/include/config/config_metadata.h index 20d656f4f..0c8c2977f 100644 --- a/include/config/config_metadata.h +++ b/include/config/config_metadata.h @@ -28,7 +28,11 @@ class ConfigSpecification { std::string helpText() const; - std::optional getValueSpecification(const ConfigKeyPath& key) const; + std::optional getValueSpecification(const ConfigKeyPath& key + ) const; + + std::optional getValueSpecificationStrict(const ConfigKeyPath& key + ) const; }; } // namespace silo::config diff --git a/include/config/config_source_interface.h b/include/config/config_source_interface.h index d91dc980d..e5c13c92a 100644 --- a/include/config/config_source_interface.h +++ b/include/config/config_source_interface.h @@ -145,15 +145,11 @@ class VerifiedConfigSource { [[nodiscard]] std::optional getUint32(const ConfigKeyPath& config_key_path) const; - [[nodiscard]] std::optional getUint16(const ConfigKeyPath& config_key_path) const; + [[nodiscard]] std::optional getUint16(const ConfigKeyPath& config_key_path) const; [[nodiscard]] std::optional getFloat(const ConfigKeyPath& config_key_path) const; [[nodiscard]] std::optional getBool(const ConfigKeyPath& config_key_path) const; - - [[nodiscard]] bool hasProperty(const ConfigKeyPath& config_key_path) const { - return getString(config_key_path).has_value(); - } }; } // namespace silo::config diff --git a/include/config/config_value.h b/include/config/config_value.h index 677ffa2b8..6e5aabfd4 100644 --- a/include/config/config_value.h +++ b/include/config/config_value.h @@ -16,7 +16,7 @@ namespace silo::config { class ConfigKeyPath { public: - std::vector path; + std::vector> path; friend bool operator==(const ConfigKeyPath& lhs, const ConfigKeyPath& rhs) { return lhs.path == rhs.path; @@ -44,9 +44,30 @@ constexpr std::string_view configValueTypeToString(ConfigValueType type) { } } +class ConfigValueSpecification; + class ConfigValue { + friend class ConfigValueSpecification; + + ConfigValue( + std::variant value + ) + : value(value) {} + public: - std::variant value; + std::variant value; + + static ConfigValue fromString(const std::string& value) { return ConfigValue{value}; } + + static ConfigValue fromPath(const std::filesystem::path& value) { return ConfigValue{value}; } + + static ConfigValue fromInt32(int32_t value) { return ConfigValue{value}; } + + static ConfigValue fromUint32(uint32_t value) { return ConfigValue{value}; } + + static ConfigValue fromUint16(uint16_t value) { return ConfigValue{value}; } + + static ConfigValue fromBool(bool value) { return ConfigValue{value}; } ConfigValueType getValueType() const { return std::visit( @@ -94,18 +115,48 @@ class ConfigValue { }; class ConfigValueSpecification { + ConfigValueSpecification() = default; + public: ConfigKeyPath key; ConfigValueType type; std::optional default_value; /// Help as shown for --help, excluding the other info above. + /// If type is bool, the command line option does not take an argument but + /// is the constant "true", which will be added to the help text std::string_view help_text; - /// If true, the command line option does not take an argument but - /// is the constant "true". - [[nodiscard]] bool isBool() const; - ConfigValue getValueFromString(std::string value_string) const; + + ConfigValue createValue( + std::variant value + ) const; + + static ConfigValueSpecification createWithoutDefault( + ConfigKeyPath key, + ConfigValueType value_type, + std::string_view help_text + ){ + ConfigValueSpecification value_specification; + value_specification.key = key; + value_specification.type = value_type; + value_specification.help_text = help_text; + return value_specification; + } + + /// No need for the value_type. It is implicitly defined by the default. Prevents misspecification. + static ConfigValueSpecification createWithDefault( + ConfigKeyPath key, + ConfigValue default_value, + std::string_view help_text + ){ + ConfigValueSpecification value_specification; + value_specification.key = key; + value_specification.type = default_value.getValueType(); + value_specification.default_value = default_value; + value_specification.help_text = help_text; + return value_specification; + } }; } // namespace silo::config @@ -113,12 +164,6 @@ class ConfigValueSpecification { namespace std { template <> struct hash { - std::size_t operator()(const silo::config::ConfigKeyPath& key) const { - std::size_t seed = 0; // Initialize seed for the hash - for (const auto& segment : key.path) { - boost::hash_combine(seed, segment); // Combine each segment's hash - } - return seed; // Return the final combined hash - } + std::size_t operator()(const silo::config::ConfigKeyPath& key) const; }; } // namespace std diff --git a/include/config/overwrite_from_interface.h b/include/config/overwrite_from_interface.h index adf4a0955..ed584b23e 100644 --- a/include/config/overwrite_from_interface.h +++ b/include/config/overwrite_from_interface.h @@ -15,9 +15,7 @@ class OverwriteFrom { /// .to_vec_reverse() and wrap in ConfigKeyPath). Throws /// `silo::config::ConfigException` for config value parse errors /// (subclass as ConfigValueParseError?). - virtual void overwriteFrom( - const VerifiedConfigSource& config_source - ) = 0; + virtual void overwriteFrom(const VerifiedConfigSource& config_source) = 0; virtual ~OverwriteFrom() = default; }; diff --git a/include/config/source/environment_variables.h b/include/config/source/environment_variables.h index eb2c9afad..0f2d15d5f 100644 --- a/include/config/source/environment_variables.h +++ b/include/config/source/environment_variables.h @@ -31,7 +31,7 @@ class EnvironmentVariables : public RawConfigSource { static std::string configKeyPathToString(const ConfigKeyPath& key_path); - static ConfigKeyPath stringToConfigKeyPath(const std::string& key_path_string); + static ConfigKeyPath stringToConfigKeyPath(const std::string& key_path_string); // TODO check whether AmbiguousKeyPath can be used }; } // namespace silo::config diff --git a/include/config/toplevel_interface.h b/include/config/toplevel_interface.h index 33dd6d5fc..0ee8a31ac 100644 --- a/include/config/toplevel_interface.h +++ b/include/config/toplevel_interface.h @@ -7,9 +7,6 @@ #include "config/config_metadata.h" #include "config/overwrite_from_interface.h" -#include "config/source/command_line_arguments.h" -#include "config/source/environment_variables.h" -#include "config/source/yaml_file.h" #include "silo/common/cons_list.h" #include "silo/common/overloaded.h" @@ -30,48 +27,7 @@ template std::optional rawGetConfig( std::span cmd, const ConfigSpecification& config_specification -) { - auto env_source = - EnvironmentVariables::decodeEnvironmentVariables().verify(config_specification); - auto cmd_source = CommandLineArguments{cmd}.verify(config_specification); - - const ConsList no_parents{}; - - C config; - - // First, only check command line arguments, for "--help"; avoid - // potential errors from env processing, and we don't have the - // path to the config file yet. Since we're only interested in the - // help option, there's no need to read config_struct first, OK? - config = {}; - config.overwriteFrom(no_parents, cmd_source); - if (config.asksForHelp()) { - return std::nullopt; - } - - // Then process env and cmd, to get to the config file - // path. Re-initialize since env must be processed for cmd. - config = {}; - config.overwriteFrom(no_parents, env_source); - config.overwriteFrom(no_parents, cmd_source); - // Would anyone request help via SILO_ENV=true? Well, allow it: - if (config.asksForHelp()) { - return std::nullopt; - } - - auto config_path = config.configPath(); - if (config_path.has_value()) { - auto file_source = YamlFile::readFile(*config_path).verify(config_specification); - // Now read again with the file first: - config = {}; - config.overwriteFrom(no_parents, file_source); - config.overwriteFrom(no_parents, env_source); - config.overwriteFrom(no_parents, cmd_source); - // (The config file might specify --help, too, but we ignore - // that.) - } - return std::optional{config}; -} +); /// In case of error, returns the exit code that the caller should /// pass to exit(): 0 if the user gave --help, 1 in case of erroneous diff --git a/include/silo/common/fmt_formatters.h b/include/silo/common/fmt_formatters.h index 6d45c1a44..1540ecd0d 100644 --- a/include/silo/common/fmt_formatters.h +++ b/include/silo/common/fmt_formatters.h @@ -5,6 +5,7 @@ #include #include +#include template struct [[maybe_unused]] fmt::formatter> : fmt::formatter { @@ -43,3 +44,17 @@ struct [[maybe_unused]] fmt::formatter< return fmt::format_to(ctx.out(), "{}", silo::common::toIsoString(val)); } }; + +namespace fmt { + +template <> +struct formatter { + constexpr auto parse(format_parse_context& ctx) -> decltype(ctx.begin()) { return ctx.end(); } + + template + auto format(const nlohmann::json& json, FormatContext& ctx) -> decltype(ctx.out()) { + return fmt::format_to(ctx.out(), "{}", json.dump()); + } +}; + +} // namespace fmt diff --git a/include/silo/config/preprocessing_config.h b/include/silo/config/preprocessing_config.h index c4cd0067a..49ab29cfe 100644 --- a/include/silo/config/preprocessing_config.h +++ b/include/silo/config/preprocessing_config.h @@ -10,214 +10,97 @@ #include #include +#include "config/toplevel_interface.h" +#include "config/source/yaml_file.h" #include "silo/config/config_defaults.h" -#include "silo/config/util/abstract_config_source.h" - -// Definition of the PreprocessingConfig struct and associated config -// processing code. - -// TUPLE(TYPE, FIELD_NAME, DEFAULT_GENERATION, DEFAULT_VALUE, OPTION_PATH, -// PARSING_ACCESSOR_TYPE_NAME, HELP_TEXT, ACCESSOR_GENERATION, ACCESSOR_NAME) -#define PREPROCESSING_CONFIG_DEFINITION \ - TUPLE( \ - std::filesystem::path, \ - input_directory, \ - HAS_DEFAULT, \ - "./", \ - {"inputDirectory"}, \ - String, \ - "the path to the directory with the input files", \ - ACCESSOR, \ - getInputDirectory \ - ); \ - TUPLE( \ - std::filesystem::path, \ - output_directory, \ - HAS_DEFAULT, \ - DEFAULT_OUTPUT_DIRECTORY, \ - {"outputDirectory"}, \ - String, \ - "the path to the directory to hold the output files", \ - ACCESSOR, \ - getOutputDirectory \ - ); \ - TUPLE( \ - std::filesystem::path, \ - intermediate_results_directory, \ - HAS_DEFAULT, \ - "./temp/", \ - {"intermediateResultsDirectory"}, \ - String, \ - "the path to the directory to hold temporary files", \ - ACCESSOR, \ - getIntermediateResultsDirectory \ - ); \ - TUPLE( \ - std::optional, \ - preprocessing_database_location, \ - HAS_NO_DEFAULT, \ - , \ - {"preprocessingDatabaseLocation"}, \ - String, \ - "XXX?", \ - ACCESSOR, \ - getPreprocessingDatabaseLocation \ - ); \ - TUPLE( \ - std::optional, \ - duckdb_memory_limit_in_g, \ - HAS_NO_DEFAULT, \ - , \ - {"duckdbMemoryLimitInG"}, \ - UInt32, \ - "DuckDB memory limit in GB", \ - ACCESSOR, \ - getDuckdbMemoryLimitInG \ - ); \ - TUPLE( \ - std::optional, \ - pango_lineage_definition_file, \ - HAS_NO_DEFAULT, \ - , \ - {"pangoLineageDefinitionFilename"}, \ - String, \ - "file name of the file holding th pango lineage definitions", \ - NOACCESSOR, \ - getPangoLineageDefinitionFile \ - ); \ - TUPLE( \ - std::optional, \ - ndjson_input_filename, \ - HAS_NO_DEFAULT, \ - , \ - {"ndjsonInputFilename"}, \ - String, \ - "file name of the file holding NDJSON input", \ - NOACCESSOR, \ - getNdjsonInputFilename \ - ); \ - TUPLE( \ - std::optional, \ - metadata_file, \ - HAS_NO_DEFAULT, \ - , \ - {"metadataFilename"}, \ - String, \ - "file name of the file holding metadata", \ - NOACCESSOR, \ - getMetadataFile \ - ); \ - TUPLE( \ - std::filesystem::path, \ - database_config_file, \ - HAS_DEFAULT, \ - "database_config.yaml", \ - {"databaseConfigFile"}, \ - String, \ - "file name of the file holding the database table configuration", \ - NOACCESSOR, \ - getDatabaseConfigFile \ - ); \ - TUPLE( \ - std::filesystem::path, \ - reference_genome_file, \ - HAS_DEFAULT, \ - "reference_genomes.json", \ - {"referenceGenomeFilename"}, \ - String, \ - "file name of the file holding the reference genome", \ - NOACCESSOR, \ - getReferenceGenomeFile \ - ); \ - TUPLE( \ - std::string, \ - nucleotide_sequence_prefix, \ - HAS_DEFAULT, \ - "nuc_", \ - {"nucleotideSequencePrefix"}, \ - String, \ - "the prefix for nucleotide sequences", \ - NOACCESSOR, \ - getNucleotideSequencePrefix \ - ); \ - TUPLE( \ - std::string, \ - unaligned_nucleotide_sequence_prefix, \ - HAS_DEFAULT, \ - "unaligned_", \ - {"unalignedNucleotideSequencePrefix"}, \ - String, \ - "the prefix for unaligned nucleotide sequences", \ - NOACCESSOR, \ - getUnalignedNucleotideSequencePrefix \ - ); \ - TUPLE( \ - std::string, \ - gene_prefix, \ - HAS_DEFAULT, \ - "gene_", \ - {"genePrefix"}, \ - String, \ - "the prefix for genes XX?", \ - NOACCESSOR, \ - getGenePrefix \ - ); \ - TUPLE( \ - std::string, \ - nuc_insertions_filename, \ - HAS_DEFAULT, \ - "nuc_insertions.tsv", \ - {"nucleotideInsertionsFilename"}, \ - String, \ - "the file name of the file holding nucleotide insertions", \ - NOACCESSOR, \ - getNucInsertionsFilename \ - ); \ - TUPLE( \ - std::string, \ - aa_insertions_filename, \ - HAS_DEFAULT, \ - "aa_insertions.tsv", \ - {"aminoAcidInsertionsFilename"}, \ - String, \ - "the file name of the file hodling amino acid insertions", \ - NOACCESSOR, \ - getAaInsertionsFilename \ - ) namespace silo::config { -class PreprocessingConfig { +const ConfigKeyPath HELP_OPTION_KEY = YamlFile::stringToConfigKeyPath("help"); +const ConfigKeyPath INPUT_DIRECTORY_OPTION_KEY = YamlFile::stringToConfigKeyPath("inputDirectory"); +const ConfigKeyPath OUTPUT_DIRECTORY_OPTION_KEY = YamlFile::stringToConfigKeyPath("outputDirectory"); +const ConfigKeyPath INTERMEDIATE_RESULTS_DIRECTORY_OPTION_KEY = YamlFile::stringToConfigKeyPath("intermediateResultsDirectory"); +const ConfigKeyPath PREPROCESSING_DATABASE_LOCATION_OPTION_KEY = YamlFile::stringToConfigKeyPath("preprocessingDatabaseLocation"); +const ConfigKeyPath DUCKDB_MEMORY_LIMIT_IN_G_OPTION_KEY = YamlFile::stringToConfigKeyPath("duckdbMemoryLimitInG"); +const ConfigKeyPath LINEAGE_DEFINITIONS_FILE_OPTION_KEY = YamlFile::stringToConfigKeyPath("lineageDefinitionsFilename"); +const ConfigKeyPath NDJSON_INPUT_FILENAME_OPTION_KEY = YamlFile::stringToConfigKeyPath("ndjsonInputFilename"); +const ConfigKeyPath DATABASE_CONFIG_FILE_OPTION_KEY = YamlFile::stringToConfigKeyPath("databaseConfigFile"); +const ConfigKeyPath REFERENCE_GENOMES_FILENAME_OPTION_KEY = YamlFile::stringToConfigKeyPath("referenceGenomeFilename"); + +// Specification of the fields in inputs to the PreprocessingConfig struct +const ConfigSpecification PREPROCESSING_CONFIG_SPECIFICATION{ + .config_name = "preprocessing_config", + // TODO .program_name = "siloApi", + .fields{ + ConfigValueSpecification::createWithoutDefault( + HELP_OPTION_KEY, + ConfigValueType::BOOL, + "Show help text." + ), + ConfigValueSpecification::createWithDefault( + INPUT_DIRECTORY_OPTION_KEY, + ConfigValue::fromPath("./"), + "the path to the directory with the input files" + ), + ConfigValueSpecification::createWithDefault( + OUTPUT_DIRECTORY_OPTION_KEY, + ConfigValue::fromPath(DEFAULT_OUTPUT_DIRECTORY), + "the path to the directory to hold the output files" + ), + ConfigValueSpecification::createWithDefault( + INTERMEDIATE_RESULTS_DIRECTORY_OPTION_KEY, + ConfigValue::fromPath("./temp/"), + "the path to the directory to hold temporary files" + ), + ConfigValueSpecification::createWithoutDefault( + PREPROCESSING_DATABASE_LOCATION_OPTION_KEY, + ConfigValueType::PATH, + "the file where the duckdb database will be stored, which is used during preprocessing" + ), + ConfigValueSpecification::createWithoutDefault( + DUCKDB_MEMORY_LIMIT_IN_G_OPTION_KEY, + ConfigValueType::UINT32, + "DuckDB memory limit in GB" + ), + ConfigValueSpecification::createWithoutDefault( + LINEAGE_DEFINITIONS_FILE_OPTION_KEY, + ConfigValueType::PATH, + "file name of the file holding the lineage definitions" + ), + ConfigValueSpecification::createWithoutDefault( + NDJSON_INPUT_FILENAME_OPTION_KEY, + ConfigValueType::PATH, + "file name of the file holding NDJSON input" + ), + ConfigValueSpecification::createWithDefault( + DATABASE_CONFIG_FILE_OPTION_KEY, + ConfigValue::fromPath("database_config.yaml"), + "file name of the file holding the database table configuration" + ), + ConfigValueSpecification::createWithDefault( + REFERENCE_GENOMES_FILENAME_OPTION_KEY, + ConfigValue::fromPath("reference_genomes.json"), + "file name of the file holding the reference genome" + ), + } +}; + +class PreprocessingConfig : public ToplevelConfig { friend class fmt::formatter; public: -#define TUPLE( \ - TYPE, \ - FIELD_NAME, \ - DEFAULT_GENERATION, \ - DEFAULT_VALUE, \ - OPTION_PATH, \ - PARSING_ACCESSOR_TYPE_NAME, \ - HELP_TEXT, \ - ACCESSOR_GENERATION, \ - ACCESSOR_NAME \ -) \ - ACCESSOR_GENERATION([[nodiscard]] TYPE ACCESSOR_NAME() const { return FIELD_NAME; }) \ - TYPE FIELD_NAME DEFAULT_GENERATION DEFAULT_VALUE -#define HAS_DEFAULT = -#define HAS_NO_DEFAULT -#define ACCESSOR(code) code -#define NOACCESSOR(code) - - PREPROCESSING_CONFIG_DEFINITION; - -#undef NOACCESSOR -#undef ACCESSOR -#undef HAS_NO_DEFAULT -#undef HAS_DEFAULT -#undef TUPLE + bool help; + std::filesystem::path input_directory = "./"; // TODO + std::filesystem::path output_directory = DEFAULT_OUTPUT_DIRECTORY; // TODO + std::filesystem::path intermediate_results_directory = std::filesystem::path{"./temp/"}; // TODO + std::optional preprocessing_database_location; + std::optional duckdb_memory_limit_in_g; + std::optional lineage_definitions_file; + std::optional ndjson_input_filename; + std::filesystem::path database_config_file = "database_config.yaml"; // TODO + std::filesystem::path reference_genome_file = + "reference_genomes.json"; // TODO remove dup default - static void addOptions(Poco::Util::OptionSet& options); void validate() const; [[nodiscard]] std::filesystem::path getDatabaseConfigFilename() const; @@ -241,7 +124,17 @@ class PreprocessingConfig { [[nodiscard]] std::filesystem::path getAminoAcidInsertionsFilename() const; - void overwrite(const silo::config::AbstractConfigSource& config_source); + [[nodiscard]] std::filesystem::path getPreprocessingDatabaseLocation() const; + + [[nodiscard]] std::filesystem::path getIntermediateResultsDirectory() const; + + [[nodiscard]] uint32_t getDuckdbMemoryLimitInG() const; + + [[nodiscard]] bool asksForHelp() const override; + + void overwriteFrom(const silo::config::VerifiedConfigSource& config_source) override; + + [[nodiscard]] std::optional configPath() const override; }; } // namespace silo::config diff --git a/include/silo/config/runtime_config.h b/include/silo/config/runtime_config.h index ec0e88464..e653de473 100644 --- a/include/silo/config/runtime_config.h +++ b/include/silo/config/runtime_config.h @@ -6,7 +6,6 @@ #include #include "config/config_metadata.h" -#include "config/ignored.h" #include "config/toplevel_interface.h" #include "silo/config/config_defaults.h" @@ -37,9 +36,7 @@ struct RuntimeConfig : public ToplevelConfig { [[nodiscard]] bool asksForHelp() const override; [[nodiscard]] std::optional configPath() const override; - void overwriteFrom( - const VerifiedConfigSource& config_source - ) override; + void overwriteFrom(const VerifiedConfigSource& config_source) override; void validate() const {}; }; diff --git a/include/silo/config/util/abstract_config_source.h b/include/silo/config/util/abstract_config_source.h deleted file mode 100644 index 9f7501d31..000000000 --- a/include/silo/config/util/abstract_config_source.h +++ /dev/null @@ -1,56 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -namespace silo::config { - -/// A source of config values (from a config file, env vars, or -/// command line arguments). -/// -/// Values may be loaded at instantiation time of the object -/// implementing `AbstractConfigSource` (as is the case for -/// `YamlFile`), or be retrieved from the environment at query time -/// (the use of the get* methods) (as is the case for -/// `EnvironmentVariables`) or held via reference from -/// instantiation time (as is the case for `CommandLineArguments`). -/// -/// Config keys (represented via class `Option`) are lists of strings -/// in camel case, and used as such in yaml config files. For command -/// line arguments those are translated to kebab case (lower-case -/// joined '-' before uppercase characters), for environment variables -/// to uppercase with underscores and prefixed with -/// "SILO_". Multi-segment paths are treated as nested dictionaries in -/// yaml config files, joined with '-' for command line arguments and -/// '_' for environment variables. -class AbstractConfigSource { - public: - class Option { - public: - /// List of hierarchical option path segments, each of which in - /// camel case. - std::vector access_path; - - /// `access_path` joined with ".". --- XXX wrong that depends! - [[nodiscard]] std::string toString() const; - }; - - /// A human-readable description including type (command line, - /// config file, env var) and if applicable path to the file. - [[nodiscard]] virtual std::string configType() const = 0; - - /// Check if a value is available for the given key. - [[nodiscard]] virtual bool hasProperty(const Option& option) const = 0; - /// Retrieve a config value for the given key as a string - /// (potentially converting other value types). - [[nodiscard]] virtual std::optional getString(const Option& option) const = 0; - [[nodiscard]] virtual std::optional getInt32(const Option& option) const; - [[nodiscard]] virtual std::optional getUInt16(const Option& option) const; - [[nodiscard]] virtual std::optional getUInt32(const Option& option) const; - [[nodiscard]] virtual std::optional getUInt64(const Option& option) const; -}; - -} // namespace silo::config diff --git a/include/silo/config/util/yaml_file.h b/include/silo/config/util/yaml_file.h deleted file mode 100644 index 6f70f09be..000000000 --- a/include/silo/config/util/yaml_file.h +++ /dev/null @@ -1 +0,0 @@ -#pragma once diff --git a/include/silo/query_engine/actions/tuple.h b/include/silo/query_engine/actions/tuple.h index dc1bdd7a5..f1daa8d02 100644 --- a/include/silo/query_engine/actions/tuple.h +++ b/include/silo/query_engine/actions/tuple.h @@ -53,6 +53,7 @@ class Tuple { typedef std::function Comparator; Tuple(const silo::storage::ColumnPartitionGroup* columns, std::byte* data, size_t data_size); + Tuple(const Tuple& other); // TODO investigate Tuple(Tuple&& other) noexcept; Tuple& operator=(const Tuple& other); Tuple& operator=(Tuple&& other) noexcept; diff --git a/include/silo/test/query_fixture.test.h b/include/silo/test/query_fixture.test.h index a7f3a0783..33dcbe012 100644 --- a/include/silo/test/query_fixture.test.h +++ b/include/silo/test/query_fixture.test.h @@ -8,9 +8,9 @@ #include #include +#include "silo/common/fmt_formatters.h" #include "silo/config/database_config.h" #include "silo/config/preprocessing_config.h" -#include "silo/config/util/yaml_file.h" #include "silo/database.h" #include "silo/database_info.h" #include "silo/preprocessing/preprocessor.h" @@ -71,7 +71,7 @@ namespace silo::test { ASSERT_EQ(actual, scenario.expected_query_result); \ } \ } \ - } // namespace \ + } // namespace struct QueryTestData { const std::vector ndjson_input_data; @@ -99,11 +99,10 @@ class QueryTestFixture : public ::testing::TestWithParam { std::filesystem::path input_directory = fmt::format("test{}", millis); std::filesystem::create_directories(input_directory); - config::PreprocessingConfig config_with_input_dir{ - .input_directory = input_directory, - .intermediate_results_directory = input_directory, - .ndjson_input_filename = "input.json" - }; + config::PreprocessingConfig config_with_input_dir; + config_with_input_dir.input_directory = input_directory; + config_with_input_dir.intermediate_results_directory = input_directory; + config_with_input_dir.ndjson_input_filename = "input.json"; config_with_input_dir.validate(); DataContainer::input_directory = input_directory; @@ -117,7 +116,7 @@ class QueryTestFixture : public ::testing::TestWithParam { std::cerr << "Could not open file for writing" << std::endl; return; } - for (const auto json : test_data.ndjson_input_data) { + for (const auto& json : test_data.ndjson_input_data) { file << json.dump() << std::endl; } file.close(); diff --git a/include/silo_api/command_line_arguments.h b/include/silo_api/command_line_arguments.h deleted file mode 100644 index e8668d20b..000000000 --- a/include/silo_api/command_line_arguments.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include - -#include "silo/config/util/abstract_config_source.h" - -namespace silo_api { - -class CommandLineArguments : public silo::config::AbstractConfigSource { - const Poco::Util::AbstractConfiguration& config; - - public: - static std::string asUnixOptionString(const Option& option); - - explicit CommandLineArguments(const Poco::Util::AbstractConfiguration& config); - - [[nodiscard]] std::string configType() const override; - - [[nodiscard]] bool hasProperty(const Option& option) const override; - - [[nodiscard]] std::optional getString(const Option& option) const override; -}; - -} // namespace silo_api diff --git a/include/silo_api/environment_variables.h b/include/silo_api/environment_variables.h deleted file mode 100644 index 090493b39..000000000 --- a/include/silo_api/environment_variables.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include - -#include "silo/config/util/abstract_config_source.h" - -namespace silo_api { - -class EnvironmentVariables : public silo::config::AbstractConfigSource { - public: - static std::string prefixedUppercase(const Option& option); - - [[nodiscard]] std::string configType() const override; - - [[nodiscard]] bool hasProperty(const Option& option) const override; - - [[nodiscard]] std::optional getString(const Option& option) const override; -}; - -} // namespace silo_api diff --git a/src/config/config_metadata.cpp b/src/config/config_metadata.cpp index 1e4c3c8ff..454beb1ae 100644 --- a/src/config/config_metadata.cpp +++ b/src/config/config_metadata.cpp @@ -41,6 +41,19 @@ std::string indent(std::string_view indentation, const std::string& str) { namespace silo::config { +std::optional ConfigSpecification::getValueSpecification(const silo::config::ConfigKeyPath& key) const { + +} + +std::optional ConfigSpecification::getValueSpecificationStrict(const silo::config::ConfigKeyPath& key) const { + auto it = std::find_if(fields.begin(), fields.end(), [&](const ConfigValueSpecification& x){return x.key.path == key.path;}); + if(it == fields.end()){ + return std::nullopt; + } + return *it; +} + + std::string ConfigSpecification::helpText() const { std::string program_name = "TODOXX"; // TODO std::ostringstream help_text; @@ -58,7 +71,7 @@ std::string ConfigSpecification::helpText() const { for (const auto& field_spec : fields) { addln(""); - const std::string_view type_text = field_spec.isBool() + const std::string_view type_text = field_spec.type == ConfigValueType::BOOL ? " (boolean, the option implies 'true')" : configValueTypeToString(field_spec.type); addln(fmt::format( diff --git a/src/config/config_source_interface.cpp b/src/config/config_source_interface.cpp index 861203ad6..d3c8c8bc6 100644 --- a/src/config/config_source_interface.cpp +++ b/src/config/config_source_interface.cpp @@ -1 +1,114 @@ #include "config/config_source_interface.h" + +#include "silo/common/panic.h" + +namespace silo::config { + +std::optional VerifiedConfigSource::getString(const ConfigKeyPath& config_key_path +) const { + auto it = config_values.find(config_key_path); + if (it != config_values.end()) { + const ConfigValue& value = it->second; + if (value.getValueType() != ConfigValueType::STRING) { + PANIC( + "Called getString called on a ConfigKeyPath ('{}') that belongs to a value of another " + "type ({}).", + config_key_path.toDebugString(), + configValueTypeToString(value.getValueType()) + ); + } + return get(value.value); + } + return std::nullopt; +} + +std::optional VerifiedConfigSource::getPath( + const ConfigKeyPath& config_key_path +) const { + auto it = config_values.find(config_key_path); + if (it != config_values.end()) { + const ConfigValue& value = it->second; + if (value.getValueType() != ConfigValueType::PATH) { + PANIC( + "Called getPath called on a ConfigKeyPath ('{}') that belongs to a value of another " + "type ({}).", + config_key_path.toDebugString(), + configValueTypeToString(value.getValueType()) + ); + } + return get(value.value); + } + return std::nullopt; +} + +std::optional VerifiedConfigSource::getInt32(const ConfigKeyPath& config_key_path) const { + auto it = config_values.find(config_key_path); + if (it != config_values.end()) { + const ConfigValue& value = it->second; + if (value.getValueType() != ConfigValueType::INT32) { + PANIC( + "Called getInt32 called on a ConfigKeyPath ('{}') that belongs to a value of another " + "type ({}).", + config_key_path.toDebugString(), + configValueTypeToString(value.getValueType()) + ); + } + return get(value.value); + } + return std::nullopt; +} + +std::optional VerifiedConfigSource::getUint32(const ConfigKeyPath& config_key_path +) const { + auto it = config_values.find(config_key_path); + if (it != config_values.end()) { + const ConfigValue& value = it->second; + if (value.getValueType() != ConfigValueType::UINT32) { + PANIC( + "Called getUint32 called on a ConfigKeyPath ('{}') that belongs to a value of another " + "type ({}).", + config_key_path.toDebugString(), + configValueTypeToString(value.getValueType()) + ); + } + return get(value.value); + } + return std::nullopt; +} + +std::optional VerifiedConfigSource::getUint16(const ConfigKeyPath& config_key_path +) const { + auto it = config_values.find(config_key_path); + if (it != config_values.end()) { + const ConfigValue& value = it->second; + if (value.getValueType() != ConfigValueType::UINT16) { + PANIC( + "Called getUint16 called on a ConfigKeyPath ('{}') that belongs to a value of another " + "type ({}).", + config_key_path.toDebugString(), + configValueTypeToString(value.getValueType()) + ); + } + return get(value.value); + } + return std::nullopt; +} + +std::optional VerifiedConfigSource::getBool(const ConfigKeyPath& config_key_path) const { + auto it = config_values.find(config_key_path); + if (it != config_values.end()) { + const ConfigValue& value = it->second; + if (value.getValueType() != ConfigValueType::BOOL) { + PANIC( + "Called getBool called on a ConfigKeyPath ('{}') that belongs to a value of another " + "type ({}).", + config_key_path.toDebugString(), + configValueTypeToString(value.getValueType()) + ); + } + return get(value.value); + } + return std::nullopt; +} + +} // namespace silo::config diff --git a/src/config/config_value.cpp b/src/config/config_value.cpp new file mode 100644 index 000000000..26ea585ef --- /dev/null +++ b/src/config/config_value.cpp @@ -0,0 +1,55 @@ +#include "config/config_value.h" + +#include +#include + +namespace silo::config { + +ConfigValue ConfigValueSpecification::getValueFromString(std::string value_string) const { + switch (type) { + case ConfigValueType::STRING: + return createValue(value_string); + case ConfigValueType::PATH: { + std::filesystem::path path = value_string; + return createValue(path); + } + case ConfigValueType::UINT32: { + uint32_t parsed_unsigned = boost::lexical_cast(value_string); + return createValue(parsed_unsigned); + } + case ConfigValueType::UINT16: { + uint16_t parsed_unsigned = boost::lexical_cast(value_string); + return createValue(parsed_unsigned); + } + case ConfigValueType::INT32: { + int32_t parsed_signed = boost::lexical_cast(value_string); + return createValue(parsed_signed); + } + case ConfigValueType::BOOL: + return createValue(true); + } +} + +ConfigValue ConfigValueSpecification::createValue( + std::variant value +) const { + ConfigValue created_value{value}; + if (created_value.getValueType() != type) { + throw std::runtime_error( + "Internal Error: value created for this specification that is of the wrong type." + ); + } // TODO change to SILO_ASSERT + return created_value; +} + +} // namespace silo::config + +std::size_t std::hash::operator()( + const silo::config::ConfigKeyPath& key +) const { + std::size_t seed = 0; // Initialize seed for the hash + for (const auto& segment : key.path) { + boost::hash_combine(seed, segment); // Combine each segment's hash + } + return seed; // Return the final combined hash +} \ No newline at end of file diff --git a/src/config/source/command_line_arguments.cpp b/src/config/source/command_line_arguments.cpp index c44816bf6..e13f38dd1 100644 --- a/src/config/source/command_line_arguments.cpp +++ b/src/config/source/command_line_arguments.cpp @@ -3,7 +3,9 @@ #include #include +#include #include +#include #include "silo/common/panic.h" #include "silo/config/util/config_exception.h" @@ -12,25 +14,34 @@ namespace silo::config { std::string asUnixOptionString(const ConfigKeyPath& config_key_path) { std::vector result{"-"}; - for (const std::string& current_string : config_key_path.path) { - std::string current_result; - for (const char character : current_string) { - if (std::isupper(character)) { - current_result += '-'; - const char char_in_lower_case = - static_cast(std::tolower(static_cast(character))); - current_result += char_in_lower_case; - } else { - current_result += character; - } + for (const auto& sublevel : config_key_path.path) { + for (const std::string& current_string : sublevel) { + result.push_back(current_string); } - result.emplace_back(current_result); } - return boost::join(result, "."); + return boost::join(result, "-"); } ConfigKeyPath asConfigKeyPath(const std::string& command_line_argument) { - return {}; // TODO + if (command_line_argument.empty() || command_line_argument[0] != '-') { + throw std::invalid_argument("Invalid Unix option string"); + } + + ConfigKeyPath config_key_path; + // Remove the leading dash(es) and split by '-' + std::string trimmed = command_line_argument.substr(1); // Skip the first '-' + std::vector tokens; + + boost::split(tokens, trimmed, boost::is_any_of("-")); + + // Here, for simplicity, treat each token as its own sublevel + for (const auto& token : tokens) { + if (!token.empty()) { + config_key_path.path.push_back({token}); + } + } + + return config_key_path; } VerifiedConfigSource CommandLineArguments::verify(const ConfigSpecification& config_struct) const { @@ -52,46 +63,25 @@ VerifiedConfigSource CommandLineArguments::verify(const ConfigSpecification& con } break; } - const ConfigKeyPath key = asConfigKeyPath(arg); - if (auto value_specification_opt = config_struct.getValueSpecification(key)) { + const ConfigKeyPath ambiguous_key = asConfigKeyPath(arg); + if (auto value_specification_opt = config_struct.getValueSpecification(ambiguous_key)) { auto value_specification = value_specification_opt.value(); - ConfigValue value; - if (!value_specification.isBool()) { + std::string value_string; + if (value_specification.type == ConfigValueType::BOOL) { + value_string = "1"; + } else { ++i; if (i == args.size()) { // VerificationError::ParseError in Rust throw silo::config::ConfigException("missing argument after option " + arg); } + value_string = args[i]; } - switch (value_specification.type) { - case ConfigValueType::STRING: { - value = ConfigValue{args[i]}; - break; - } - case ConfigValueType::PATH: { - std::filesystem::path path = args[i]; - value = ConfigValue{path}; - break; - } - case ConfigValueType::UINT32: { - uint32_t parsed_unsigned = boost::lexical_cast(args[i]); - value = ConfigValue{parsed_unsigned}; - break; - } - case ConfigValueType::INT32: { - int32_t parsed_signed = boost::lexical_cast(args[i]); - value = ConfigValue{parsed_signed}; - break; - } - case ConfigValueType::BOOL: { - value = ConfigValue{true}; - break; - } - }; + ConfigValue value = value_specification.getValueFromString(value_string); // Overwrite value with the last occurrence // (i.e. `silo --foo 4 --foo 5` will leave "--foo" // => "5" in the map). - config_value_by_option.emplace(key, value); + config_value_by_option.emplace(value_specification.key, value); } else { invalid_config_keys.push_back(arg); } diff --git a/src/config/source/environment_variables.cpp b/src/config/source/environment_variables.cpp index 955e6c124..bd5c85251 100644 --- a/src/config/source/environment_variables.cpp +++ b/src/config/source/environment_variables.cpp @@ -15,19 +15,14 @@ using silo::config::ConfigKeyPath; std::string prefixedUppercase(const ConfigKeyPath& option) { std::vector result; - for (const std::string& current_string : option.path) { - std::string current_result; - for (const char character : current_string) { - if (std::isupper(character)) { - current_result += '_'; - current_result += character; - } else { - const char char_in_upper_case = - static_cast(std::toupper(static_cast(character))); - current_result += char_in_upper_case; - } + for (const auto& sublevel : option.path) { + for(const std::string& current_string : sublevel){ + std::string current_string_all_uppercase; + std::transform(current_string.begin(), current_string.end(), + std::back_inserter(current_string_all_uppercase), + [](unsigned char c) { return std::toupper(c); }); + result.push_back(current_string_all_uppercase); } - result.emplace_back(current_result); } return fmt::format("{}{}", env_var_prefix, boost::join(result, "_")); } @@ -64,12 +59,12 @@ EnvironmentVariables EnvironmentVariables::decodeEnvironmentVariables(const char std::unordered_map config_values; std::vector invalid_config_keys; for (const auto& [key_string, value_string] : alist) { - auto key = EnvironmentVariables::stringToConfigKeyPath(key_string); - auto value_specification_opt = config_specification.getValueSpecification(key); + auto ambiguous_key = EnvironmentVariables::stringToConfigKeyPath(key_string); + auto value_specification_opt = config_specification.getValueSpecification(ambiguous_key); if (value_specification_opt.has_value()) { auto value_specification = value_specification_opt.value(); ConfigValue value = value_specification.getValueFromString(value_string); - config_values[key] = value; + config_values.emplace(value_specification.key, value); } else { if (key_string == "SILO_PANIC") { SPDLOG_TRACE( diff --git a/src/config/source/yaml_file.cpp b/src/config/source/yaml_file.cpp index 1fe908d13..26d30fabd 100644 --- a/src/config/source/yaml_file.cpp +++ b/src/config/source/yaml_file.cpp @@ -32,19 +32,65 @@ bool isProperSingularValue(const YAML::Node& node) { return true; } +std::vector splitCamelCase(const std::string& camelCaseString){ + std::vector result; + std::string current; + + for (char ch : camelCaseString) { + if (std::isupper(ch)) { + // If current is not empty, push it to the result + if (!current.empty()) { + result.push_back(current); + current.clear(); + } + // Add the lowercase version of the uppercase char as the start of a new substring + current += std::tolower(ch); + } else { + // Append lowercase or non-uppercase char to current + current += ch; + } + } + // Push the last accumulated string to result + if (!current.empty()) { + result.push_back(current); + } + + return result; +} + +std::string joinCamelCase(const std::vector& words) { + std::string camelCaseString; + + for (size_t i = 0; i < words.size(); ++i) { + if (i == 0) { + // Add the first word as is (lowercase) + camelCaseString += words[i]; + } else { + // Capitalize the first character of subsequent words and append them + std::string word = words[i]; + if (!word.empty()) { + word[0] = std::toupper(word[0]); + camelCaseString += word; + } + } + } + + return camelCaseString; +} + void yamlToPaths( const std::string& config_context, const YAML::Node& node, - const ConsList& parents, + const ConsList>& parents, std::unordered_map& paths ) { if (node.IsMap()) { for (const auto& key_value : node) { const auto key = key_value.first.as(); // ^ XX what if key is not a string? - const auto parents2 = parents.cons(key); - const auto node = key_value.second; - yamlToPaths(config_context, node, parents2, paths); + const auto parents2 = parents.cons(splitCamelCase(key)); + const auto child_node = key_value.second; + yamlToPaths(config_context, child_node, parents2, paths); } } else { ConfigKeyPath path{parents.toVecReverse()}; @@ -63,7 +109,11 @@ void yamlToPaths( namespace silo::config { std::string YamlFile::configKeyPathToString(const ConfigKeyPath& config_key_path) { - return boost::join(config_key_path.path, "."); + std::vector camelCaseStrings; + for(const auto& list : config_key_path.path){ + camelCaseStrings.emplace_back(joinCamelCase(list)); + } + return boost::join(camelCaseStrings, "."); } YamlFile YamlFile::readFile(const std::filesystem::path& path) { @@ -73,7 +123,7 @@ YamlFile YamlFile::readFile(const std::filesystem::path& path) { auto node = YAML::LoadFile(path_string); // Collect all paths present std::unordered_map paths; - yamlToPaths(fmt::format("YAML file '{}'", path_string), node, ConsList{}, paths); + yamlToPaths(fmt::format("YAML file '{}'", path_string), node, ConsList>{}, paths); return YamlFile{path, paths}; } catch (const YAML::Exception& e) { @@ -87,7 +137,27 @@ std::string YamlFile::configContext() const { return fmt::format("YAML file '{}'", content_source.string()); } - +namespace { +ConfigValue yamlNodeToConfigValue( + const ConfigValueSpecification& value_specification, + const YAML::Node& yaml +) { + switch (value_specification.type) { + case ConfigValueType::STRING: + return value_specification.createValue(yaml.as()); + case ConfigValueType::PATH: + return value_specification.createValue({std::filesystem::path{yaml.as()}}); + case ConfigValueType::INT32: + return value_specification.createValue(yaml.as()); + case ConfigValueType::UINT32: + return value_specification.createValue(yaml.as()); + case ConfigValueType::UINT16: + return value_specification.createValue(yaml.as()); + case ConfigValueType::BOOL: + return value_specification.createValue(yaml.as()); + } +} +} // namespace VerifiedConfigSource YamlFile::verify(const ConfigSpecification& config_specification) const { // No need to stringify and do duplicate check since @@ -97,31 +167,12 @@ VerifiedConfigSource YamlFile::verify(const ConfigSpecification& config_specific std::vector invalid_config_keys; std::unordered_map provided_config_values; for (const auto& [key, yaml] : getYamlFields()) { - auto value_specification = config_specification.getValueSpecification(key); + auto value_specification = config_specification.getValueSpecificationStrict(key); if (!value_specification.has_value()) { invalid_config_keys.push_back(configKeyPathToString(key)); } else { - switch (value_specification->type) { - case ConfigValueType::STRING: - provided_config_values[key] = ConfigValue{yaml.as()}; - break; - case ConfigValueType::PATH: - provided_config_values[key] = - ConfigValue{std::filesystem::path{yaml.as()}}; - break; - case ConfigValueType::INT32: - provided_config_values[key] = - ConfigValue{yaml.as()}; - break; - case ConfigValueType::UINT32: - provided_config_values[key] = - ConfigValue{yaml.as()}; - break; - case ConfigValueType::BOOL: - provided_config_values[key] = - ConfigValue{yaml.as()}; - break; - } + ConfigValue value = yamlNodeToConfigValue(value_specification.value(), yaml); + provided_config_values.emplace(key, value); } } @@ -138,4 +189,8 @@ VerifiedConfigSource YamlFile::verify(const ConfigSpecification& config_specific return VerifiedConfigSource{provided_config_values}; } +const std::unordered_map& YamlFile::getYamlFields() const { + return yaml_fields; +} + } // namespace silo::config diff --git a/src/config/toplevel_interface.cpp b/src/config/toplevel_interface.cpp new file mode 100644 index 000000000..e7889fb8f --- /dev/null +++ b/src/config/toplevel_interface.cpp @@ -0,0 +1,56 @@ +#include "config/toplevel_interface.h" + +#include "config/source/command_line_arguments.h" +#include "config/source/environment_variables.h" +#include "config/source/yaml_file.h" + +namespace silo::config { + +template +std::optional rawGetConfig( + std::span cmd, + const ConfigSpecification& config_specification +) { + auto env_source = + EnvironmentVariables::decodeEnvironmentVariables().verify(config_specification); + auto cmd_source = CommandLineArguments{cmd}.verify(config_specification); + + const ConsList no_parents{}; + + C config; + + // First, only check command line arguments, for "--help"; avoid + // potential errors from env processing, and we don't have the + // path to the config file yet. Since we're only interested in the + // help option, there's no need to read config_struct first, OK? + config = {}; + config.overwriteFrom(no_parents, cmd_source); + if (config.asksForHelp()) { + return std::nullopt; + } + + // Then process env and cmd, to get to the config file + // path. Re-initialize since env must be processed for cmd. + config = {}; + config.overwriteFrom(no_parents, env_source); + config.overwriteFrom(no_parents, cmd_source); + // Would anyone request help via SILO_ENV=true? Well, allow it: + if (config.asksForHelp()) { + return std::nullopt; + } + + auto config_path = config.configPath(); + if (config_path.has_value()) { + auto file_source = YamlFile::readFile(*config_path).verify(config_specification); + // Now read again with the file first: + config = {}; + config.overwriteFrom(no_parents, file_source); + config.overwriteFrom(no_parents, env_source); + config.overwriteFrom(no_parents, cmd_source); + // (The config file might specify --help, too, but we ignore + // that.) + } + return std::optional{config}; +} + +} // namespace silo::config diff --git a/src/silo/config/preprocessing_config.cpp b/src/silo/config/preprocessing_config.cpp index ebb764659..d43603772 100644 --- a/src/silo/config/preprocessing_config.cpp +++ b/src/silo/config/preprocessing_config.cpp @@ -6,9 +6,7 @@ #include #include "silo/common/fmt_formatters.h" -#include "silo/config/util/abstract_config_source.h" #include "silo/preprocessing/preprocessing_exception.h" -#include "silo_api/command_line_arguments.h" namespace silo::config { @@ -16,19 +14,12 @@ void PreprocessingConfig::validate() const { if (!std::filesystem::exists(input_directory)) { throw preprocessing::PreprocessingException(input_directory.string() + " does not exist"); } - if (ndjson_input_filename.has_value() && metadata_file.has_value()) { + if (!ndjson_input_filename.has_value()) { throw preprocessing::PreprocessingException(fmt::format( - "Cannot specify both a ndjsonInputFilename ('{}') and metadataFilename('{}').", - ndjson_input_filename.value().string(), - metadata_file.value().string() + "TODO", // TODO get error message after rebase + ndjson_input_filename.value().string() )); } - if (!ndjson_input_filename.has_value() && !metadata_file.has_value()) { - throw preprocessing::PreprocessingException( - fmt::format("Neither a ndjsonInputFilename nor a metadataFilename was specified as " - "preprocessing option.") - ); - } } std::filesystem::path PreprocessingConfig::getDatabaseConfigFilename() const { @@ -37,8 +28,8 @@ std::filesystem::path PreprocessingConfig::getDatabaseConfigFilename() const { std::optional PreprocessingConfig::getPangoLineageDefinitionFilename( ) const { - return pango_lineage_definition_file.has_value() - ? std::optional(input_directory / *pango_lineage_definition_file) + return lineage_definitions_file.has_value() + ? std::optional(input_directory / lineage_definitions_file.value()) : std::nullopt; } @@ -46,104 +37,47 @@ std::filesystem::path PreprocessingConfig::getReferenceGenomeFilename() const { return input_directory / reference_genome_file; } -std::optional PreprocessingConfig::getMetadataInputFilename() const { - return metadata_file.has_value() ? std::optional(input_directory / *metadata_file) - : std::nullopt; -} - std::optional PreprocessingConfig::getNdjsonInputFilename() const { return ndjson_input_filename.has_value() ? std::optional(input_directory / *ndjson_input_filename) : std::nullopt; } -std::filesystem::path PreprocessingConfig::getNucFilenameNoExtension(size_t sequence_idx) const { - std::filesystem::path filename = input_directory; - filename /= fmt::format("{}{}", nucleotide_sequence_prefix, sequence_idx); - return filename; -} - -std::filesystem::path PreprocessingConfig::getUnalignedNucFilenameNoExtension(size_t sequence_idx -) const { - std::filesystem::path filename = input_directory; - filename /= fmt::format("{}{}", unaligned_nucleotide_sequence_prefix, sequence_idx); - return filename; -} - -std::filesystem::path PreprocessingConfig::getGeneFilenameNoExtension(size_t sequence_idx) const { - std::filesystem::path filename = input_directory; - filename /= fmt::format("{}{}", gene_prefix, sequence_idx); - return filename; -} - -std::filesystem::path PreprocessingConfig::getNucleotideInsertionsFilename() const { - return input_directory / nuc_insertions_filename; +bool PreprocessingConfig::asksForHelp() const { + return help; } -std::filesystem::path PreprocessingConfig::getAminoAcidInsertionsFilename() const { - return input_directory / aa_insertions_filename; -} - -namespace { -std::string toUnix(const AbstractConfigSource::Option& option) { - return silo_api::CommandLineArguments::asUnixOptionString(option); -} -} // namespace - -void PreprocessingConfig::addOptions(Poco::Util::OptionSet& options) { -#define TUPLE( \ - TYPE, \ - FIELD_NAME, \ - DEFAULT_GENERATION, \ - DEFAULT_VALUE, \ - OPTION_PATH, \ - PARSING_ACCESSOR_TYPE_NAME, \ - HELP_TEXT, \ - ACCESSOR_GENERATION, \ - ACCESSOR_NAME \ -) \ - { \ - const AbstractConfigSource::Option opt{OPTION_PATH}; \ - std::string option_string = toUnix(opt); \ - options.addOption(Poco::Util::Option() \ - .fullName(option_string) \ - .description(HELP_TEXT) \ - .required(false) \ - .repeatable(false) \ - .argument(#PARSING_ACCESSOR_TYPE_NAME) \ - .binding(option_string)); \ +void PreprocessingConfig::overwriteFrom(const VerifiedConfigSource& config_source) { + if (auto var = config_source.getBool(HELP_OPTION_KEY)) { + help = var.value(); } - - PREPROCESSING_CONFIG_DEFINITION; - -#undef TUPLE -} - -void PreprocessingConfig::overwrite(const silo::config::AbstractConfigSource& config_source) { -#define TUPLE( \ - TYPE, \ - FIELD_NAME, \ - DEFAULT_GENERATION, \ - DEFAULT_VALUE, \ - OPTION_PATH, \ - PARSING_ACCESSOR_TYPE_NAME, \ - HELP_TEXT, \ - ACCESSOR_GENERATION, \ - ACCESSOR_NAME \ -) \ - { \ - const AbstractConfigSource::Option opt{OPTION_PATH}; \ - if (auto value = config_source.get##PARSING_ACCESSOR_TYPE_NAME(opt)) { \ - SPDLOG_DEBUG( \ - "Using {} as passed via {}: {}", opt.toString(), config_source.configType(), *value \ - ); \ - (FIELD_NAME) = *value; \ - } \ + if (auto var = config_source.getPath(INPUT_DIRECTORY_OPTION_KEY)) { + input_directory = var.value(); + } + if (auto var = config_source.getPath(OUTPUT_DIRECTORY_OPTION_KEY)) { + output_directory = var.value(); + } + if (auto var = config_source.getPath(INTERMEDIATE_RESULTS_DIRECTORY_OPTION_KEY)) { + intermediate_results_directory = var.value(); + } + if (auto var = config_source.getPath(PREPROCESSING_DATABASE_LOCATION_OPTION_KEY)) { + preprocessing_database_location = var.value(); + } + if (auto var = config_source.getUint32(DUCKDB_MEMORY_LIMIT_IN_G_OPTION_KEY)) { + duckdb_memory_limit_in_g = var.value(); + } + if (auto var = config_source.getPath(LINEAGE_DEFINITIONS_FILE_OPTION_KEY)) { + lineage_definitions_file = var.value(); + } + if (auto var = config_source.getPath(NDJSON_INPUT_FILENAME_OPTION_KEY)) { + ndjson_input_filename = var.value(); + } + if (auto var = config_source.getPath(DATABASE_CONFIG_FILE_OPTION_KEY)) { + database_config_file = var.value(); + } + if (auto var = config_source.getPath(REFERENCE_GENOMES_FILENAME_OPTION_KEY)) { + reference_genome_file = var.value(); } - - PREPROCESSING_CONFIG_DEFINITION; - -#undef TUPLE } } // namespace silo::config @@ -155,25 +89,9 @@ void PreprocessingConfig::overwrite(const silo::config::AbstractConfigSource& co fmt::format_to(ctx.out(), "{{\n"); const char* perhaps_comma = " "; -#define TUPLE( \ - TYPE, \ - FIELD_NAME, \ - DEFAULT_GENERATION, \ - DEFAULT_VALUE, \ - OPTION_PATH, \ - PARSING_ACCESSOR_TYPE_NAME, \ - HELP_TEXT, \ - ACCESSOR_GENERATION, \ - ACCESSOR_NAME \ -) \ - fmt::format_to( \ - ctx.out(), "{} {}: ''", perhaps_comma, "#FIELD_NAME", preprocessing_config.FIELD_NAME \ - ); \ - perhaps_comma = ","; - - PREPROCESSING_CONFIG_DEFINITION; - -#undef TUPLE + + // TODO + (void) perhaps_comma; return fmt::format_to(ctx.out(), "}}\n"); } diff --git a/src/silo/config/runtime_config.cpp b/src/silo/config/runtime_config.cpp index 96918cc00..f77e6954c 100644 --- a/src/silo/config/runtime_config.cpp +++ b/src/silo/config/runtime_config.cpp @@ -6,71 +6,65 @@ #include #include "silo/common/fmt_formatters.h" -#include "silo/config/util/abstract_config_source.h" +#include "config/source/yaml_file.h" namespace silo::config { -const ConfigKeyPath HELP_OPTION_KEY{{"help"}}; -const ConfigKeyPath RUNTIME_CONFIG_OPTION_KEY{{"runtimeConfig"}}; -const ConfigKeyPath DATA_DIRECTORY_OPTION_KEY{{"dataDirectory"}}; -const ConfigKeyPath API_PORT_OPTION_KEY{{"api", "port"}}; -const ConfigKeyPath API_MAX_CONNECTIONS_OPTION_KEY{{"api", "maxQueuedHttpConnections"}}; -const ConfigKeyPath API_PARALLEL_THREADS_OPTION_KEY{{"api", "threadsForHttpConnections"}}; -const ConfigKeyPath API_ESTIMATED_STARTUP_TIME_OPTION_KEY{{"api", "estimatedStartupTimeInMinutes"}}; -const ConfigKeyPath QUERY_MATERIALIZATION_CUTOFF_OPTION_KEY{{"query", "materializationCutoff"}}; +const ConfigKeyPath HELP_OPTION_KEY = YamlFile::stringToConfigKeyPath("help"); +const ConfigKeyPath RUNTIME_CONFIG_OPTION_KEY = YamlFile::stringToConfigKeyPath("runtimeConfig"); +const ConfigKeyPath DATA_DIRECTORY_OPTION_KEY = YamlFile::stringToConfigKeyPath("dataDirectory"); +const ConfigKeyPath API_PORT_OPTION_KEY = YamlFile::stringToConfigKeyPath("api.port"); +const ConfigKeyPath API_MAX_CONNECTIONS_OPTION_KEY = YamlFile::stringToConfigKeyPath("api.maxQueuedHttpConnections"); +const ConfigKeyPath API_PARALLEL_THREADS_OPTION_KEY = YamlFile::stringToConfigKeyPath("api.threadsForHttpConnections"); +const ConfigKeyPath API_ESTIMATED_STARTUP_TIME_OPTION_KEY = YamlFile::stringToConfigKeyPath("api.estimatedStartupTimeInMinutes"); +const ConfigKeyPath QUERY_MATERIALIZATION_CUTOFF_OPTION_KEY = YamlFile::stringToConfigKeyPath("query.materializationCutoff"); -const ConfigSpecification API_OPTIONS_METADATA{ +const ConfigSpecification RUNTIME_CONFIG_SPECIFICATION{ "siloServer", { - ConfigValueSpecification{ - .key = HELP_OPTION_KEY, - .type = ConfigValueType::BOOL, - .help_text = "Show help text.", - }, - ConfigValueSpecification{ - .key = RUNTIME_CONFIG_OPTION_KEY, - .type = ConfigValueType::PATH, - .help_text = "Path to config file in YAML format.", - }, - ConfigValueSpecification{ - .key = DATA_DIRECTORY_OPTION_KEY, - .type = ConfigValueType::PATH, - .default_value = ConfigValue{DEFAULT_OUTPUT_DIRECTORY}, - .help_text = - "The path to the directory with the data files (output from preprocessing).", - }, - ConfigValueSpecification{ - .key = API_MAX_CONNECTIONS_OPTION_KEY, - .type = ConfigValueType::UINT32, - .default_value = ConfigValue{64}, - .help_text = "The maximum number of concurrent connections accepted at any time.", - }, - ConfigValueSpecification{ - .key = API_PARALLEL_THREADS_OPTION_KEY, - .type = ConfigValueType::UINT32, - .default_value = ConfigValue{4}, - .help_text = "The number of worker threads.", - }, - ConfigValueSpecification{ - .key = API_PORT_OPTION_KEY, - .type = ConfigValueType::UINT16, - .default_value = ConfigValue{8081}, - .help_text = "The port number on which to listen for incoming HTTP connections.", - }, - ConfigValueSpecification{ - .key = API_ESTIMATED_STARTUP_TIME_OPTION_KEY, - .type = ConfigValueType::UINT32, - .help_text = "Estimated time in minutes that the initial loading of the database takes. \n" - "As long as no database is loaded yet, SILO will throw a 503 error. \n" - "This option allows SILO to compute a Retry-After header for the 503 response.", - }, - ConfigValueSpecification{ - .key = QUERY_MATERIALIZATION_CUTOFF_OPTION_KEY, - .type = ConfigValueType::UINT32, - .default_value = ConfigValue{10000}, - .help_text = "Above how many records in a result set the result rows are to be constructed\n" - "lazily (by streaming).", - }, + ConfigValueSpecification::createWithoutDefault( + HELP_OPTION_KEY, + ConfigValueType::BOOL, + "Show help text." + ), + ConfigValueSpecification::createWithoutDefault( + RUNTIME_CONFIG_OPTION_KEY, + ConfigValueType::PATH, + "Path to config file in YAML format." + ), + ConfigValueSpecification::createWithDefault( + DATA_DIRECTORY_OPTION_KEY, + ConfigValue::fromPath(DEFAULT_OUTPUT_DIRECTORY), + "The path to the directory with the data files (output from preprocessing)." + ), + ConfigValueSpecification::createWithDefault( + API_MAX_CONNECTIONS_OPTION_KEY, + ConfigValue::fromUint32(64), + "The maximum number of concurrent connections accepted at any time." + ), + ConfigValueSpecification::createWithDefault( + API_PARALLEL_THREADS_OPTION_KEY, + ConfigValue::fromUint32(4), + "The number of worker threads." + ), + ConfigValueSpecification::createWithDefault( + API_PORT_OPTION_KEY, + ConfigValue::fromUint16(8081), + "The port number on which to listen for incoming HTTP connections." + ), + ConfigValueSpecification::createWithoutDefault( + API_ESTIMATED_STARTUP_TIME_OPTION_KEY, + ConfigValueType::UINT32, + "Estimated time in minutes that the initial loading of the database takes. \n" + "As long as no database is loaded yet, SILO will throw a 503 error. \n" + "This option allows SILO to compute a Retry-After header for the 503 response." + ), + ConfigValueSpecification::createWithDefault( + QUERY_MATERIALIZATION_CUTOFF_OPTION_KEY, + ConfigValue::fromUint32(10000), + "Above how many records in a result set the result rows are to be constructed\n" + "lazily (by streaming)." + ), } }; @@ -82,15 +76,11 @@ std::optional RuntimeConfig::configPath() const { return runtime_config; } -const ConfigKeyPath HELP_OPTION = {{"help"}}; - -void RuntimeConfig::overwriteFrom( - const VerifiedConfigSource& config_source -) { - if(auto var = config_source.getBool(HELP_OPTION)){ +void RuntimeConfig::overwriteFrom(const VerifiedConfigSource& config_source) { + if (auto var = config_source.getBool(HELP_OPTION_KEY)) { help = var.value(); } - if(auto var = config_source.getPath(RUNTIME_CONFIG_OPTION_KEY)){ + if (auto var = config_source.getPath(RUNTIME_CONFIG_OPTION_KEY)) { runtime_config = var.value(); } if (auto var = config_source.getPath(DATA_DIRECTORY_OPTION_KEY)) { @@ -122,11 +112,7 @@ void RuntimeConfig::overwriteFrom( const char* perhaps_comma = " "; fmt::format_to( - ctx.out(), - "{} {}: '{}'", - perhaps_comma, - "data_directory", - runtime_config.data_directory + ctx.out(), "{} {}: '{}'", perhaps_comma, "data_directory", runtime_config.data_directory ); #define CODE_FOR_FIELD(TOPLEVEL_FIELD, FIELD_NAME) \ diff --git a/src/silo/config/runtime_config.test.cpp b/src/silo/config/runtime_config.test.cpp index 250c3497c..301ffced8 100644 --- a/src/silo/config/runtime_config.test.cpp +++ b/src/silo/config/runtime_config.test.cpp @@ -2,7 +2,7 @@ #include -#include "silo/config/util/yaml_file.h" +#include "config/source/yaml_file.h" TEST(RuntimeConfig, shouldReadConfig) { silo::config::RuntimeConfig runtime_config; diff --git a/src/silo/config/util/abstract_config_source.cpp b/src/silo/config/util/abstract_config_source.cpp index 4f10dd687..e69de29bb 100644 --- a/src/silo/config/util/abstract_config_source.cpp +++ b/src/silo/config/util/abstract_config_source.cpp @@ -1,93 +0,0 @@ -#include "silo/config/util/abstract_config_source.h" - -#include -#include -#include -#include -#include - -#include "silo/config/util/config_exception.h" - -namespace silo::config { - -std::string AbstractConfigSource::Option::toString() const { - return boost::join(access_path, "."); -} - -std::optional AbstractConfigSource::getInt32(const Option& option) const { - const auto string_value = getString(option); - if (string_value == std::nullopt) { - return std::nullopt; - } - try { - return boost::lexical_cast(*string_value); - } catch (boost::bad_lexical_cast&) { - const std::string error_message = fmt::format( - "Could not cast the value '{}' from the {} option '{}' to a 32-bit signed integer.", - *string_value, - configType(), - option.toString() - ); - SPDLOG_ERROR(error_message); - throw ConfigException(error_message); - } -} - -std::optional AbstractConfigSource::getUInt16(const Option& option) const { - const auto string_value = getString(option); - if (string_value == std::nullopt) { - return std::nullopt; - } - try { - return boost::lexical_cast(*string_value); - } catch (boost::bad_lexical_cast&) { - const std::string error_message = fmt::format( - "Could not cast the value '{}' from the {} option '{}' to a 16-bit unsigned integer.", - *string_value, - configType(), - option.toString() - ); - SPDLOG_ERROR(error_message); - throw ConfigException(error_message); - } -} - -std::optional AbstractConfigSource::getUInt32(const Option& option) const { - const auto string_value = getString(option); - if (string_value == std::nullopt) { - return std::nullopt; - } - try { - return boost::lexical_cast(*string_value); - } catch (boost::bad_lexical_cast&) { - const std::string error_message = fmt::format( - "Could not cast the value '{}' from the {} option '{}' to a 32-bit unsigned integer.", - *string_value, - configType(), - option.toString() - ); - SPDLOG_ERROR(error_message); - throw ConfigException(error_message); - } -} - -std::optional AbstractConfigSource::getUInt64(const Option& option) const { - const auto string_value = getString(option); - if (string_value == std::nullopt) { - return std::nullopt; - } - try { - return boost::lexical_cast(*string_value); - } catch (boost::bad_lexical_cast&) { - const std::string error_message = fmt::format( - "Could not cast the value '{}' from the {} option '{}' to a 64-bit unsigned integer.", - *string_value, - configType(), - option.toString() - ); - SPDLOG_ERROR(error_message); - throw ConfigException(error_message); - } -} - -} // namespace silo::config diff --git a/src/silo/config/util/yaml_file.cpp b/src/silo/config/util/yaml_file.cpp deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/silo/config/util/yaml_file.test.cpp b/src/silo/config/util/yaml_file.test.cpp index 14a4c8fa3..94173d2df 100644 --- a/src/silo/config/util/yaml_file.test.cpp +++ b/src/silo/config/util/yaml_file.test.cpp @@ -3,18 +3,18 @@ #include #include -using silo::config::YamlFile; using silo::config::ConfigKeyPath; +using silo::config::YamlFile; TEST(YamlFile, canCorrectlyCheckForPresentPropertiesCaseSensitively) { const YamlFile under_test = YamlFile::readFile("./testBaseData/test_preprocessing_config.yaml"); const std::unordered_map expected_result{ - {ConfigKeyPath{{"inputDirectory"}}, YAML::Node{"./testBaseData/exampleDataset/"}}, - {ConfigKeyPath{{"outputDirectory"}}, YAML::Node{"./output/"}}, - {ConfigKeyPath{{"metadataFilename"}}, YAML::Node{"small_metadata_set.tsv"}}, - {ConfigKeyPath{{"pangoLineageDefinitionFilename"}}, YAML::Node{"pangolineage_alias.json"}}, - {ConfigKeyPath{{"referenceGenomeFilename"}}, YAML::Node{"reference_genomes.json"}}, + {YamlFile::stringToConfigKeyPath("inputDirectory"), YAML::Node{"./testBaseData/exampleDataset/"}}, + {YamlFile::stringToConfigKeyPath("outputDirectory"), YAML::Node{"./output/"}}, + {YamlFile::stringToConfigKeyPath("metadataFilename"), YAML::Node{"small_metadata_set.tsv"}}, + {YamlFile::stringToConfigKeyPath("pangoLineageDefinitionFilename"), YAML::Node{"pangolineage_alias.json"}}, + {YamlFile::stringToConfigKeyPath("referenceGenomeFilename"), YAML::Node{"reference_genomes.json"}}, }; ASSERT_EQ(under_test.getYamlFields(), expected_result); @@ -28,16 +28,19 @@ TEST(YamlFile, shouldThrowExceptionWhenConfigFileDoesNotExist) { } TEST(YamlFile, shouldReadAnotherConfig) { - ASSERT_NO_THROW(config.overwrite( - YamlFile("./testBaseData/test_preprocessing_config_with_overridden_defaults.yaml") - );); - - const std::string input_directory = "./testBaseData/exampleDataset/"; - ASSERT_EQ(config.getMetadataInputFilename(), input_directory + "small_metadata_set.tsv"); - ASSERT_EQ( - config.getPangoLineageDefinitionFilename(), input_directory + "pangolineage_alias.json" - ); + YamlFile under_test = + YamlFile::readFile("./testBaseData/test_preprocessing_config_with_overridden_defaults.yaml"); - ASSERT_EQ(config.getNucFilenameNoExtension(0), input_directory + "0"); - ASSERT_EQ(config.getOutputDirectory(), "./output/custom/"); + const std::unordered_map expected_result{ + {YamlFile::stringToConfigKeyPath("inputDirectory"), YAML::Node{"./testBaseData/exampleDataset/"}}, + {YamlFile::stringToConfigKeyPath("outputDirectory"), YAML::Node{"./output/custom/"}}, + {YamlFile::stringToConfigKeyPath("intermediateResultsDirectory"), YAML::Node{"./output/overriddenTemp/"}}, + {YamlFile::stringToConfigKeyPath("metadataFilename"), YAML::Node{"small_metadata_set.tsv"}}, + {YamlFile::stringToConfigKeyPath("pangoLineageDefinitionFilename"), YAML::Node{"pangolineage_alias.json"}}, + {YamlFile::stringToConfigKeyPath("referenceGenomeFilename"), YAML::Node{"reference_genomes.json"}}, + {YamlFile::stringToConfigKeyPath("genePrefix"), YAML::Node{"aaSeq_"}}, + {YamlFile::stringToConfigKeyPath("nucleotideSequencePrefix"), YAML::Node{""}}, + }; + + ASSERT_EQ(under_test.getYamlFields(), expected_result); } diff --git a/src/silo/database.test.cpp b/src/silo/database.test.cpp index 1f194dbbd..7a2bedf61 100644 --- a/src/silo/database.test.cpp +++ b/src/silo/database.test.cpp @@ -4,10 +4,10 @@ #include +#include "config/source/yaml_file.h" #include "silo/common/nucleotide_symbols.h" #include "silo/config/preprocessing_config.h" #include "silo/config/util/config_repository.h" -#include "silo/config/util/yaml_file.h" #include "silo/database_info.h" #include "silo/preprocessing/preprocessor.h" #include "silo/preprocessing/sql_function.h" @@ -19,7 +19,10 @@ silo::Database buildTestDatabase() { const std::string input_directory{"./testBaseData/unitTestDummyDataset/"}; silo::config::PreprocessingConfig config; - config.overwrite(silo::config::YamlFile(input_directory + "preprocessing_config.yaml")); + config.overwriteFrom( + silo::config::YamlFile::readFile(input_directory + "preprocessing_config.yaml") + .verify(silo::config::PREPROCESSING_CONFIG_SPECIFICATION) + ); const auto database_config = silo::config::ConfigRepository().getValidatedConfig(input_directory + "database_config.yaml"); @@ -51,7 +54,10 @@ TEST(DatabaseTest, shouldSuccessfullyBuildDatabaseWithoutPartitionBy) { const std::string input_directory{"./testBaseData/"}; silo::config::PreprocessingConfig config; - config.overwrite(silo::config::YamlFile(input_directory + "test_preprocessing_config.yaml")); + config.overwriteFrom( + silo::config::YamlFile::readFile(input_directory + "test_preprocessing_config.yaml") + .verify(silo::config::PREPROCESSING_CONFIG_SPECIFICATION) + ); const auto database_config = silo::config::ConfigRepository().getValidatedConfig( input_directory + "test_database_config_without_partition_by.yaml" diff --git a/src/silo/preprocessing/preprocessor.test.cpp b/src/silo/preprocessing/preprocessor.test.cpp index 9ce80e315..dc00c70b1 100644 --- a/src/silo/preprocessing/preprocessor.test.cpp +++ b/src/silo/preprocessing/preprocessor.test.cpp @@ -3,8 +3,8 @@ #include #include +#include "config/source/yaml_file.h" #include "silo/config/util/config_repository.h" -#include "silo/config/util/yaml_file.h" #include "silo/database.h" #include "silo/database_info.h" #include "silo/preprocessing/sql_function.h" @@ -301,9 +301,13 @@ INSTANTIATE_TEST_SUITE_P( TEST_P(PreprocessorTestFixture, shouldProcessData) { const auto scenario = GetParam(); - silo::config::PreprocessingConfig config{.input_directory = scenario.input_directory}; + silo::config::PreprocessingConfig config; + config.input_directory = scenario.input_directory; - config.overwrite(silo::config::YamlFile(scenario.input_directory / "preprocessing_config.yaml")); + config.overwriteFrom( + silo::config::YamlFile::readFile(scenario.input_directory / "preprocessing_config.yaml") + .verify(silo::config::PREPROCESSING_CONFIG_SPECIFICATION) + ); const auto database_config = silo::config::ConfigRepository().getValidatedConfig( scenario.input_directory / "database_config.yaml" diff --git a/src/silo/query_engine/actions/tuple.test.cpp b/src/silo/query_engine/actions/tuple.test.cpp index b961044c2..d2915dcad 100644 --- a/src/silo/query_engine/actions/tuple.test.cpp +++ b/src/silo/query_engine/actions/tuple.test.cpp @@ -184,7 +184,7 @@ TEST(TupleFactory, allocatesOneAllocatesManyEqual) { auto under_test_vector = factory.allocateMany(1); ASSERT_EQ(under_test_vector.size(), 1); factory.overwrite(under_test_vector.front(), 0); - ASSERT_EQ(under_test1, under_test_vector.front()); + ASSERT(under_test1 == under_test_vector.front()); } TEST(Tuple, equalityOperatorEquatesCorrectly) { @@ -194,12 +194,12 @@ TEST(Tuple, equalityOperatorEquatesCorrectly) { const Tuple under_test0b = factory.allocateOne(0); const Tuple under_test1 = factory.allocateOne(1); const Tuple under_test2 = factory.allocateOne(2); - ASSERT_EQ(under_test0a, under_test0b); - ASSERT_NE(under_test0a, under_test1); - ASSERT_EQ(under_test0a, under_test2); - ASSERT_NE(under_test0b, under_test1); - ASSERT_EQ(under_test0b, under_test2); - ASSERT_NE(under_test1, under_test2); + ASSERT(under_test0a == under_test0b); + ASSERT(under_test0a != under_test1); + ASSERT(under_test0a == under_test2); + ASSERT(under_test0b != under_test1); + ASSERT(under_test0b == under_test2); + ASSERT(under_test1 != under_test2); } TEST(Tuple, comparesFieldsCorrectly) { diff --git a/src/silo/query_engine/operators/index_scan.test.cpp b/src/silo/query_engine/operators/index_scan.test.cpp index 6ec76a90b..66f4c769a 100644 --- a/src/silo/query_engine/operators/index_scan.test.cpp +++ b/src/silo/query_engine/operators/index_scan.test.cpp @@ -1,5 +1,6 @@ #include "silo/query_engine/operators/index_scan.h" +#include #include #include @@ -13,7 +14,7 @@ using silo::query_engine::operators::IndexScan; TEST(OperatorIndexScan, evaluateShouldReturnCorrectValues) { const roaring::Roaring test_bitmap(roaring::Roaring({1, 3})); const IndexScan under_test(&test_bitmap, 5); - ASSERT_EQ(*under_test.evaluate(), roaring::Roaring({1, 3})); + ASSERT(*under_test.evaluate() == roaring::Roaring({1, 3})); // TODO } TEST(OperatorIndexScan, correctTypeInfo) { @@ -21,7 +22,7 @@ TEST(OperatorIndexScan, correctTypeInfo) { const IndexScan under_test(&test_bitmap, 5); - ASSERT_EQ(under_test.type(), silo::query_engine::operators::INDEX_SCAN); + ASSERT(under_test.type() == silo::query_engine::operators::INDEX_SCAN); // TODO } TEST(OperatorIndexScan, correctLogicalEquivalent) { diff --git a/src/silo_api/command_line_arguments.cpp b/src/silo_api/command_line_arguments.cpp deleted file mode 100644 index 1492facef..000000000 --- a/src/silo_api/command_line_arguments.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include "silo_api/command_line_arguments.h" - -#include -#include -#include - -#include "silo/config/util/config_exception.h" - -namespace silo_api { - -std::string CommandLineArguments::asUnixOptionString( - const silo::config::AbstractConfigSource::Option& option -) { - std::vector result; - for (const std::string& current_string : option.access_path) { - std::string current_result; - for (const char character : current_string) { - if (std::isupper(character)) { - current_result += '-'; - const char char_in_lower_case = - static_cast(std::tolower(static_cast(character))); - current_result += char_in_lower_case; - } else { - current_result += character; - } - } - result.emplace_back(current_result); - } - return boost::join(result, "-"); -} - -CommandLineArguments::CommandLineArguments(const Poco::Util::AbstractConfiguration& config) - : config(config) {} - -std::string CommandLineArguments::configType() const { - return "command line argument"; -} - -bool CommandLineArguments::hasProperty(const Option& option) const { - return config.hasProperty(asUnixOptionString(option)); -} - -std::optional CommandLineArguments::getString(const Option& option) const { - if (hasProperty(option)) { - return config.getString(asUnixOptionString(option)); - } - return std::nullopt; -} - -} // namespace silo_api diff --git a/src/silo_api/command_line_arguments.test.cpp b/src/silo_api/command_line_arguments.test.cpp index 08e08df76..39ee7ce2b 100644 --- a/src/silo_api/command_line_arguments.test.cpp +++ b/src/silo_api/command_line_arguments.test.cpp @@ -1,27 +1,25 @@ -#include "silo_api/command_line_arguments.h" +#include "config/source/command_line_arguments.h" #include +using silo::config::CommandLineArguments; + TEST(CommandLineArguments, correctUnixOptionString) { - ASSERT_EQ(silo_api::CommandLineArguments::asUnixOptionString({{""}}), ""); - ASSERT_EQ(silo_api::CommandLineArguments::asUnixOptionString({{"A"}}), "-a"); - ASSERT_EQ(silo_api::CommandLineArguments::asUnixOptionString({{"abc"}}), "abc"); - ASSERT_EQ( - silo_api::CommandLineArguments::asUnixOptionString({{"someCamelCase"}}), "some-camel-case" - ); - ASSERT_EQ( - silo_api::CommandLineArguments::asUnixOptionString({{"BADCamelCase"}}), "-b-a-d-camel-case" - ); + ASSERT_EQ(CommandLineArguments::configKeyPathToString({{{""}}}), ""); + ASSERT_EQ(CommandLineArguments::configKeyPathToString({{{"", "A"}}}), "-a"); + ASSERT_EQ(CommandLineArguments::configKeyPathToString({{{"abc"}}}), "abc"); + ASSERT_EQ(CommandLineArguments::configKeyPathToString({{{"someCamelCase"}}}), "some-camel-case"); + ASSERT_EQ(CommandLineArguments::configKeyPathToString({{{"BADCamelCase"}}}), "-b-a-d-camel-case"); ASSERT_EQ( - silo_api::CommandLineArguments::asUnixOptionString({{"something_with_underscores"}}), + CommandLineArguments::configKeyPathToString({{{"something_with_underscores"}}}), "something_with_underscores" ); ASSERT_EQ( - silo_api::CommandLineArguments::asUnixOptionString({{"some", "subsectionedSequence"}}), + CommandLineArguments::configKeyPathToString({{"some", "subsectionedSequence"}}), "some-subsectioned-sequence" ); ASSERT_EQ( - silo_api::CommandLineArguments::asUnixOptionString({{"some", "more", "sections"}}), + CommandLineArguments::configKeyPathToString({{{"some", "more", "sections"}}}), "some-more-sections" ); } \ No newline at end of file diff --git a/src/silo_api/environment_variables.cpp b/src/silo_api/environment_variables.cpp deleted file mode 100644 index 503573d3c..000000000 --- a/src/silo_api/environment_variables.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include "silo_api/environment_variables.h" - -#include -#include -#include -#include - -namespace silo_api { - -std::string EnvironmentVariables::prefixedUppercase(const Option& option) { - std::vector result; - for (const std::string& current_string : option.access_path) { - std::string current_result; - for (const char character : current_string) { - if (std::isupper(character)) { - current_result += '_'; - current_result += character; - } else { - const char char_in_upper_case = - static_cast(std::toupper(static_cast(character))); - current_result += char_in_upper_case; - } - } - result.emplace_back(current_result); - } - return "SILO_" + boost::join(result, "_"); -} - -std::string EnvironmentVariables::configType() const { - return "environment variable"; -} - -bool EnvironmentVariables::hasProperty(const Option& option) const { - return Poco::Environment::has(prefixedUppercase(option)); -} - -std::optional EnvironmentVariables::getString(const Option& option) const { - if (hasProperty(option)) { - return Poco::Environment::get(prefixedUppercase(option)); - } - return std::nullopt; -} - -} // namespace silo_api diff --git a/src/silo_api/environment_variables.test.cpp b/src/silo_api/environment_variables.test.cpp index bcce98499..3dc4cc892 100644 --- a/src/silo_api/environment_variables.test.cpp +++ b/src/silo_api/environment_variables.test.cpp @@ -1,33 +1,34 @@ -#include "silo_api/environment_variables.h" +#include "config/source/environment_variables.h" #include #include +using silo::config::EnvironmentVariables; + TEST(EnvironmentVariables, correctPrefixedUppercase) { - ASSERT_EQ(silo_api::EnvironmentVariables::prefixedUppercase({{""}}), "SILO_"); - ASSERT_EQ(silo_api::EnvironmentVariables::prefixedUppercase({{"A"}}), "SILO__A"); - ASSERT_EQ(silo_api::EnvironmentVariables::prefixedUppercase({{"abc"}}), "SILO_ABC"); + ASSERT_EQ(EnvironmentVariables::configKeyPathToString({{{""}}}), "SILO_"); + ASSERT_EQ(EnvironmentVariables::configKeyPathToString({{{"A"}}}), "SILO__A"); + ASSERT_EQ(EnvironmentVariables::configKeyPathToString({{{"abc"}}}), "SILO_ABC"); ASSERT_EQ( - silo_api::EnvironmentVariables::prefixedUppercase({{"someCamelCase"}}), "SILO_SOME_CAMEL_CASE" + EnvironmentVariables::configKeyPathToString({{{"someCamelCase"}}}), "SILO_SOME_CAMEL_CASE" ); ASSERT_EQ( - silo_api::EnvironmentVariables::prefixedUppercase({{"BADCamelCase"}}), - "SILO__B_A_D_CAMEL_CASE" + EnvironmentVariables::configKeyPathToString({{{"BADCamelCase"}}}), "SILO__B_A_D_CAMEL_CASE" ); ASSERT_EQ( - silo_api::EnvironmentVariables::prefixedUppercase({{"something_with_underscores"}}), + EnvironmentVariables::configKeyPathToString({{{"something_with_underscores"}}}), "SILO_SOMETHING_WITH_UNDERSCORES" ); ASSERT_EQ( - silo_api::EnvironmentVariables::prefixedUppercase({{"something_with_underscores"}}), + EnvironmentVariables::configKeyPathToString({{{"something_with_underscores"}}}), "SILO_SOMETHING_WITH_UNDERSCORES" ); ASSERT_EQ( - silo_api::EnvironmentVariables::prefixedUppercase({{"some", "subsectionedSequence"}}), + EnvironmentVariables::configKeyPathToString({{{"some"}, {"subsectionedSequence"}}}), "SILO_SOME_SUBSECTIONED_SEQUENCE" ); ASSERT_EQ( - silo_api::EnvironmentVariables::prefixedUppercase({{"some", "more", "sections"}}), + EnvironmentVariables::configKeyPathToString({{{"some"}, {"more"}, {"sections"}}}), "SILO_SOME_MORE_SECTIONS" ); } \ No newline at end of file