wip

.run/silo api.run.xml

@@ -0,0 +1,10 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="silo api" type="CMakeRunConfiguration" factoryName="Application" PROGRAM_PARAMS="api --data-directory output/ --api-estimated-startup-time-in-minutes 2" REDIRECT_INPUT="false" ELEVATE="false" USE_EXTERNAL_CONSOLE="false" EMULATE_TERMINAL="false" WORKING_DIR="file://.." PASS_PARENT_ENVS_2="true" PROJECT_NAME="SILO" TARGET_NAME="siloApi" CONFIG_NAME="Debug" RUN_TARGET_PROJECT_NAME="SILO" RUN_TARGET_NAME="siloApi">
+    <envs>
+      <env name="SPDLOG_LEVEL" value="debug" />
+    </envs>
+    <method v="2">
+      <option name="com.jetbrains.cidr.execution.CidrBuildBeforeRunTaskProvider$BuildBeforeRunTask" enabled="true" />
+    </method>
+  </configuration>
+</component>

.run/silo preprocessing.run.xml

@@ -0,0 +1,10 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="silo preprocessing" type="CMakeRunConfiguration" factoryName="Application" PROGRAM_PARAMS="preprocessing --preprocessing-config testBaseData/test_preprocessing_config.yaml --database-config testBaseData/exampleDataset/database_config.yaml" REDIRECT_INPUT="false" ELEVATE="false" USE_EXTERNAL_CONSOLE="false" EMULATE_TERMINAL="false" WORKING_DIR="file://../" PASS_PARENT_ENVS_2="true" PROJECT_NAME="SILO" TARGET_NAME="siloApi" CONFIG_NAME="Debug" RUN_TARGET_PROJECT_NAME="SILO" RUN_TARGET_NAME="siloApi">
+    <envs>
+      <env name="SPDLOG_LEVEL" value="debug" />
+    </envs>
+    <method v="2">
+      <option name="com.jetbrains.cidr.execution.CidrBuildBeforeRunTaskProvider$BuildBeforeRunTask" enabled="true" />
+    </method>
+  </configuration>
+</component>

include/config/config_interface.h

@@ -16,27 +16,21 @@
 
 namespace silo::config {
 
-/* #[error("error in {config_context}: unknown key(s) {invalid_config_keys:?}")] */
-struct InvalidConfigKeyError {  // TODO move?
-   std::string config_context;
-   std::vector<std::string> invalid_config_keys;
-};
-/* #[error("error in {config_context}: {message}")] */
-struct ParseError {  // TODO move?
-   std::string config_context;
-   std::string message;
-};
-
-/// For top-level config structs (containing help and possibly config
+/// For config structs (containing help and possibly config
 /// file paths):
-class Config {
+template <typename C>
+concept Config = requires(C c, const C cc, const VerifiedConfigSource& config_source) {
+
    /// Whether the user gave the --help option or environment
    /// variable equivalent.
-   [[nodiscard]] virtual bool asksForHelp() const = 0;
+   /// bool asksForHelp() const = 0;
+   { cc.asksForHelp() } -> std::same_as<bool>;
+
 
    /// Optional config file that the user gave (or that is provided
    /// by the type via its defaults) that should be loaded.
-   [[nodiscard]] virtual std::optional<std::filesystem::path> configPath() const = 0;
+   /// std::optional<std::filesystem::path> configPath() const = 0;
+   { cc.configPath() } -> std::same_as<std::optional<std::filesystem::path>>;
 
    /// Overwrite the fields of an instance of the target type; done
    /// that way so that multiple kinds of config sources can shadow
@@ -45,72 +39,55 @@ class Config {
    /// .to_vec_reverse() and wrap in ConfigKeyPath). Throws
    /// `silo::config::ConfigException` for config value parse errors
    /// (subclass as ConfigValueParseError?).
-   virtual void overwriteFrom(const VerifiedConfigSource& config_source) = 0;
-};
-
-template <typename C>
-std::optional<C> rawGetConfig(
-   std::span<const std::string> cmd,
-   const ConfigSpecification& config_specification
-) {
-   auto env_source =
-      EnvironmentVariables::decodeEnvironmentVariables().verify(config_specification);
-   auto cmd_source = CommandLineArguments{cmd}.verify(config_specification);
-
-   const ConsList<std::string> no_parents{};
-
-   C config;
-
-   // First, only check command line arguments, for "--help"; avoid
-   // potential errors from env processing, and we don't have the
-   // path to the config file yet. Since we're only interested in the
-   // help option, there's no need to read config_struct first, OK?
-   config = {};
-   config.overwriteFrom(cmd_source);
-   if (config.asksForHelp()) {
-      return std::nullopt;
-   }
-
-   // Then process env and cmd, to get to the config file
-   // path. Re-initialize since env must be processed for cmd.
-   config = {};
-   config.overwriteFrom(env_source);
-   config.overwriteFrom(cmd_source);
-   // Would anyone request help via SILO_ENV=true? Well, allow it:
-   if (config.asksForHelp()) {
-      return std::nullopt;
-   }
+   /// void overwriteFrom(const VerifiedConfigSource& config_source) = 0;
+   { c.overwriteFrom(config_source) } -> std::same_as<void>;
 
-   auto config_path = config.configPath();
-   if (config_path.has_value()) {
-      auto file_source = YamlFile::readFile(*config_path).verify(config_specification);
-      // Now read again with the file first:
-      config = {};
-      config.overwriteFrom(file_source);
-      config.overwriteFrom(env_source);
-      config.overwriteFrom(cmd_source);
-      // (The config file might specify --help, too, but we ignore
-      // that.)
-   }
-   return std::optional{config};
-}
+   /// Validation / Sanity checks about the values of this config
+   /// void validate() = 0;
+   { c.validate() } -> std::same_as<void>;
+};
 
 /// In case of error, returns the exit code that the caller should
 /// pass to exit(): 0 if the user gave --help, 1 in case of erroneous
 /// usage (the error is already printed in that case).
-template <typename C>
+template <Config C>
 std::variant<C, int32_t> getConfig(
-   std::span<const std::string> cmd,
-   const ConfigSpecification& config_specification
+   std::span<const std::string> cmd
 ) {
+   const auto config_specification = C::getConfigSpecification();
    try {
-      auto config = rawGetConfig<C>(cmd, config_specification);
-      if (!config.has_value()) {
-         // User requested help
+
+      auto env_source =
+         EnvironmentVariables::decodeEnvironmentVariables().verify(config_specification);
+      auto cmd_source = CommandLineArguments{cmd}.verify(config_specification);
+
+      C config;
+
+      // First, only check command line arguments, for "--help"; avoid
+      // potential errors from env processing, and we don't have the
+      // path to the config file yet.
+      config = {};
+      config.overwriteFrom(cmd_source);
+      if (config.asksForHelp()) {
          std::cout << config_specification.helpText() << "\n" << std::flush;
          return 0;
       }
-      return std::move(*config);
+      config.overwriteFrom(env_source);
+
+      // Was a config file given as an argument or by environment variable?
+      auto config_path = config.configPath();
+      if (config_path.has_value()) {
+         auto file_source = YamlFile::readFile(*config_path).verify(config_specification);
+         // Now read again with the file first:
+         config = {};
+         config.overwriteFrom(file_source);
+         config.overwriteFrom(env_source);
+         config.overwriteFrom(cmd_source);
+         // (The config file might specify --help, too, but we ignore
+         // that.)
+      }
+      config.validate();
+      return std::move(config);
    } catch (const silo::config::ConfigException& e) {
       std::cerr << fmt::format(
                       "Usage error: {}.\n\nRun with the --help option for help.\n", e.what()

include/config/config_specification.h

@@ -20,7 +20,6 @@ namespace silo::config {
 /// currently.
 class ConfigSpecification : public VerifiedConfigSource {
   public:
-   std::string_view config_name;  // TODO check unused
    /// The name of the program for which this config is used. This will be printed in the help text
    std::string_view program_name;
    // Using std::vector so that initialization in place is possible;

include/silo/config/preprocessing_config.h

@@ -16,7 +16,7 @@
 
 namespace silo::config {
 
-class PreprocessingConfig : public Config {
+class PreprocessingConfig {
    friend class fmt::formatter<silo::config::PreprocessingConfig>;
 
   public:
@@ -31,8 +31,7 @@ class PreprocessingConfig : public Config {
    std::filesystem::path database_config_file = "database_config.yaml";  // TODO
    std::filesystem::path reference_genome_file =
       "reference_genomes.json";  // TODO remove dup default
-
-   std::filesystem::path preprocessing_config;
+   std::optional<std::filesystem::path> preprocessing_config;
 
    /// Create PreprocessingConfig with all default values from the specification
    PreprocessingConfig();
@@ -51,11 +50,11 @@ class PreprocessingConfig : public Config {
 
    [[nodiscard]] uint32_t getDuckdbMemoryLimitInG() const;
 
-   [[nodiscard]] bool asksForHelp() const override;
+   [[nodiscard]] bool asksForHelp() const;
 
-   void overwriteFrom(const silo::config::VerifiedConfigSource& config_source) override;
+   void overwriteFrom(const silo::config::VerifiedConfigSource& config_source);
 
-   [[nodiscard]] std::optional<std::filesystem::path> configPath() const override;
+   [[nodiscard]] std::optional<std::filesystem::path> configPath() const;
 };
 
 }  // namespace silo::config

include/silo/config/runtime_config.h

@@ -25,7 +25,7 @@ struct QueryOptions {
    size_t materialization_cutoff = 10000;
 };
 
-struct RuntimeConfig : public Config {
+struct RuntimeConfig {
    bool help;
    std::optional<std::filesystem::path> runtime_config;
    std::filesystem::path data_directory = silo::config::DEFAULT_OUTPUT_DIRECTORY;
@@ -34,12 +34,14 @@ struct RuntimeConfig : public Config {
 
    static ConfigSpecification getConfigSpecification();
 
-   [[nodiscard]] bool asksForHelp() const override;
-   [[nodiscard]] std::optional<std::filesystem::path> configPath() const override;
+   void validate() const {};
 
-   void overwriteFrom(const VerifiedConfigSource& config_source) override;
+   [[nodiscard]] bool asksForHelp() const;
+
+   [[nodiscard]] std::optional<std::filesystem::path> configPath() const;
+
+   void overwriteFrom(const VerifiedConfigSource& config_source);
 
-   void validate() const {};
 };
 
 }  // namespace silo::config

src/silo/config/preprocessing_config.cpp

@@ -27,14 +27,13 @@ const ConfigKeyPath LINEAGE_DEFINITIONS_FILE_OPTION_KEY =
 const ConfigKeyPath NDJSON_INPUT_FILENAME_OPTION_KEY =
    YamlFile::stringToConfigKeyPath("ndjsonInputFilename");
 const ConfigKeyPath DATABASE_CONFIG_FILE_OPTION_KEY =
-   YamlFile::stringToConfigKeyPath("databaseConfigFile");
+   YamlFile::stringToConfigKeyPath("databaseConfig");
 const ConfigKeyPath REFERENCE_GENOMES_FILENAME_OPTION_KEY =
    YamlFile::stringToConfigKeyPath("referenceGenomeFilename");
 
 // Specification of the fields in inputs to the PreprocessingConfig struct
 ConfigSpecification PreprocessingConfig::getConfigSpecification() {
    return ConfigSpecification{
-      .config_name = "preprocessing_config",
       .program_name = "siloPreprocessing",
       .fields{
          ConfigValueSpecification::createWithoutDefault(
@@ -95,7 +94,9 @@ ConfigSpecification PreprocessingConfig::getConfigSpecification() {
    };
 }
 
-PreprocessingConfig::PreprocessingConfig() {}
+PreprocessingConfig::PreprocessingConfig() {
+   overwriteFrom(getConfigSpecification());
+}
 
 void PreprocessingConfig::validate() const {
    if (!std::filesystem::exists(input_directory)) {

src/silo/config/runtime_config.cpp

@@ -25,7 +25,6 @@ const ConfigKeyPath QUERY_MATERIALIZATION_CUTOFF_OPTION_KEY =
 
 ConfigSpecification RuntimeConfig::getConfigSpecification() {
    return {
-      .config_name = "runtime config",
       .program_name = "siloServer",
       .fields =
          {

src/silo_api/main.cpp

@@ -6,6 +6,7 @@
 
 #include "silo/common/overloaded.h"
 #include "silo/config/runtime_config.h"
+#include "silo/config/preprocessing_config.h"
 #include "silo_api/logging.h"
 
 int main(int argc, char** argv) {
@@ -27,24 +28,33 @@ int main(int argc, char** argv) {
    } else if (!args.empty()) {
       const std::string& mode_argument = args[0];
       args = {args.begin() + 1, args.end()};
-      if (mode_argument == "preprocess") {
+      if (mode_argument == "preprocessing") {
          mode = 0;
       } else if (mode_argument == "api") {
          mode = 1;
       } else {
          std::cerr << program_name
-                   << ": need either 'preprocess' or 'api' as the first program argument, got '"
+                   << ": need either 'preprocessing' or 'api' as the first program argument, got '"
                    << mode_argument << "'\n";
          return 1;
       }
    } else {
       std::cerr << program_name
-                << ": need either 'preprocess' or 'api' as the first program argument\n";
+                << ": need either 'preprocessing' or 'api' as the first program argument\n";
       return 1;
    }
 
    if (mode == 0) {
-      // XXX preprocessing config
+      return std::visit(
+         overloaded{
+            [&](const silo::config::PreprocessingConfig& preprocessing_config) {
+               SPDLOG_TRACE("preprocessing_config = {}", preprocessing_config);
+               return 0;
+            },
+            [&](int32_t exit_code) { return exit_code; }
+         },
+         silo::config::getConfig<silo::config::PreprocessingConfig>(args)
+      );
    } else {
       return std::visit(
          overloaded{
@@ -54,9 +64,7 @@ int main(int argc, char** argv) {
             },
             [&](int32_t exit_code) { return exit_code; }
          },
-         getConfig<silo::config::RuntimeConfig>(
-            args, silo::config::RuntimeConfig::getConfigSpecification()
-         )  // TODO duplication of RuntimeConfig
+         silo::config::getConfig<silo::config::RuntimeConfig>(args)
       );
    }
 }