fixup! feat!: remove tsv/fasta/sam input format (#562)

GenSpectrum · Nov 14, 2024 · ac9fda0 · ac9fda0
1 parent 0245972
commit ac9fda0
Show file tree

Hide file tree

Showing 7 changed files with 5 additions and 146 deletions.
diff --git a/include/silo/common/input_stream_wrapper.h b/include/silo/common/input_stream_wrapper.h
diff --git a/include/silo/preprocessing/preprocessor.h b/include/silo/preprocessing/preprocessor.h
@@ -87,12 +87,6 @@ class Preprocessor {
       silo::SequenceStorePartition<SymbolType>& sequence_store
    );
 
-   template <typename SymbolType>
-   ColumnFunction createInsertionLambda(
-      const std::string& sequence_name,
-      silo::SequenceStorePartition<SymbolType>& sequence_store
-   );
-
    template <typename SymbolType>
    void buildSequenceStore(
       Database& database,

diff --git a/src/silo/common/input_stream_wrapper.cpp b/src/silo/common/input_stream_wrapper.cpp
diff --git a/src/silo/config/preprocessing_config.test.cpp b/src/silo/config/preprocessing_config.test.cpp
@@ -37,6 +37,8 @@ TEST(PreprocessingConfig, shouldReadConfigWithOverriddenDefaults) {
    const std::string input_directory = "./testBaseData/exampleDataset/";
    ASSERT_EQ(config.getNdjsonInputFilename(), input_directory + "input_file.ndjson");
    ASSERT_EQ(config.getLineageDefinitionsFilename(), input_directory + "lineage_definitions.yaml");
+   ASSERT_EQ(config.getDuckdbMemoryLimitInG(), 8);
+   ASSERT_EQ(config.getPreprocessingDatabaseLocation(), "preprocessing.duckdb");
 
    ASSERT_EQ(config.getOutputDirectory(), "./output/custom/");
 }

diff --git a/src/silo/preprocessing/preprocessor.cpp b/src/silo/preprocessing/preprocessor.cpp
@@ -12,12 +12,9 @@
 #include "silo/common/table_reader.h"
 #include "silo/config/preprocessing_config.h"
 #include "silo/database.h"
-#include "silo/database_info.h"
 #include "silo/preprocessing/metadata_info.h"
 #include "silo/preprocessing/preprocessing_database.h"
 #include "silo/preprocessing/preprocessing_exception.h"
-#include "silo/preprocessing/sequence_info.h"
-#include "silo/preprocessing/sql_function.h"
 #include "silo/preprocessing/validated_ndjson_file.h"
 #include "silo/storage/reference_genomes.h"
 #include "silo/storage/unaligned_sequence_store.h"

diff --git a/src/silo/preprocessing/preprocessor.test.cpp b/src/silo/preprocessing/preprocessor.test.cpp
@@ -99,62 +99,6 @@ const Scenario NDJSON_WITH_NUMERIC_NAMES = {
    )
 };
 
-const Scenario TSV_FILE_WITH_SQL_KEYWORD_AS_FIELD = {
-   .input_directory = "testBaseData/tsvWithSqlKeywordField/",
-   .expected_sequence_count = NDJSON_WITH_SQL_KEYWORD_AS_FIELD.expected_sequence_count,
-   .query = NDJSON_WITH_SQL_KEYWORD_AS_FIELD.query,
-   .expected_query_result = NDJSON_WITH_SQL_KEYWORD_AS_FIELD.expected_query_result
-};
-
-const Scenario TSV_FILE_WITH_QUOTE_IN_FIELD_NAME = {
-   .input_directory = "testBaseData/tsvWithQuoteInFieldName/",
-   .expected_sequence_count = 2,
-   .query = R"(
-{
-   "action": {
-      "type": "Aggregated",
-      "groupByFields": ["x\"y"],
-      "orderByFields": ["x\"y"]
-   },
-   "filterExpression": {
-      "type": "StringEquals",
-      "column": "x\"y",
-      "value": "a"
-   }
-}
-   )",
-   .expected_query_result = nlohmann::json::parse(
-      R"([
-         {"count": 1, "x\"y": "a"}
-   ])"
-   )
-};
-
-const Scenario TSV_FILE_WITH_QUOTE_IN_PARTITION_BY = {
-   .input_directory = "testBaseData/tsvWithQuoteInPartitionBy/",
-   .expected_sequence_count = 100,
-   .query = R"(
-{
-   "action": {
-      "type": "Aggregated",
-      "groupByFields": ["pango_\"lineage"],
-      "orderByFields": ["pango_\"lineage"],
-      "limit": 3
-   },
-   "filterExpression": {
-      "type": "True"
-   }
-}
-   )",
-   .expected_query_result = nlohmann::json::parse(
-      R"([
-         {"count":1,"pango_\"lineage":null},
-         {"count":1,"pango_\"lineage":"AY.122"},
-         {"count":4,"pango_\"lineage":"AY.43"}
-   ])"
-   )
-};
-
 const Scenario EMPTY_INPUT_NDJSON = {
    .input_directory = "testBaseData/emptyInputNdjson/",
    .expected_sequence_count = 0,

diff --git a/testBaseData/test_preprocessing_config_with_overridden_defaults.yaml b/testBaseData/test_preprocessing_config_with_overridden_defaults.yaml
@@ -3,4 +3,6 @@ outputDirectory: "./output/custom/"
 intermediateResultsDirectory: "./output/overriddenTemp/"
 ndjsonInputFilename: "input_file.ndjson"
 lineageDefinitionsFilename: "lineage_definitions.yaml"
-referenceGenomeFilename: "reference_genomes.json"
+referenceGenomeFilename: "reference_genomes.json"
+preprocessingDatabaseLocation: "preprocessing.duckdb"
+duckdbMemoryLimitInG: 8