temporary commandlinesplit alex's branch. For github diff and better reading for team

.gitignore

@@ -69,3 +69,6 @@ conanprofile
 **/temp/
 
 **/node_modules/
+
+
+/test*

.run/silo api.run.xml

@@ -0,0 +1,10 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="silo api" type="CMakeRunConfiguration" factoryName="Application" PROGRAM_PARAMS="api --data-directory output/ --api-estimated-startup-time-in-minutes 2" REDIRECT_INPUT="false" ELEVATE="false" USE_EXTERNAL_CONSOLE="false" EMULATE_TERMINAL="false" WORKING_DIR="file://.." PASS_PARENT_ENVS_2="true" PROJECT_NAME="SILO" TARGET_NAME="silo" CONFIG_NAME="Debug" RUN_TARGET_PROJECT_NAME="SILO" RUN_TARGET_NAME="silo">
+    <envs>
+      <env name="SPDLOG_LEVEL" value="debug" />
+    </envs>
+    <method v="2">
+      <option name="com.jetbrains.cidr.execution.CidrBuildBeforeRunTaskProvider$BuildBeforeRunTask" enabled="true" />
+    </method>
+  </configuration>
+</component>

.run/silo preprocessing.run.xml

@@ -0,0 +1,10 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="silo preprocessing" type="CMakeRunConfiguration" factoryName="Application" PROGRAM_PARAMS="preprocessing --preprocessing-config testBaseData/test_preprocessing_config.yaml --database-config database_config.yaml --preprocessing-config testBaseData/exampleDataset/preprocessing_config.yaml" REDIRECT_INPUT="false" ELEVATE="false" USE_EXTERNAL_CONSOLE="false" EMULATE_TERMINAL="false" WORKING_DIR="file://../" PASS_PARENT_ENVS_2="true" PROJECT_NAME="SILO" TARGET_NAME="silo" CONFIG_NAME="Debug" RUN_TARGET_PROJECT_NAME="SILO" RUN_TARGET_NAME="silo">
+    <envs>
+      <env name="SPDLOG_LEVEL" value="debug" />
+    </envs>
+    <method v="2">
+      <option name="com.jetbrains.cidr.execution.CidrBuildBeforeRunTaskProvider$BuildBeforeRunTask" enabled="true" />
+    </method>
+  </configuration>
+</component>

CMakeLists.txt

@@ -65,7 +65,7 @@ file(GLOB_RECURSE SRC_SILO "src/*.cpp")
 list(REMOVE_ITEM SRC_SILO ${SRC_TEST})
 
 set(SRC_SILO_WITHOUT_MAIN ${SRC_SILO})
-list(REMOVE_ITEM SRC_SILO_WITHOUT_MAIN "${CMAKE_SOURCE_DIR}/src/silo_api/api.cpp")
+list(REMOVE_ITEM SRC_SILO_WITHOUT_MAIN "${CMAKE_SOURCE_DIR}/src/main.cpp")
 
 # ---------------------------------------------------------------------------
 # Linter
@@ -105,9 +105,9 @@ target_link_libraries(
         re2::re2
 )
 
-add_executable(siloApi "${CMAKE_SOURCE_DIR}/src/silo_api/api.cpp" $<TARGET_OBJECTS:silolib>)
+add_executable(silo "${CMAKE_SOURCE_DIR}/src/main.cpp" $<TARGET_OBJECTS:silolib>)
 target_link_libraries(
-        siloApi
+        silo
         PUBLIC
         silolib
 )

Dockerfile

@@ -7,15 +7,15 @@ COPY . ./
 RUN  \
     python3 ./build_with_conan.py --release --parallel 4\
     && cp build/Release/silo_test . \
-    && cp build/Release/siloApi .
+    && cp --no-dereference build/Release/{silo,siloServer,siloPreprocessor} .
 
 
 FROM ubuntu:22.04 AS server
 
 WORKDIR /app
 COPY docker_default_preprocessing_config.yaml ./default_preprocessing_config.yaml
 COPY docker_runtime_config.yaml ./runtime_config.yaml
-COPY --from=builder /src/siloApi ./
+COPY --from=builder /src/{silo,siloServer,siloPreprocessor} ./
 
 RUN apt update && apt dist-upgrade -y \
     &&  apt install -y libtbb12 curl jq
@@ -25,7 +25,7 @@ HEALTHCHECK --start-period=20s CMD curl --fail --silent localhost:8081/info | jq
 
 EXPOSE 8081
 
-ENTRYPOINT ["./siloApi"]
+ENTRYPOINT ["./silo"]
 
 LABEL org.opencontainers.image.source="https://github.com/GenSpectrum/LAPIS-SILO"
 LABEL org.opencontainers.image.description="Sequence Indexing engine for Large Order of genomic data"

Makefile

@@ -0,0 +1,61 @@
+# Note: you can set the COLOR environment variable to 1 to get the log
+# files produced with color codes.
+
+export PATH := bin:$(PATH)
+
+all: build/siloServer build/siloPreprocessor build/silo_test
+
+clean:
+	find build -name "*.o" -print0 | xargs -0 rm -f
+	rm -f build/siloApi build/silo_test
+
+# Have separate targets for the binaries, but share a single build
+# (faster to build them both?); use `run-cached` to only actually run one
+# build, though.
+
+build/siloServer: build/silo
+
+build/siloPreprocessor: build/silo
+
+build/silo: $(shell bin/cplusplus-source-files)
+	run-cached build/.exit-code build $@
+
+build/silo_test: $(shell bin/cplusplus-source-files)
+	run-cached build/.exit-code build $@
+
+
+# Tests produce log files if successful (if not successful, the log
+# file can be found with .tmp appended, but is also printed to stdout).
+
+build/unit-tests.log: build/silo_test
+	run-with-log $@ build/silo_test
+
+build/preprocessing-tsv.log: \
+        build/siloPreprocessor \
+        $(shell bin/preprocessing-input-files-in testBaseData/exampleDataset)
+	run-with-log $@ preprocessing-in testBaseData/exampleDataset
+
+build/preprocessing-ndjson.log: \
+        build/siloPreprocessor \
+        $(shell bin/preprocessing-input-files-in testBaseData/exampleDatasetAsNdjson)
+	run-with-log $@ preprocessing-in testBaseData/exampleDatasetAsNdjson
+
+build/tsv-tests.log: build/siloServer build/preprocessing-tsv.log $(shell bin/test-query-files)
+	run-with-log $@ runtests-e2e testBaseData/exampleDataset 7001
+
+build/ndjson-tests.log: build/siloServer build/preprocessing-ndjson.log $(shell bin/test-query-files)
+	run-with-log $@ runtests-e2e testBaseData/exampleDatasetAsNdjson 7002
+
+test: build/unit-tests.log build/tsv-tests.log build/ndjson-tests.log
+
+
+# Manually run the api so that it can be queried interactively.
+
+runapi-tsv: build/siloServer build/preprocessing-tsv.log
+	runapi-in testBaseData/exampleDataset 8081
+
+runapi-ndjson: build/siloServer build/preprocessing-tsv.log
+	runapi-in testBaseData/exampleDatasetAsNdjson 8081
+
+
+.PHONY: all clean test runapi-tsv runapi-ndjson

bin/build

@@ -0,0 +1,52 @@
+#!/bin/bash
+set -meuo pipefail
+IFS=
+
+usage() {
+    echo "usage: $0 [...]"
+    echo "  Auto-detect the mold linker and ninja builder and"
+    echo "  call cmake accordingly. Requires up to date configuration"
+    echo "  via conan first. Currently only fully runs on Linux and"
+    echo "  Mac OS, detection for core count on Windows is missing."
+    echo "  Arguments are passed on to make/ninja."
+    exit 1
+}
+
+if [[ $# -gt 0 ]] && { [[ "$1" = "-h" ]] || [[ "$1" = "--help" ]] ; }; then
+    usage
+fi
+
+args=(../)
+
+if _mold_path=$(which mold); then
+    args=("${args[@]}" -D CMAKE_EXE_LINKER_FLAGS=-fuse-ld=mold)
+fi
+
+if _ninja_path=$(which ninja); then
+    args=("${args[@]}" -G Ninja)
+    make=(ninja)
+else
+    # Linux: /proc/cpuinfo
+    # Mac OS: sysctl -n hw.ncpu
+    corecount=$(grep -c ^processor /proc/cpuinfo || sysctl -n hw.ncpu)
+    make=(make "-j$corecount")
+fi
+
+_do() {
+    echo "+" "$@"
+    "$@"
+}
+
+_do cd build
+_do cmake "${args[@]}"
+# Make all binaries..
+_do "${make[@]}" 
+
+# ..but touch the desired ones afterwards for the top-level Makefile's
+# sake (XX sigh, alternatives?)
+#if [ $# -gt 0 ]; then
+#    cd ..
+#    touch "$@"
+#fi
+# --- nah, just let it re-run build, sadly, still better than re-running tests unnecessarily.
+

bin/cplusplus-source-files

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -meuo pipefail
+IFS=
+
+find src/ include/ -name "*.h" -o -name "*.cpp" | grep -v '#'
+

bin/gco-make-test

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -emuo pipefail
+IFS=
+
+set -x
+
+while read -r commit; do
+    git checkout "$commit"
+    make -j16 test
+done
+

bin/preprocessing-in

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -meuo pipefail
+IFS=
+
+usage() {
+    echo "$0 test-input-dir" >&2
+    false
+}
+
+if [[ $# != 1 ]]; then
+    usage
+fi
+if [[ "$1" == "-h" ]] || [[ "$1" == "--help" ]]; then
+    usage
+fi
+testdir="$1"
+mypath=$(readlink -f "$0")
+mydir=$(dirname "$mypath")
+builddir=$(dirname "$mydir")/build
+
+IFS=' '
+
+set +x
+
+rm -rf temp
+
+cd "$testdir"
+
+rm -rf {output,temp,test?*}
+
+
+#SPDLOG_LEVEL=debug
+${DEBUGGER-} "$builddir"/siloPreprocessor

bin/preprocessing-input-files-in

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -meuo pipefail
+IFS=
+
+# preprocessing inputs
+bin/verified-find "$1" -maxdepth 1 -a \( -name "*.fasta" -o -name "*.yaml" -o -name "*.xz" \
+     -o -name "*.zst" -o -name "*.json" -o -name "*.tsv" -o -name "*.ndjson" \)

bin/run-cached

@@ -0,0 +1,80 @@
+#!/bin/bash
+set -meuo pipefail
+IFS=
+
+usage() {
+    echo "usage: $0 [-v|--verbose] file-for-exitcode cmd [args...]"
+    echo "  Takes a lock on file-for-exitcode, if it gets the lock,"
+    echo "  runs cmd with args and stores the exit code in file-for-exitcode."
+    echo "  If the file is already locked, waits until unlocked then reads"
+    echo "  the file contents as the exit code."
+    echo "  Exits with the exitcode in either case."
+    echo "  If -v/--verbose is given, prints to stderr when it's waiting for"
+    echo "  another instance."
+    exit 1
+}
+
+if [[ $# -lt 1 ]]; then
+    usage
+fi
+
+if [[ "$1" = -v ]] || [[ "$1" = --verbose ]]; then
+    verbose=1
+    shift
+else
+    verbose=0
+fi
+
+if [[ $# -lt 2 ]]; then
+    usage
+fi
+
+cachepath="$1"
+shift
+
+info() {
+    if [[ "$verbose" = 1 ]]; then
+        echo "$@" >&2
+    fi
+}
+
+
+# Open the cache file read/write to an unused file descriptor.
+exec {fd}<>"$cachepath"
+
+if flock -E77 --nonblock "$fd"; then
+    # We have an exclusive lock, so we are supposed to run the cmd;
+    # don't fail if it doesn't exit successfully.
+
+    # Remove previously stored code. (Have to reopen, `>&"$fd"` does
+    # not truncate.)
+    true > "$cachepath"
+
+    if "$@"; then
+        code=$?
+    else
+        code=$?
+    fi
+    echo "$code" >&"$fd"
+    exit "$code"
+else
+    e=$?
+    if [[ "$e" == 77 ]]; then
+        # Another instance is running the command already; wait it out
+        # then read out the exit code.
+        info -n "$0 $cachepath: waiting for concurrent run to finish..."
+        flock "$fd"
+        # `read` will fail if the file is empty (possible if the other
+        # $0-instance was killed)
+        if read -r code <&"$fd"; then
+            info "exited with code $code"
+            exit "$code"
+        else
+            info "other $0 instance was killed"
+            exit 130
+        fi
+    else
+        echo "$0: got exit code $e, is the 'flock' command not available?" >&2
+        false
+    fi
+fi