Merge pull request #243 from waveygang/static-build

reimplement static builds

.github/workflows/test_on_push.yml

@@ -27,14 +27,15 @@ jobs:
           autoconf
           libgsl-dev
           zlib1g-dev
+          libdeflate-dev
           libhts-dev
           samtools
           libjemalloc-dev
           bedtools
       - name: Init and update submodules
         run: git submodule update --init --recursive
       - name: Build wfmash
-        run: cmake -H. -Bbuild -D CMAKE_BUILD_TYPE=Debug -DWFA_PNG_TSV_TIMING=ON && cmake --build build -- -j 2
+        run: cmake -H. -Bbuild -D CMAKE_BUILD_TYPE=Debug -DWFA_PNG_AND_TSV=ON && cmake --build build -- -j 2
       - name: Test mapping coverage with 8 yeast genomes (PAF output)
         run: ASAN_OPTIONS=detect_leaks=1:symbolize=1 LSAN_OPTIONS=verbosity=0:log_threads=1 build/bin/wfmash data/scerevisiae8.fa.gz -p 95 -n 7 -m -L -Y '#' > scerevisiae8.paf; scripts/test.sh data/scerevisiae8.fa.gz.fai scerevisiae8.paf 0.92
       - name: Test mapping+alignment with a subset of the LPA dataset (PAF output)

CMakeLists.txt

@@ -1,17 +1,34 @@
-cmake_minimum_required(VERSION 3.2 FATAL_ERROR)
-project(wfmash VERSION 0.0.1)
+cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
+project(wfmash)
+
+enable_testing()
+
+include(CheckIPOSupported) # adds lto
+check_ipo_supported(RESULT ipo_supported OUTPUT output)
 
 include(GNUInstallDirs)
 include(CheckCXXCompilerFlag)
 
 set(CMAKE_CXX_STANDARD 17)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)  # Falling back to different standard it not allowed.
+set(CMAKE_CXX_STANDARD_REQUIRED ON)  # Falling back to different standard is not allowed.
 set(CMAKE_CXX_EXTENSIONS OFF)  # Make sure no compiler-specific features are used.
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 
-#set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE) # does not work
+option(BUILD_STATIC "Build static binary" OFF)
+
+if (BUILD_STATIC)
+  set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
+  set(BUILD_SHARED_LIBS OFF)
+  set(CMAKE_EXE_LINKER_FLAGS "-static")
+endif ()
+
+find_package(PkgConfig REQUIRED)
+include(FindCURL) # for htslib?
+find_package(ZLIB REQUIRED)
+find_package(BZip2 REQUIRED)
+find_package(LibLZMA REQUIRED)
+find_package(Threads REQUIRED)
 
-#set(CMAKE_BUILD_TYPE Release)
 if(NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE Release CACHE STRING
             "Choose the type of build, options are: Release Debug Generic." FORCE)
@@ -20,22 +37,22 @@ endif()
 message(STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
 
 if (${CMAKE_BUILD_TYPE} MATCHES Release)
-    #set(EXTRA_FLAGS "-Ofast -march=native -flto -fno-fat-lto-objects")
-    set(EXTRA_FLAGS "-Ofast -march=native")
-    set(CMAKE_CXX_FLAGS_RELEASE "-DNDEBUG") # reset CXX_FLAGS to replace -O3 with -Ofast
+    set(CMAKE_CXX_FLAGS_RELEASE "-DNDEBUG")
+    set(CMAKE_C_FLAGS_RELEASE "-DNDEBUG")
+    if (NOT EXTRA_FLAGS)
+      set(EXTRA_FLAGS "-Ofast -march=native")
+    endif()
 endif ()
 
 if (${CMAKE_BUILD_TYPE} MATCHES Debug)
-    # Debug use the defaults - so commenting out:
     set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O -g -fsanitize=address")
     set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O -g -fsanitize=address")
 else()
-    # Use all standard-compliant optimizations - always add these:
     set (CMAKE_C_FLAGS "${OpenMP_C_FLAGS} ${PIC_FLAG} ${EXTRA_FLAGS}")
     set (CMAKE_CXX_FLAGS "${OpenMP_CXX_FLAGS} ${PIC_FLAG} ${EXTRA_FLAGS}")
 endif ()
 
-if(WFA_PNG_TSV_TIMING)
+if(WFA_PNG_AND_TSV)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DWFA_PNG_TSV_TIMING")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DWFA_PNG_TSV_TIMING")
 endif ()
@@ -45,19 +62,18 @@ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 
 add_subdirectory(src/common/wflign EXCLUDE_FROM_ALL)
-#add_subdirectory(src/common/wflign/deps/WFA2-lib EXCLUDE_FROM_ALL)
 
 add_executable(wfmash
   src/common/utils.cpp
-  src/interface/main.cpp) #$(CXX) $(CXXFLAGS) $(CPPFLAGS) -Lsrc/common/wflign/build/lib -Isrc/common/wflign/deps -Isrc/common $(SOURCE_1) -o wfmash @mathlib@ -lstdc++ -lz -lm -lpthread -lwflambda -lwflign
+  src/interface/main.cpp)
+
 target_include_directories(wfmash PRIVATE
   src
   src/common
   src/common/wflign/deps
   src/common/wflign/deps/WFA2-lib
-  )
-#target_link_directories(wfmash PRIVATE
-#  src/common/wflign/deps/WFA2-lib/lib)
+)
+
 target_link_libraries(wfmash
   gsl
   gslcblas
@@ -66,22 +82,31 @@ target_link_libraries(wfmash
   libwflign_static
   hts
   rt
-  wfa2cpp
+  wfa2cpp_static
   jemalloc
+  lzma
+  bz2
   z
+  deflate
+  Threads::Threads
 )
 
+configure_file(${CMAKE_SOURCE_DIR}/CTestCustom.cmake ${CMAKE_BINARY_DIR})
+
+add_test(
+  NAME wfmash-test
+  COMMAND ./build/bin/wfmash data/LPA.subset.fa.gz -p 80 -n 5 -t 8
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+
 install(TARGETS wfmash DESTINATION bin)
 
-install(TARGETS wfa2cpp
+install(TARGETS wfa2cpp_static
         LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
         PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 
-install(TARGETS wfa2
+install(TARGETS wfa2_static
         LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
         PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 
-
-# version stuff
 file(MAKE_DIRECTORY ${CMAKE_SOURCE_DIR}/include)
 execute_process(COMMAND bash ${CMAKE_SOURCE_DIR}/scripts/generate_git_version.sh ${CMAKE_SOURCE_DIR}/src)

CTestCustom.cmake

@@ -0,0 +1,3 @@
+CTestCustom.cmake
+
+

README.md

@@ -133,18 +133,28 @@ After installing the required dependencies, clone the `wfmash` git repository an
 ```
 git clone --recursive https://github.com/ekg/wfmash.git
 cd wfmash
-cmake -H. -Bbuild && cmake --build build -- -j 3
+cmake -H. -Bbuild && cmake --build build -- -j 8
 ```
 
+Of course, you can use as many cores as you like.
+
 If your system has several versions of the `gcc`/`g++` compilers you might tell `cmake` which one to use with:
 
 ```
 cmake -H. -Bbuild -DCMAKE_C_COMPILER='/usr/bin/gcc-10' -DCMAKE_CXX_COMPILER='/usr/bin/g++-10'
-cmake --build build -- -j 3
+cmake --build build -- -j 8
 ```
-
 The `wfmash` binary will be in `build/bin`.
 
+#### Static compilation
+
+By default, we build `wfmash` in Release mode (with optimizations) and as a dynamically linked executable.
+Alternatively we can build a static binary:
+
+```
+cmake -H. -Bbuild -DBUILD_STATIC=ON && cmake --build build -- -j 16
+```
+
 #### Notes for distribution
 
 If you need to avoid machine-specific optimizations, use the `CMAKE_BUILD_TYPE=Generic` build type:

guix-static.scm

@@ -0,0 +1,97 @@
+;; To use this file to build a static version of wfmash using git HEAD:
+;;
+;;   guix build -f guix.scm
+;;
+;; To get a development container using a recent guix (see `guix pull`)
+;;
+;;   guix shell -C -D -f guix.scm
+;;
+;; and inside the container
+;;
+;;   mkdir build
+;;   cd build
+;;   cmake ..
+;;   make
+;;
+;; For the tests you may need /usr/bin/env. Inside the container:
+;;
+;;   mkdir -p /usr/bin ; ln -s $GUIX_ENVIRONMENT/bin/env /usr/bin/env
+;;
+;; by Pjotr Prins (c) 2023
+
+(use-modules
+ ((guix licenses) #:prefix license:)
+  (guix gexp)
+  (guix packages)
+  (guix git-download)
+  (guix build-system cmake)
+  ; (guix gexp)
+  (guix utils)
+  (gnu packages algebra)
+  (gnu packages base)
+  (gnu packages bioinformatics)
+  (gnu packages build-tools)
+  (gnu packages compression)
+  ; (gnu packages curl)
+  (gnu packages gcc)
+  (gnu packages jemalloc)
+  (gnu packages llvm)
+  (gnu packages maths)
+  (gnu packages multiprecision)
+  (gnu packages pkg-config)
+  (gnu packages python)
+  (gnu packages version-control)
+  (srfi srfi-1)
+  (ice-9 popen)
+  (ice-9 rdelim))
+
+;; A minimal version of htslib that does not depend on curl and openssl. This
+;; reduces the number of higher order dependencies in static linking.
+(define-public htslib-static
+  (package
+    (inherit htslib)
+    (name "htslib-static")
+    (arguments
+     (substitute-keyword-arguments (package-arguments htslib)
+       ((#:configure-flags flags ''())
+        ''())))
+    (inputs
+     (list bzip2 xz))))
+
+(define %source-dir (dirname (current-filename)))
+
+(define %git-commit
+    (read-string (open-pipe "git show HEAD | head -1 | cut -d ' ' -f 2" OPEN_READ)))
+
+(define-public wfmash-git
+  (package
+    (name "wfmash-git")
+    (version (git-version "0.10.7" "HEAD" %git-commit))
+    (source (local-file %source-dir #:recursive? #t))
+    (build-system cmake-build-system)
+    (inputs
+     `(
+       ;; ("clang" ,clang)      ; add this to test clang builds
+       ;; ("lld" ,lld)          ; add this to test clang builds
+       ("gcc" ,gcc-12)
+       ("gsl-static" ,gsl-static)
+       ("gmp" ,gmp)
+       ("make" ,gnu-make)
+       ("pkg-config" ,pkg-config)
+       ("jemalloc" ,jemalloc)
+       ("htslib" ,htslib-static)
+       ("git" ,git)
+       ; ("bc" ,bc)               ; for tests
+       ("coreutils" ,coreutils) ; for echo and env in tests
+       ; ("curl" ,curl)
+       ; ("parallel" ,parallel) ; for wfmash-parallel
+       ("bzip2-static" ,bzip2 "static")    ; libz2 part of htslib for static builds
+       ("xz-static" ,xz "static")     ; for static builds
+       ("zlib-static" ,zlib "static")))
+     (synopsis "wfmash")
+     (description
+      "wfmash.")
+     (home-page "https://github.com/waveygang/wfmash")
+     (license license:expat)))
+
+wfmash-git

guix.scm

@@ -1,6 +1,7 @@
-;; To use this file to build a static version of wfmash using git HEAD:
+;; To use this file to build a version of wfmash using git HEAD:
 ;;
-;;   guix build -f guix.scm
+;;  rm -rf build/*
+;;  guix build -f guix.scm
 ;;
 ;; To get a development container using a recent guix (see `guix pull`)
 ;;
@@ -17,33 +18,33 @@
 ;;
 ;;   mkdir -p /usr/bin ; ln -s $GUIX_ENVIRONMENT/bin/env /usr/bin/env
 ;;
-;; by Pjotr Prins & Andrea Guarracino (c) 2023
+;; by Pjotr Prins (c) 2023
 
 (use-modules
  ((guix licenses) #:prefix license:)
-  (guix gexp)
-  (guix packages)
-  (guix git-download)
-  (guix build-system cmake)
-; (guix gexp)
-  (guix utils)
-  (gnu packages algebra)
-  (gnu packages base)
-  (gnu packages bioinformatics)
-  (gnu packages build-tools)
-  (gnu packages compression)
-; (gnu packages curl)
-  (gnu packages gcc)
-  (gnu packages jemalloc)
-  (gnu packages llvm)
-  (gnu packages maths)
-  (gnu packages multiprecision)
-  (gnu packages pkg-config)
-  (gnu packages python)
-  (gnu packages version-control)
-  (srfi srfi-1)
-  (ice-9 popen)
-  (ice-9 rdelim))
+ (guix gexp)
+ (guix packages)
+ (guix git-download)
+ (guix build-system cmake)
+                                        ; (guix gexp)
+ (guix utils)
+ (gnu packages algebra)
+ (gnu packages base)
+ (gnu packages bioinformatics)
+ (gnu packages build-tools)
+ (gnu packages compression)
+                                        ; (gnu packages curl)
+ (gnu packages gcc)
+ (gnu packages jemalloc)
+ (gnu packages llvm)
+ (gnu packages maths)
+ (gnu packages multiprecision)
+ (gnu packages pkg-config)
+ (gnu packages python)
+ (gnu packages version-control)
+ (srfi srfi-1)
+ (ice-9 popen)
+ (ice-9 rdelim))
 
 (define %source-dir (dirname (current-filename)))
 
@@ -56,8 +57,6 @@
     (version (git-version "0.10.7" "HEAD" %git-commit))
     (source (local-file %source-dir #:recursive? #t))
     (build-system cmake-build-system)
-    (arguments
-     `(#:tests? #f)) ;; Disable the test phase
     (inputs
      `(
        ;; ("clang" ,clang)      ; add this to test clang builds
@@ -74,13 +73,13 @@
        ("coreutils" ,coreutils) ; for echo and env in tests
        ; ("curl" ,curl)
        ; ("parallel" ,parallel) ; for wfmash-parallel
-       ("bzip2" ,bzip2)    ; libz2 part of htslib
-       ("xz" ,xz)     ;
+       ("bzip2" ,bzip2)
+       ("xz" ,xz)
        ("zlib" ,zlib)))
-    (synopsis "wfmash")
-    (description
-     "wfmash.")
-    (home-page "https://github.com/waveygang/wfmash")
-    (license license:expat)))
+     (synopsis "wfmash")
+     (description
+      "wfmash.")
+     (home-page "https://github.com/waveygang/wfmash")
+     (license license:expat)))
 
-wfmash-git
+wfmash-git

src/common/wflign/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.2 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
 project(wflign VERSION 0.0.1)
 
 include(GNUInstallDirs)
@@ -35,7 +35,7 @@ else()
     set (CMAKE_CXX_FLAGS "${OpenMP_CXX_FLAGS} ${PIC_FLAG} ${EXTRA_FLAGS}")
 endif ()
 
-if(WFA_PNG_TSV_TIMING)
+if(WFA_PNG_AND_TSV)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DWFA_PNG_TSV_TIMING")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DWFA_PNG_TSV_TIMING")
 endif ()