main.yml

name: Flow-IPC pipeline

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
  # To run a workflow manually, the workflow must be configured to run on the workflow_dispatch event.
  workflow_dispatch:
  
jobs:
  doc:
    strategy:
      fail-fast: false
      matrix:
        # Just pick one that's pre-installed and reasonably new.
        compiler:
          - id: clang-15
            name: clang
            version: 15
            c-path: /usr/bin/clang-15
            cpp-path: /usr/bin/clang++-15
        build-type:
          - id: release
            conan-build-type: Release
            conan-preset: release

    runs-on: ubuntu-latest
        
    name: |
      doc-${{ matrix.compiler.id }}-${{ matrix.build-type.id }}
      
    steps: 
    - name: Update available software list for apt-get
      run: sudo apt-get update
      
    - name: Checkout `ipc` repository and submodules (`flow`, `ipc_*`)
      uses: actions/checkout@v4
      with:
        submodules: true

    - name: Install Flow-IPC dependencies (inc. Graphviz) with apt-get
      run: |
        sudo apt-get install -y graphviz

    - name: Install the latest version of Conan which is less than 2
      run: |
        pip install "conan<2"
    
    - name: Create Conan profile
      run: |
        cat <<EOF > conan_profile   
        [settings]
        compiler = ${{ matrix.compiler.name }}
        compiler.version = ${{ matrix.compiler.version }}
        compiler.cppstd = 17
        compiler.libcxx = libstdc++11
        arch = x86_64
        os = Linux
        build_type = ${{ matrix.build-type.conan-build-type }}
          
        [conf]
        tools.build:compiler_executables = {"c": "${{ matrix.compiler.c-path }}", "cpp": "${{ matrix.compiler.cpp-path }}"}
  
        [buildenv]
        CC = ${{ matrix.compiler.c-path }}
        CXX = ${{ matrix.compiler.cpp-path }}

        [options]
        ipc:build = False
        ipc:doc = True
        EOF
    
    - name: Install Flow-IPC dependencies (inc. Doxygen) with Conan using the profile
      run: |
        conan install \
          . \
          --profile:build conan_profile \
          --profile:host conan_profile \
          --build missing

    - name: Prepare Makefile using CMake
      run: |
        cmake \
          --preset ${{ matrix.build-type.conan-preset }} \
          -DCFG_ENABLE_DOC_GEN=ON \
          -DCFG_SKIP_CODE_GEN=ON

    - name: Get the number of processor cores for parallelized work
      uses: SimenB/github-actions-cpu-cores@v1
      id: cpu-cores

    # Note that `flow_doc_public` and `flow_doc_full` targets are also available; and a user locally
    # may well want to use them.  We, however, have chosen to dedicate a Flow-only workflow
    # (in `flow` repo) to Flow; it will generate the Flow docs.  Meanwhile ipc_doc_* targets create
    # monolithic documentation from the ipc_* submodules and `ipc` itself -- but *not* from
    # `flow` submodule.
    - name: Doxygen-generate documentation sets using Makefile
      run: |
        VERBOSE=1 make \
          ipc_doc_public \
          ipc_doc_full \
          --directory $GITHUB_WORKSPACE/build/${{ matrix.build-type.conan-build-type }} \
          -j${{ steps.cpu-cores.outputs.count }}

    - name: Package doc tarball [Doxygen documentation sets (full, API-only), landing page]
      run: |
        cd $GITHUB_WORKSPACE/doc/ipc_doc
        $GITHUB_WORKSPACE/tools/doc/stage_generated_docs.sh $GITHUB_WORKSPACE/build/${{ matrix.build-type.conan-build-type }}
        rm -rf generated # Save runner space.

    - name: Upload documentation tarball
      uses: actions/upload-artifact@v3
      with:
        name: ipc-doc-${{ matrix.compiler.id }}-${{ matrix.build-type.id }}
        path: |
          ${{ github.workspace }}/doc/ipc_doc.tgz

  build:
    strategy:
      fail-fast: false
      matrix:
        compiler:
          - id: gcc-9
            name: gcc
            version: 9
            c-path: /usr/bin/gcc-9
            cpp-path: /usr/bin/g++-9  
          - id: gcc-10
            name: gcc
            version: 10
            c-path: /usr/bin/gcc-10
            cpp-path: /usr/bin/g++-10
          - id: gcc-11
            name: gcc
            version: 11
            c-path: /usr/bin/gcc-11
            cpp-path: /usr/bin/g++-11
          - id: gcc-13
            name: gcc
            version: 13
            c-path: /usr/bin/gcc-13
            cpp-path: /usr/bin/g++-13        
          - id: clang-13
            name: clang
            version: 13
            c-path: /usr/bin/clang-13
            cpp-path: /usr/bin/clang++-13  
          - id: clang-15
            name: clang
            version: 15
            c-path: /usr/bin/clang-15
            cpp-path: /usr/bin/clang++-15     
          - id: clang-16
            name: clang
            version: 16 
            c-path: /usr/bin/clang-16 
            cpp-path: /usr/bin/clang++-16
            install: True      
          - id: clang-17 
            name: clang
            version: 17
            c-path: /usr/bin/clang-17
            cpp-path: /usr/bin/clang++-17
            install: True     
        build-type:
          - id: debug
            conan-build-type: Debug
            conan-jemalloc-build-type: Debug
            conan-preset: debug
          - id: release
            conan-build-type: Release
            conan-jemalloc-build-type: Release
            conan-preset: release    
          - id: relwithdebinfo-asan
            conan-build-type: RelWithDebInfo
            conan-jemalloc-build-type: Release
            conan-preset: relwithdebinfo
            conan-custom-settings: | # Could we not copy/paste these 4x?
              data['compiler']['gcc']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined']
              data['compiler']['clang']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined']
            conan-profile-custom-conf: |
              # no-omit-frame-pointer recommended in (A|UB|M)SAN docs for nice stack traces.
              tools.build:cflags = ["-fsanitize=address", "-fno-omit-frame-pointer"]
              tools.build:cxxflags = ["-fsanitize=address", "-fno-omit-frame-pointer"]
              tools.build:sharedlinkflags = ["-fsanitize=address"]
              tools.build:exelinkflags = ["-fsanitize=address"]
            conan-profile-custom-settings: |
              compiler.sanitizer = address
            conan-profile-custom-buildenv: |
              CXXFLAGS = -fsanitize=address -fno-omit-frame-pointer
              CFLAGS = -fsanitize=address -fno-omit-frame-pointer
              LDFLAGS = -fsanitize=address
            cmake-flags: | # At least ASAN with clang + LTO => cryptic link error.  No need for LTO when *SAN.
              -DCFG_NO_LTO=ON
          - id: relwithdebinfo-ubsan
            conan-build-type: RelWithDebInfo
            conan-jemalloc-build-type: Release
            conan-preset: relwithdebinfo
            conan-custom-settings: |
              data['compiler']['gcc']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined']
              data['compiler']['clang']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined']
            conan-profile-custom-conf: |
              tools.build:cflags = ["-fsanitize=undefined", "-fno-omit-frame-pointer"]
              tools.build:cxxflags = ["-fsanitize=undefined", "-fno-omit-frame-pointer"]
              tools.build:sharedlinkflags = ["-fsanitize=undefined"]
              tools.build:exelinkflags = ["-fsanitize=undefined"]
            conan-profile-custom-settings: |
              compiler.sanitizer = undefined
            conan-profile-custom-buildenv: |
              CXXFLAGS = -fsanitize=undefined -fno-omit-frame-pointer
              CFLAGS = -fsanitize=undefined -fno-omit-frame-pointer
              LDFLAGS = -fsanitize=undefined
            cmake-flags: | # While UBSAN might work with LTO, I do not want the aggravation/entropy.  Turn it off.
              -DCFG_NO_LTO=ON
          - id: relwithdebinfo-msan
            conan-build-type: RelWithDebInfo
            conan-jemalloc-build-type: Release
            conan-preset: relwithdebinfo
            conan-custom-settings: |
              data['compiler']['gcc']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined']
              data['compiler']['clang']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined']
            conan-profile-custom-conf: |
              # See comment near step below where this ignorelist file is prepped.
              tools.build:cflags = ["-fsanitize=memory", "-fno-omit-frame-pointer", "-fsanitize-ignorelist=/tmp/msan_ignore_list.cfg"]
              tools.build:cxxflags = ["-fsanitize=memory", "-fno-omit-frame-pointer", "-fsanitize-ignorelist=/tmp/msan_ignore_list.cfg"]
              tools.build:sharedlinkflags = ["-fsanitize=memory"]
              tools.build:exelinkflags = ["-fsanitize=memory"]
            conan-profile-custom-settings: |
              compiler.sanitizer = memory
            conan-profile-custom-buildenv: |
              LDFLAGS = -fsanitize=memory
              CXXFLAGS = -fsanitize=memory -fno-omit-frame-pointer -fsanitize-ignorelist=/tmp/msan_ignore_list.cfg
              CFLAGS = -fsanitize=memory -fno-omit-frame-pointer -fsanitize-ignorelist=/tmp/msan_ignore_list.cfg
            cmake-flags: | # While MSAN might work with LTO, I do not want the aggravation/entropy.  Turn it off.
              -DCFG_NO_LTO=ON
          - id: relwithdebinfo-tsan
            conan-build-type: RelWithDebInfo
            conan-jemalloc-build-type: Release
            conan-preset: relwithdebinfo
            conan-custom-settings: |
              data['compiler']['gcc']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined']
              data['compiler']['clang']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined']
            conan-profile-custom-conf: |
              tools.build:cflags = ["-fsanitize=thread"]
              tools.build:cxxflags = ["-fsanitize=thread"]
              tools.build:sharedlinkflags = ["-fsanitize=thread"]
              tools.build:exelinkflags = ["-fsanitize=thread"]
            conan-profile-custom-settings: |
              compiler.sanitizer = thread   
            conan-profile-custom-buildenv: |
              LDFLAGS = -fsanitize=thread
              CXXFLAGS = -fsanitize=thread
              CFLAGS = -fsanitize=thread
            gtest-unit-tests-params: | # As of this writing these trigger assert+bugs in TSAN runtime.  TODO: Revisit.
              --gtest_filter=-\
                Shm_session_test.In_process_array:\
                Shm_session_test.In_process_vector_offset_ptr:\
                Shm_session_test.In_process_string_offset_ptr:\
                Shm_session_test.In_process_list_offset_ptr:\
                Shm_session_test.Multisession_in_process:\
                Shm_session_test.Allocation_performance_five:\
                Jemalloc_shm_pool_collection_test.Interface:\
                Jemalloc_shm_pool_collection_test.Multiprocess:\
                Jemalloc_shm_pool_collection_test.Owner_shm_pool_removal:\
                Jemalloc_shm_pool_collection_test.Multithread_load:\
                Borrower_shm_pool_collection_test.Interface:\
                Borrower_shm_pool_collection_test.Multiprocess:\
                Shm_pool_collection_test.Interface:\
                Shm_pool_collection_test.Multiprocess:\
                Shm_session_data_test.Multithread_object_database
            cmake-flags: # See the other *SAN; but for now I heed jkontrik's words to keep consistent with non-*SAN.
          - id: relwithdebinfo
            conan-build-type: RelWithDebInfo
            conan-jemalloc-build-type: Release
            conan-preset: relwithdebinfo      
          - id: minsizerel
            conan-build-type: MinSizeRel
            conan-jemalloc-build-type: Release
            conan-preset: minsizerel

        # We concentrate on clang sanitizers; they are newer/nicer; also MSAN is clang-only.  So gcc ones excluded.
        # TODO: Consider reducing to the newest clang.  Generally they just keep improving; the chances of something
        # being uncaught (incorrectly) with a later version are slim.  Not impossible though.  Look into it.
        exclude:
          - compiler:  { id: gcc-9 }
            build-type: { id: relwithdebinfo-asan }
          - compiler:  { id: gcc-10 }
            build-type: { id: relwithdebinfo-asan }
          - compiler:  { id: gcc-11 }
            build-type: { id: relwithdebinfo-asan }
          - compiler:  { id: gcc-13 }
            build-type: { id: relwithdebinfo-asan }
          - compiler:  { id: gcc-9 }
            build-type: { id: relwithdebinfo-ubsan }
          - compiler:  { id: gcc-10 }
            build-type: { id: relwithdebinfo-ubsan }
          - compiler:  { id: gcc-11 }
            build-type: { id: relwithdebinfo-ubsan }
          - compiler:  { id: gcc-13 }
            build-type: { id: relwithdebinfo-ubsan }
          - compiler:  { id: gcc-9 }
            build-type: { id: relwithdebinfo-msan }
          - compiler:  { id: gcc-10 }
            build-type: { id: relwithdebinfo-msan }
          - compiler:  { id: gcc-11 }
            build-type: { id: relwithdebinfo-msan }
          - compiler:  { id: gcc-13 }
            build-type: { id: relwithdebinfo-msan }
          - compiler:  { id: gcc-9 }
            build-type: { id: relwithdebinfo-tsan }
          - compiler:  { id: gcc-10 }
            build-type: { id: relwithdebinfo-tsan }
          - compiler:  { id: gcc-11 }
            build-type: { id: relwithdebinfo-tsan }
          - compiler:  { id: gcc-13 }
            build-type: { id: relwithdebinfo-tsan }

    runs-on: ubuntu-latest
        
    name: |
      build-${{ matrix.compiler.id }}-${{ matrix.build-type.id }}

    env:
      # Run-time controls for various sanitizers (ignored when test/demo is not build with
      # a *SAN build-type).  Some notes:
      #
      # Unclear if we need disable_coredump=0; it might be a gcc thing due to historic
      # issues with core size with ASAN; it is not documented in the clang *SAN docs for any
      # sanitizers; but it is accepted for ASAN and TSAN, seemingly, and maybe is at worst
      # harmless with out compiler versions.  TODO: Maybe revisit.
      #
      # print_stacktrace=1 for UBSAN is recommended by documentation for nice stack traces
      # (also that is why we apply sanitizers to RelWithDebInfo; DebInfo part for the nice
      # stack traces/etc.; Rel part to reduce the considerable slowdown from some of the
      # sanitizers).
      #
      # second_deadlock_stack=1 for TSAN: TODO: Explain.  Don't see it in clang TSAN docs.
      before_each_test: |
        export ASAN_OPTIONS="disable_coredump=0"
        export UBSAN_OPTIONS="disable_coredump=0 print_stacktrace=1 suppressions=${{ github.workspace }}/ubsan_suppressions_${{ matrix.compiler.name }}_${{ matrix.compiler.version }}.cfg"
        export TSAN_OPTIONS="disable_coredump=0 second_deadlock_stack=1 suppressions=${{ github.workspace }}/tsan_suppressions_${{ matrix.compiler.name }}_${{ matrix.compiler.version }}.cfg"
        # XXX
        echo "$UBSAN_OPTIONS"
        cat ${{ github.workspace }}/ubsan_suppressions_${{ matrix.compiler.name }}_${{ matrix.compiler.version }}.cfg
        echo "$TSAN_OPTIONS"
        cat ${{ github.workspace }}/tsan_suppressions_${{ matrix.compiler.name }}_${{ matrix.compiler.version }}.cfg
        echo "--"
    
    steps: 
    - name: Update available software list for apt-get
      run: sudo apt-get update
      
    - name: Checkout `ipc` repository and submodules (`flow`, `ipc_*`)
      uses: actions/checkout@v4
      with:
        submodules: true

    # Many compilers are pre-installed; if not then fetch it.
    - name: Install clang compiler
      if: |
        matrix.compiler.install && (matrix.compiler.name == 'clang')
      run: |
        wget https://apt.llvm.org/llvm.sh
        chmod u+x llvm.sh
        sudo ./llvm.sh ${{ matrix.compiler.version }}

    - name: Install gcc compiler
      if: |
        matrix.compiler.install && (matrix.compiler.name == 'gcc')
      run: |
        sudo apt-get install -y software-properties-common
        sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y
        sudo apt-get update
        sudo apt-get install -y gcc-${{ matrix.compiler.version }} g++-${{ matrix.compiler.version }}

    - name: Install the latest version of Conan which is less than 2
      run: |
        pip install "conan<2"

    - name: Add custom settings for Conan packages
      if: |
        matrix.build-type.conan-custom-settings
      run: |
        conan config init
        
        pip install PyYAML

        CONAN_SETTINGS_PATH=$(conan config home)/settings.yml

        python -c "
        import yaml

        with open('$CONAN_SETTINGS_PATH', 'r') as file:
          data = yaml.safe_load(file)
        
        ${{ matrix.build-type.conan-custom-settings }}
      
        with open('$CONAN_SETTINGS_PATH', 'w') as file:
          yaml.dump(data, file)
        "

    - name: Create Conan profile
      run: |
        cat <<EOF > conan_profile   
        [settings]
        compiler = ${{ matrix.compiler.name }}
        compiler.version = ${{ matrix.compiler.version }}
        compiler.cppstd = 17
        compiler.libcxx = libstdc++11
        arch = x86_64
        os = Linux
        build_type = ${{ matrix.build-type.conan-build-type }}
        jemalloc:build_type = ${{ matrix.build-type.conan-jemalloc-build-type }}
        ${{ matrix.build-type.conan-profile-custom-settings }}
          
        [conf]
        tools.build:compiler_executables = {"c": "${{ matrix.compiler.c-path }}", "cpp": "${{ matrix.compiler.cpp-path }}"}
        tools.env.virtualenv:auto_use = True
        ${{ matrix.build-type.conan-profile-custom-conf }}
  
        [buildenv]
        CC = ${{ matrix.compiler.c-path }}
        CXX = ${{ matrix.compiler.cpp-path }}
        ${{ matrix.build-type.conan-profile-custom-buildenv }}

        [options]
        jemalloc:enable_cxx = False 
        jemalloc:prefix = je_
        ipc:build = True
        ipc:doc = False
        flow:build = True
        flow:doc = False
        EOF

    # We need to prepare a sanitizer ignore-list in MSAN mode.  Background for this is subtle and annoying:
    # As it stands, whatever matrix compiler/build-type is chosen applies not just to our code (correct)
    # and 3rd party libraries we link like lib{boost_*|capnp|kj|jemalloc} (semi-optional but good) but also
    # unfortunately any items built from source during "Install Flow-IPC dependencies" step that we then
    # use during during the build step for our own code subsequently.  At a minimum this will slow down
    # such programs.  (For the time being we accept this as not-so-bad; to target this config at some things
    # but not others is hard/a ticket.)  In particular, though, the capnp compiler binary is built this way;
    # and our "Build targets" step uses it to build key things (namely convert .capnp schemas into .c++
    # and .h sources which are then themselves compiled/used in compilation).  In the case of MSAN, this
    # version of capnp compiler happens to trigger several MSAN failures (presumably they are not a true
    # problem, and it's not our job really to sanitize capnp -- though we can file tickets for them and/or
    # issue PRs; but I digress).  So in MSAN mode our build step fails, when capnp compiler itself aborts
    # with MSAN failures.  One approach is to not apply MSAN to capnp compiler (no-go for now; it's a ticket
    # as mentioned); another is to uses Conan to patch capnp package for them (not a bad idea; ticket filed);
    # and lastly we can put the specific failures on the ignore-list for MSAN.  For now we do that; while
    # unpleasant it does get the job done.  TODO: Revisit/resolve tickets/improve (see above).
    - name: Prepare MSAN sanitizer compile-time config file(s)
      if: |
        (!cancelled()) && (matrix.build-type.id == 'relwithdebinfo-msan')
      run: |
        # TODO: Ideally would go, like other things, under github.workspace somewhere (maybe build/),
        # but that var isn't available in `strategy` up above, where we specify the compiler option
        # pointing to this file.  Surely we could cook something up; but meantime /tmp should work;
        # I think jobs don't execute concurrently in one file-system, and it should be safe enough in /tmp.
        # Though /tmp can get cleared spuriously or something....
        cat <<'EOF' > /tmp/msan_ignore_list.cfg
        [memory]
        # Warning: In clang-18 there are breaking changes in how globs/regexes are interpreted.  See docs.
        # Currently assuming clang-17 or lower.
        #
        # capnp compiler MSAN failures suppressed:
        src:*/kj/filesystem-disk-unix.*
        src:*/bits/stl_tree.h
        EOF
        # Append to that the suppressed items from the source tree.  These apply to the real code being
        # built below, so just stylistically it is nicer to keep them there along with other
        # sanitizers' suppressions.  (Note, though, that MSAN does not have a run-time suppression
        # system; only these ignore-lists.  The others do also have ignore-lists though.
        # The format is totally different between the 2 types of suppression.)
        cat $GITHUB_WORKSPACE/msan_ignore_list_${{ matrix.compiler.name }}.cfg >> /tmp/msan_ignore_list.cfg
        echo "The combined MSAN ignore-list config file follows:"
        cat /tmp/msan_ignore_list.cfg

    - name: Install Flow-IPC dependencies with Conan using the profile
      run: |
        ${{ env.before_each_test }}
        # XXX
        echo "$UBSAN_OPTIONS"
        cat ${{ github.workspace }}/ubsan_suppressions_${{ matrix.compiler.name }}_${{ matrix.compiler.version }}.cfg
        echo "$TSAN_OPTIONS"
        cat ${{ github.workspace }}/tsan_suppressions_${{ matrix.compiler.name }}_${{ matrix.compiler.version }}.cfg
        echo "--"
        conan editable add flow flow/1.0
        conan install \
          . \
          --profile:build conan_profile \
          --profile:host conan_profile \
          --build missing

    - name: Get the number of processor cores for parallelized work
      uses: SimenB/github-actions-cpu-cores@v1
      id: cpu-cores

    # TODO: Ideally let our CMake interrogate jemalloc-config to find jemalloc-prefix itself.
    # Less stuff for us to worry about that way here.  If it has not been tried, try it.
    # If it does not work, apply a small effort to see if it can be easily made to work.
    - name: Prepare Makefile using CMake
      run: |
        cmake \
          --preset ${{ matrix.build-type.conan-preset }} \
          -DCFG_ENABLE_TEST_SUITE=ON \
          -DJEMALLOC_PREFIX=je_ \
          -DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }} \
          ${{ matrix.build-type.cmake-flags }}

    - name: Build targets (libraries, demos/tests) with Makefile
      run: |
        VERBOSE=1 make \
          --keep-going \
          --directory $GITHUB_WORKSPACE/build/${{ matrix.build-type.conan-build-type }} \
          -j${{ steps.cpu-cores.outputs.count }}

    - name: Install targets with Makefile
      run: |
        VERBOSE=1 make install \
          --directory $GITHUB_WORKSPACE/build/${{ matrix.build-type.conan-build-type }} \
          -j${{ steps.cpu-cores.outputs.count }}
        # Save runner space: blow away build dir after install.
        rm -rf $GITHUB_WORKSPACE/build/${{ matrix.build-type.conan-build-type }}

    # From now on use !cancelled() to try to run any test/demo that exists regardless
    # of preceding failures if any.  Same-ish (always()) with the log-upload at the end.
    # Worst-case they'll all fail in super-basic ways and immediately; no extra harm done.

    # From now on save logs in install-dir/bin/logs.  They will be tarred up and uploaded
    # as artifacts at the end.  For those tests below that produce separate logs as files
    # to begin-with, this is a no-brainer.  For those (as of this writing the main one
    # is unit_test; but also the link tests produce short logs too) who use stdout/stderr
    # the reason to do this is 2-fold.
    #   - UBSAN in particular produces non-fatal error output about problems it detects.
    #     We need to analyze them after the fact and fail a step below in that case to
    #     alert developers who would then fix such new problems.
    #     - One can also build UBSAN-mode things with -fno-sanitize-recover which would
    #       cause abnormal program exit on *first* error.  We do not do this for these
    #       reasons:
    #       - It is more convenient to let it continue and thus show all problems in one
    #         shot.
    #       - There is a side problem: At this time compile settings, including
    #         in UBSAN case `-fsanitize=undefined -fno-sanitize-recover`, are applied not
    #         just to our code or 3rd party libraries but also any other stuff built
    #         (due to setting C[XX]FLAGS in Conan profile).  Targeting just the exact
    #         stuff we want with those is hard and a separate project/ticket.
    #         In the meantime -fno-sanitize-recover causes completely unrealted program
    #         halts during the very-early step, when building dependencies including
    #         capnp; some autotools configure.sh fails crazily, and nothing can work
    #         from that point on due to dependencies-install step failing.  So at this
    #         time -fno-sanitize-recover is a no-go; it affects too much unrelated stuff.
    #   - It is more consistent/convenient to get all the bulky logs in one place as an
    #     artifact, rather than some in the pipeline output, others in the artifacts.
    #     (This preference is subjective, yes.)

    - name: Run link test [`ipc_core` - Flow-IPC Core]
      if: |
        !cancelled()
      run: |
        ${{ env.before_each_test }}
        # XXX
        echo "$UBSAN_OPTIONS"
        cat ${{ github.workspace }}/ubsan_suppressions_${{ matrix.compiler.name }}_${{ matrix.compiler.version }}.cfg
        echo "$TSAN_OPTIONS"
        cat ${{ github.workspace }}/tsan_suppressions_${{ matrix.compiler.name }}_${{ matrix.compiler.version }}.cfg
        echo "--"
        cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin
        mkdir -p logs/ipc_core_link_test
        ./ipc_core_link_test.exec > logs/ipc_core_link_test/console.log 2>&1

    - name: Run link test [`ipc_transport_structured` - Flow-IPC Structured Transport]
      if: |
        !cancelled()
      run: |
        ${{ env.before_each_test }}
        cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin
        mkdir -p logs/ipc_transport_structured_link_test
        ./ipc_transport_structured_link_test.exec > logs/ipc_transport_structured_link_test/console.log 2>&1

    # Server will exit automatically (same below).
    - name: Run link test [`ipc_session` - Flow-IPC Sessions]
      if: |
        !cancelled()
      run: |
        ${{ env.before_each_test }}
        cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin
        mkdir -p logs/ipc_session_link_test
        ./ipc_session_link_test_srv.exec > logs/ipc_session_link_test/srv.console.log 2>&1 &
        sleep 1
        ./ipc_session_link_test_cli.exec > logs/ipc_session_link_test/cli.console.log 2>&1

    - name: Run link test [`ipc_shm` - Flow-IPC Shared Memory]
      if: |
        !cancelled()
      run: |
        ${{ env.before_each_test }}
        cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin
        mkdir -p logs/ipc_shm_link_test
        ./ipc_shm_link_test_srv.exec > logs/ipc_shm_link_test/srv.console.log 2>&1 &
        sleep 1
        ./ipc_shm_link_test_cli.exec > logs/ipc_shm_link_test/cli.console.log 2>&1
        
    - name: Run link test [`ipc_shm_arena_lend` - Flow-IPC SHM-jemalloc]
      if: |
        !cancelled()
      run: |
        ${{ env.before_each_test }}
        cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin
        mkdir -p logs/ipc_shm_arena_lend_link_test
        ./ipc_shm_arena_lend_link_test_srv.exec > logs/ipc_shm_arena_lend_link_test/srv.console.log 2>&1 &
        sleep 1
        ./ipc_shm_arena_lend_link_test_cli.exec > logs/ipc_shm_arena_lend_link_test/cli.console.log 2>&1

    # As of this writing this runs with high verbosity, so we do not re-run on failure
    # like the ones below.
    - name: Run unit tests
      if: |
        !cancelled()
      run: |
        ${{ env.before_each_test }}
        echo $TSAN_OPTIONS
        cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin
        mkdir -p logs/libipc_unit_test
        # Some newline issues with the possible additional args; so need to make a wrapper script
        # and then redirect, as desired, its output.
        cat <<'EOF' > $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_unit_test.sh
        ./libipc_unit_test.exec ${{ matrix.build-type.gtest-unit-tests-params }}
        EOF
        /usr/bin/bash -e $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_unit_test.sh \
          > logs/libipc_unit_test/console.log 2>&1

    # For tests below where on failure it can be very helpful to look at higher-verbosity logs,
    # any failing step is repeated with logging hiked up.  Note that formally (and practically)
    # higher-than-INFO verbosity is allowed to affect performance and thus is *not* right for
    # the main integration test run.  However on failure all bets are off, and we just want the
    # info we can get.  (Also higher-verbosity runs can take significantly longer; no one wants that.)

    # This follows the instructions in bin/transport_test/README.txt.
    - name: Prepare run script for [transport_test - Scripted mode] variations below
      if: |
        (!cancelled()) && (matrix.build-type.id != 'relwithdebinfo-tsan')
      run: |
        cat <<'EOF' > $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_sc.sh
        # Script created by pipeline during job.
        echo "Log level: [$1]."
        cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/transport_test
        OUT_DIR_NAME=log_level_$1
        OUT_DIR=../logs/transport_test/scripted/$OUT_DIR_NAME
        mkdir -p $OUT_DIR
        ./transport_test.exec scripted $OUT_DIR/srv.log info $1 \
          < srv-script.txt > $OUT_DIR/srv.console.log 2>&1 &
        SRV_PID=$!
        sleep 1
        ./transport_test.exec scripted $OUT_DIR/cli.log info $1 \
          < cli-script.txt > $OUT_DIR/cli.console.log 2>&1 &
        CLI_PID=$!
        if wait $SRV_PID; then SRV_EC=0; else SRV_EC=$?; fi
        echo "Server finished with code [$SRV_EC]."
        if wait $CLI_PID; then CLI_EC=0; else CLI_EC=$?; fi
        echo "Client finished with code [$CLI_EC]."
        [ $SRV_EC -eq 0 ] && [ $CLI_EC -eq 0 ]
        EOF

    - name: Run integration test [transport_test - Scripted mode]
      id: transport_test_scripted
      if: |
        (!cancelled()) && (matrix.build-type.id != 'relwithdebinfo-tsan')
      run: |
        ${{ env.before_each_test }}
        /usr/bin/bash -e \
          $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_sc.sh \
            info

    - name: Re-run with increased logging, on failure only
      if: |
        (!cancelled()) && (steps.transport_test_scripted.outcome == 'failure')
      run: |
        ${{ env.before_each_test }}
        /usr/bin/bash -e \
          $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_sc.sh \
            data

    # The following [Exercise mode] tests follow the instructions in bin/transport_test/README.txt.
    # Note that the creation of ~/bin/ex_..._run and placement of executables there, plus
    # /tmp/var/run for run-time files (PID files and similar), is a necessary consequence of
    # the ipc::session safety model for estabshing IPC conversations (sessions).

    - name: Prepare IPC-session safety-friendly run-time environment for [transport_test - Exercise mode]
      if: |
        (!cancelled()) && (matrix.build-type.id != 'relwithdebinfo-tsan')
      run: |
        mkdir -p ~/bin/ex_srv_run ~/bin/ex_cli_run
        mkdir -p /tmp/var/run
        cp -v $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/transport_test/transport_test.exec \
          ~/bin/ex_srv.exec
        cp -v ~/bin/ex_srv.exec ~/bin/ex_cli.exec

    - name: Prepare run script for [transport_test - Exercise mode] variations below
      if: |
        (!cancelled()) && (matrix.build-type.id != 'relwithdebinfo-tsan')
      run: |
        cat <<'EOF' > $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_ex.sh
        # Script created by pipeline during job.
        echo "Log level: [$1]."
        echo "Exercise sub-mode: [$2]."
        echo "Sub-mode snippet (none or 'shm-?'): [$3]."
        OUT_DIR=$GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/logs/transport_test/exercise/$2
        mkdir -p $OUT_DIR
        cd ~/bin/ex_srv_run
        ~/bin/ex_srv.exec exercise-srv$3 $OUT_DIR/srv.log info $1 \
          > $OUT_DIR/srv.console.log 2>&1 &
        SRV_PID=$!
        sleep 5
        ~/bin/ex_cli.exec exercise-cli$3 $OUT_DIR/cli.log info $1 \
          > $OUT_DIR/cli.console.log 2>&1 &
        CLI_PID=$!
        if wait $SRV_PID; then SRV_EC=0; else SRV_EC=$?; fi
        echo "Server finished with code [$SRV_EC]."
        if wait $CLI_PID; then CLI_EC=0; else CLI_EC=$?; fi
        echo "Client finished with code [$CLI_EC]."
        [ $SRV_EC -eq 0 ] && [ $CLI_EC -eq 0 ]
        EOF

    - name: Run integration test [transport_test - Exercise mode - Heap sub-mode]
      id: transport_test_ex_heap
      if: |
        (!cancelled()) && (matrix.build-type.id != 'relwithdebinfo-tsan')
      run: |
        ${{ env.before_each_test }}
        /usr/bin/bash -e \
          $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_ex.sh \
            info heap

    - name: Re-run with increased logging, on failure only
      if: |
        (!cancelled()) && (steps.transport_test_ex_heap.outcome == 'failure')
      run: |
        ${{ env.before_each_test }}
        /usr/bin/bash -e \
          $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_ex.sh \
            data heap_log_level_data

    - name: Run integration test [transport_test - Exercise mode - SHM-classic sub-mode]
      id: transport_test_ex_shm_c
      if: |
        (!cancelled()) && (matrix.build-type.id != 'relwithdebinfo-tsan')
      run: |
        ${{ env.before_each_test }}
        /usr/bin/bash -e \
          $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_ex.sh \
            info shm_classic -shm-c

    - name: Re-run with increased logging, on failure only
      if: |
        (!cancelled()) && (steps.transport_test_ex_shm_c.outcome == 'failure')
      run: |
        ${{ env.before_each_test }}
        /usr/bin/bash -e \
          $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_ex.sh \
            data shm_classic_log_level_data -shm-c

    - name: Run integration test [transport_test - Exercise mode - SHM-jemalloc sub-mode]
      id: transport_test_ex_shm_j
      if: |
        (!cancelled()) && (matrix.build-type.id != 'relwithdebinfo-tsan')
      run: |
        ${{ env.before_each_test }}
        /usr/bin/bash -e \
          $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_ex.sh \
            info shm_jemalloc -shm-j

    - name: Re-run with increased logging, on failure only
      if: |
        (!cancelled()) && (steps.transport_test_ex_shm_j.outcome == 'failure')
      run: |
        ${{ env.before_each_test }}
        /usr/bin/bash -e \
          $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_ex.sh \
            data shm_jemalloc_log_level_data -shm-j

    # See earlier comment block about why we saved all the logs including for console-output-only tests/demos.
    - name: Check test/demo logs for non-fatal sanitizer error(s)
      if: |
        (!cancelled()) && (matrix.build-type.id == 'relwithdebinfo-ubsan')
      run: |
        cd ${{ github.workspace }}/install/${{ matrix.build-type.conan-build-type }}/bin/logs
        # grep returns 0 if 1+ found, 1 if none found, 2+ on error.  So check results explicitly instead of -e.
        # Namely, for us, the only OK result is an empty stdout and empty stderr.
        # Otherwise either grep failed (unlikely) or found 1+ problems; either way redirection target will
        # be not-empty, and we will force failure.
        grep 'SUMMARY: UndefinedBehaviorSanitizer:' `find . -type f` > san_failure_summaries.txt 2>&1 || true
        if [ -s san_failure_summaries.txt ]; then
          echo "Error(s) found.  Pipeline will fail.  Failures summarized below."
          echo "Please peruse uploaded log artifacts, resolve the issues, and run pipeline again."
          echo "---"
          cat san_failure_summaries.txt
          false
        fi
        echo "No errors found in logs."

    - name: Package test/demo logs tarball
      if: |
        always()
      run: |
        cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin
        tar cvzf logs.tgz logs
        rm -rf logs # Save runner space.

    - name: Upload test/demo logs (please inspect if failure(s) seen above)
      if: |
        always()
      uses: actions/upload-artifact@v3
      with:
        name: ipc-logs-${{ matrix.compiler.id }}-${{ matrix.build-type.id }}
        path: |
          ${{ github.workspace }}/install/${{ matrix.build-type.conan-build-type }}/bin/logs.tgz

# TODO: Look into the topic of debuggability in case of a crash.  Is a core generated?  Is it saved?
# Do we need to manually save it as an artifact?  For that matter we would then need the binary and
# ideally the source.  For now we save any logs that are not printed to console, and where possible
# our tests/demos keep it the console exclusively (but in some cases, such as transport_test, it
# is not practical due to parallel execution and other aspects).  In general it is always better that
# logs are sufficient; but look into situations where they are not.
#
# Don't forget situation where program exits due to a sanitizer reporting problem (ASAN, etc.); is
# there a core?  Do we need one?  Etc.
#
# Possibly this is all handled beautifully automatically; then this should be deleted.