Flow-IPC pipeline #414
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Flow-IPC pipeline | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| # To run a workflow manually, the workflow must be configured to run on the workflow_dispatch event. | |
| workflow_dispatch: | |
| jobs: | |
| doc: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Just pick one that's pre-installed and reasonably new. | |
| compiler: | |
| - id: clang-15 | |
| name: clang | |
| version: 15 | |
| c-path: /usr/bin/clang-15 | |
| cpp-path: /usr/bin/clang++-15 | |
| build-type: | |
| - id: release | |
| conan-build-type: Release | |
| conan-preset: release | |
| runs-on: ubuntu-latest | |
| name: | | |
| doc-${{ matrix.compiler.id }}-${{ matrix.build-type.id }} | |
| steps: | |
| - name: Update available software list for apt-get | |
| run: sudo apt-get update | |
| - name: Checkout `ipc` repository and submodules (`flow`, `ipc_*`) | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: Install Flow-IPC dependencies (inc. Graphviz) with apt-get | |
| run: | | |
| sudo apt-get install -y graphviz | |
| - name: Install the latest version of Conan which is less than 2 | |
| run: | | |
| pip install "conan<2" | |
| - name: Create Conan profile | |
| run: | | |
| cat <<EOF > conan_profile | |
| [settings] | |
| compiler = ${{ matrix.compiler.name }} | |
| compiler.version = ${{ matrix.compiler.version }} | |
| compiler.cppstd = 17 | |
| compiler.libcxx = libstdc++11 | |
| arch = x86_64 | |
| os = Linux | |
| build_type = ${{ matrix.build-type.conan-build-type }} | |
| [conf] | |
| tools.build:compiler_executables = {"c": "${{ matrix.compiler.c-path }}", "cpp": "${{ matrix.compiler.cpp-path }}"} | |
| [buildenv] | |
| CC = ${{ matrix.compiler.c-path }} | |
| CXX = ${{ matrix.compiler.cpp-path }} | |
| [options] | |
| ipc:build = False | |
| ipc:doc = True | |
| EOF | |
| - name: Install Flow-IPC dependencies (inc. Doxygen) with Conan using the profile | |
| run: | | |
| conan install \ | |
| . \ | |
| --profile:build conan_profile \ | |
| --profile:host conan_profile \ | |
| --build missing | |
| - name: Prepare Makefile using CMake | |
| run: | | |
| cmake \ | |
| --preset ${{ matrix.build-type.conan-preset }} \ | |
| -DCFG_ENABLE_DOC_GEN=ON \ | |
| -DCFG_SKIP_CODE_GEN=ON | |
| - name: Get the number of processor cores for parallelized work | |
| uses: SimenB/github-actions-cpu-cores@v1 | |
| id: cpu-cores | |
| # Note that `flow_doc_public` and `flow_doc_full` targets are also available; and a user locally | |
| # may well want to use them. We, however, have chosen to dedicate a Flow-only workflow | |
| # (in `flow` repo) to Flow; it will generate the Flow docs. Meanwhile ipc_doc_* targets create | |
| # monolithic documentation from the ipc_* submodules and `ipc` itself -- but *not* from | |
| # `flow` submodule. | |
| - name: Doxygen-generate documentation sets using Makefile | |
| run: | | |
| VERBOSE=1 make \ | |
| ipc_doc_public \ | |
| ipc_doc_full \ | |
| --directory $GITHUB_WORKSPACE/build/${{ matrix.build-type.conan-build-type }} \ | |
| -j${{ steps.cpu-cores.outputs.count }} | |
| - name: Package doc tarball [Doxygen documentation sets (full, API-only), landing page] | |
| run: | | |
| cd $GITHUB_WORKSPACE/doc/ipc_doc | |
| $GITHUB_WORKSPACE/tools/doc/stage_generated_docs.sh $GITHUB_WORKSPACE/build/${{ matrix.build-type.conan-build-type }} | |
| rm -rf generated # Save runner space. | |
| - name: Upload documentation tarball | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ipc-doc-${{ matrix.compiler.id }}-${{ matrix.build-type.id }} | |
| path: | | |
| ${{ github.workspace }}/doc/ipc_doc.tgz | |
| build: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| compiler: | |
| - id: gcc-9 | |
| name: gcc | |
| version: 9 | |
| c-path: /usr/bin/gcc-9 | |
| cpp-path: /usr/bin/g++-9 | |
| - id: gcc-10 | |
| name: gcc | |
| version: 10 | |
| c-path: /usr/bin/gcc-10 | |
| cpp-path: /usr/bin/g++-10 | |
| - id: gcc-11 | |
| name: gcc | |
| version: 11 | |
| c-path: /usr/bin/gcc-11 | |
| cpp-path: /usr/bin/g++-11 | |
| - id: gcc-13 | |
| name: gcc | |
| version: 13 | |
| c-path: /usr/bin/gcc-13 | |
| cpp-path: /usr/bin/g++-13 | |
| - id: clang-13 | |
| name: clang | |
| version: 13 | |
| c-path: /usr/bin/clang-13 | |
| cpp-path: /usr/bin/clang++-13 | |
| - id: clang-15 | |
| name: clang | |
| version: 15 | |
| c-path: /usr/bin/clang-15 | |
| cpp-path: /usr/bin/clang++-15 | |
| - id: clang-16 | |
| name: clang | |
| version: 16 | |
| c-path: /usr/bin/clang-16 | |
| cpp-path: /usr/bin/clang++-16 | |
| install: True | |
| - id: clang-17 | |
| name: clang | |
| version: 17 | |
| c-path: /usr/bin/clang-17 | |
| cpp-path: /usr/bin/clang++-17 | |
| install: True | |
| build-type: | |
| - id: debug | |
| conan-build-type: Debug | |
| conan-jemalloc-build-type: Debug | |
| conan-preset: debug | |
| - id: release | |
| conan-build-type: Release | |
| conan-jemalloc-build-type: Release | |
| conan-preset: release | |
| - id: relwithdebinfo-asan | |
| conan-build-type: RelWithDebInfo | |
| conan-jemalloc-build-type: Release | |
| conan-preset: relwithdebinfo | |
| conan-custom-settings: | # Could we not copy/paste these 4x? | |
| data['compiler']['gcc']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined'] | |
| data['compiler']['clang']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined'] | |
| conan-profile-custom-conf: | | |
| # no-omit-frame-pointer recommended in (A|UB|M)SAN docs for nice stack traces. | |
| tools.build:cflags = ["-fsanitize=address", "-fno-omit-frame-pointer"] | |
| tools.build:cxxflags = ["-fsanitize=address", "-fno-omit-frame-pointer"] | |
| tools.build:sharedlinkflags = ["-fsanitize=address"] | |
| tools.build:exelinkflags = ["-fsanitize=address"] | |
| conan-profile-custom-settings: | | |
| compiler.sanitizer = address | |
| conan-profile-custom-buildenv: | | |
| CXXFLAGS = -fsanitize=address -fno-omit-frame-pointer | |
| CFLAGS = -fsanitize=address -fno-omit-frame-pointer | |
| LDFLAGS = -fsanitize=address | |
| - id: relwithdebinfo-ubsan | |
| conan-build-type: RelWithDebInfo | |
| conan-jemalloc-build-type: Release | |
| conan-preset: relwithdebinfo | |
| conan-custom-settings: | | |
| data['compiler']['gcc']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined'] | |
| data['compiler']['clang']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined'] | |
| conan-profile-custom-conf: | | |
| tools.build:cflags = ["-fsanitize=undefined", "-fno-omit-frame-pointer"] | |
| tools.build:cxxflags = ["-fsanitize=undefined", "-fno-omit-frame-pointer"] | |
| tools.build:sharedlinkflags = ["-fsanitize=undefined"] | |
| tools.build:exelinkflags = ["-fsanitize=undefined"] | |
| conan-profile-custom-settings: | | |
| compiler.sanitizer = undefined | |
| conan-profile-custom-buildenv: | | |
| CXXFLAGS = -fsanitize=undefined -fno-omit-frame-pointer | |
| CFLAGS = -fsanitize=undefined -fno-omit-frame-pointer | |
| LDFLAGS = -fsanitize=undefined | |
| - id: relwithdebinfo-msan | |
| conan-build-type: RelWithDebInfo | |
| conan-jemalloc-build-type: Release | |
| conan-preset: relwithdebinfo | |
| conan-custom-settings: | | |
| data['compiler']['gcc']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined'] | |
| data['compiler']['clang']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined'] | |
| conan-profile-custom-conf: | | |
| # See comment near step below where this ignorelist file is prepped. | |
| tools.build:cflags = ["-fsanitize=memory", "-fno-omit-frame-pointer", "-fsanitize-ignorelist=/tmp/msan_ignore_list.cfg"] | |
| tools.build:cxxflags = ["-fsanitize=memory", "-fno-omit-frame-pointer", "-fsanitize-ignorelist=/tmp/msan_ignore_list.cfg"] | |
| tools.build:sharedlinkflags = ["-fsanitize=memory"] | |
| tools.build:exelinkflags = ["-fsanitize=memory"] | |
| conan-profile-custom-settings: | | |
| compiler.sanitizer = memory | |
| conan-profile-custom-buildenv: | | |
| LDFLAGS = -fsanitize=memory | |
| CXXFLAGS = -fsanitize=memory -fno-omit-frame-pointer -fsanitize-ignorelist=/tmp/msan_ignore_list.cfg | |
| CFLAGS = -fsanitize=memory -fno-omit-frame-pointer -fsanitize-ignorelist=/tmp/msan_ignore_list.cfg | |
| - id: relwithdebinfo-tsan | |
| conan-build-type: RelWithDebInfo | |
| conan-jemalloc-build-type: Release | |
| conan-preset: relwithdebinfo | |
| conan-custom-settings: | | |
| data['compiler']['gcc']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined'] | |
| data['compiler']['clang']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined'] | |
| conan-profile-custom-conf: | | |
| tools.build:cflags = ["-fsanitize=thread"] | |
| tools.build:cxxflags = ["-fsanitize=thread"] | |
| tools.build:sharedlinkflags = ["-fsanitize=thread"] | |
| tools.build:exelinkflags = ["-fsanitize=thread"] | |
| conan-profile-custom-settings: | | |
| compiler.sanitizer = thread | |
| conan-profile-custom-buildenv: | | |
| LDFLAGS = -fsanitize=thread | |
| CXXFLAGS = -fsanitize=thread | |
| CFLAGS = -fsanitize=thread | |
| gtest-unit-tests-params: | # As of this writing these trigger assert+bugs in TSAN runtime. TODO: Revisit. | |
| --gtest_filter=-\ | |
| Shm_session_test.In_process_array:\ | |
| Shm_session_test.In_process_vector_offset_ptr:\ | |
| Shm_session_test.In_process_string_offset_ptr:\ | |
| Shm_session_test.In_process_list_offset_ptr:\ | |
| Shm_session_test.Multisession_in_process:\ | |
| Shm_session_test.Allocation_performance_five:\ | |
| Jemalloc_shm_pool_collection_test.Interface:\ | |
| Jemalloc_shm_pool_collection_test.Multiprocess:\ | |
| Jemalloc_shm_pool_collection_test.Owner_shm_pool_removal:\ | |
| Jemalloc_shm_pool_collection_test.Multithread_load:\ | |
| Borrower_shm_pool_collection_test.Interface:\ | |
| Borrower_shm_pool_collection_test.Multiprocess:\ | |
| Shm_pool_collection_test.Interface:\ | |
| Shm_pool_collection_test.Multiprocess:\ | |
| Shm_session_data_test.Multithread_object_database | |
| - id: relwithdebinfo | |
| conan-build-type: RelWithDebInfo | |
| conan-jemalloc-build-type: Release | |
| conan-preset: relwithdebinfo | |
| - id: minsizerel | |
| conan-build-type: MinSizeRel | |
| conan-jemalloc-build-type: Release | |
| conan-preset: minsizerel | |
| # XXX (pretty much everything off except ASAN + clang-*... eep) | |
| # XXX (ASAN is good; now trying UBSAN and MSAN) | |
| exclude: | |
| - build-type: { id: debug } | |
| - build-type: { id: release } | |
| - build-type: { id: minsizerel } | |
| - build-type: { id: relwithdebinfo } | |
| - build-type: { id: relwithdebinfo-tsan } | |
| - compiler: { id: gcc-9 } | |
| - compiler: { id: gcc-10 } | |
| - compiler: { id: gcc-11 } | |
| - compiler: { id: gcc-13 } | |
| #- compiler: { id: clang-13 } | |
| # - compiler: { id: clang-15 } | |
| # - compiler: { id: clang-16 } | |
| runs-on: ubuntu-latest | |
| name: | | |
| build-${{ matrix.compiler.id }}-${{ matrix.build-type.id }} | |
| env: | |
| # Run-time controls for various sanitizers (ignored when test/demo is not build with | |
| # a *SAN build-type). Some notes: | |
| # | |
| # Unclear if we need disable_coredump=0; it might be a gcc thing due to historic | |
| # issues with core size with ASAN; it is not documented in the clang *SAN docs for any | |
| # sanitizers; but it is accepted for ASAN and TSAN, seemingly, and maybe is at worst | |
| # harmless with out compiler versions. TODO: Maybe revisit. | |
| # | |
| # print_stacktrace=1 for UBSAN is recommended by documentation for nice stack traces | |
| # (also that is why we apply sanitizers to RelWithDebInfo; DebInfo part for the nice | |
| # stack traces/etc.; Rel part to reduce the considerable slowdown from some of the | |
| # sanitizers). | |
| # | |
| # second_deadlock_stack=1 for TSAN: TODO: Explain. Don't see it in clang TSAN docs. | |
| before_each_test: | | |
| export ASAN_OPTIONS="disable_coredump=0" | |
| export UBSAN_OPTIONS="disable_coredump=0 print_stacktrace=1 suppressions=${{ github.workspace }}/ubsan_suppressions_${{ matrix.compiler.name }}_${{ matrix.compiler.version }}.cfg" | |
| export TSAN_OPTIONS="disable_coredump=0 second_deadlock_stack=1 suppressions=${{ github.workspace }}/tsan_suppressions_${{ matrix.compiler.name }}_${{ matrix.compiler.version }}.cfg" | |
| steps: | |
| - name: Update available software list for apt-get | |
| run: sudo apt-get update | |
| - name: Checkout `ipc` repository and submodules (`flow`, `ipc_*`) | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| # Many compilers are pre-installed; if not then fetch it. | |
| - name: Install clang compiler | |
| if: | | |
| matrix.compiler.install && (matrix.compiler.name == 'clang') | |
| run: | | |
| wget https://apt.llvm.org/llvm.sh | |
| chmod u+x llvm.sh | |
| sudo ./llvm.sh ${{ matrix.compiler.version }} | |
| - name: Install gcc compiler | |
| if: | | |
| matrix.compiler.install && (matrix.compiler.name == 'gcc') | |
| run: | | |
| sudo apt-get install -y software-properties-common | |
| sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y | |
| sudo apt-get update | |
| sudo apt-get install -y gcc-${{ matrix.compiler.version }} g++-${{ matrix.compiler.version }} | |
| - name: Install the latest version of Conan which is less than 2 | |
| run: | | |
| pip install "conan<2" | |
| - name: Add custom settings for Conan packages | |
| if: | | |
| matrix.build-type.conan-custom-settings | |
| run: | | |
| conan config init | |
| pip install PyYAML | |
| CONAN_SETTINGS_PATH=$(conan config home)/settings.yml | |
| python -c " | |
| import yaml | |
| with open('$CONAN_SETTINGS_PATH', 'r') as file: | |
| data = yaml.safe_load(file) | |
| ${{ matrix.build-type.conan-custom-settings }} | |
| with open('$CONAN_SETTINGS_PATH', 'w') as file: | |
| yaml.dump(data, file) | |
| " | |
| - name: Create Conan profile | |
| run: | | |
| cat <<EOF > conan_profile | |
| [settings] | |
| compiler = ${{ matrix.compiler.name }} | |
| compiler.version = ${{ matrix.compiler.version }} | |
| compiler.cppstd = 17 | |
| compiler.libcxx = libstdc++11 | |
| arch = x86_64 | |
| os = Linux | |
| build_type = ${{ matrix.build-type.conan-build-type }} | |
| jemalloc:build_type = ${{ matrix.build-type.conan-jemalloc-build-type }} | |
| ${{ matrix.build-type.conan-profile-custom-settings }} | |
| [conf] | |
| tools.build:compiler_executables = {"c": "${{ matrix.compiler.c-path }}", "cpp": "${{ matrix.compiler.cpp-path }}"} | |
| tools.env.virtualenv:auto_use = True | |
| ${{ matrix.build-type.conan-profile-custom-conf }} | |
| [buildenv] | |
| CC = ${{ matrix.compiler.c-path }} | |
| CXX = ${{ matrix.compiler.cpp-path }} | |
| ${{ matrix.build-type.conan-profile-custom-buildenv }} | |
| [options] | |
| jemalloc:enable_cxx = False | |
| jemalloc:prefix = je_ | |
| ipc:build = True | |
| ipc:doc = False | |
| flow:build = True | |
| flow:doc = False | |
| EOF | |
| # We need to prepare a sanitizer ignore-list in MSAN mode. Background for this is subtle and annoying: | |
| # As it stands, whatever matrix compiler/build-type is chosen applies not just to our code (correct) | |
| # and 3rd party libraries we link like lib{boost_*|capnp|kj|jemalloc} (semi-optional but good) but also | |
| # unfortunately any items built from source during "Install Flow-IPC dependencies" step that we then | |
| # use during during the build step for our own code subsequently. At a minimum this will slow down | |
| # such programs. (For the time being we accept this as not-so-bad; to target this config at some things | |
| # but not others is hard/a ticket.) In particular, though, the capnp compiler binary is built this way; | |
| # and our "Build targets" step uses it to build key things (namely convert .capnp schemas into .c++ | |
| # and .h sources which are then themselves compiled/used in compilation). In the case of MSAN, this | |
| # version of capnp compiler happens to trigger several MSAN failures (presumably they are not a true | |
| # problem, and it's not our job really to sanitize capnp -- though we can file tickets for them and/or | |
| # issue PRs; but I digress). So in MSAN mode our build step fails, when capnp compiler itself aborts | |
| # with MSAN failures. One approach is to not apply MSAN to capnp compiler (no-go for now; it's a ticket | |
| # as mentioned); another is to uses Conan to patch capnp package for them (not a bad idea; ticket filed); | |
| # and lastly we can put the specific failures on the ignore-list for MSAN. For now we do that; while | |
| # unpleasant it does get the job done. TODO: Revisit/resolve tickets/improve (see above). | |
| - name: Prepare MSAN sanitizer compile-time config file(s) | |
| if: | | |
| (!cancelled()) && (matrix.build-type.id == 'relwithdebinfo-msan') | |
| run: | | |
| # TODO: Ideally would go, like other things, under github.workspace somewhere (maybe build/), | |
| # but that var isn't available in `strategy` up above, where we specify the compiler option | |
| # pointing to this file. Surely we could cook something up; but meantime /tmp should work; | |
| # I think jobs don't execute concurrently in one file-system, and it should be safe enough in /tmp. | |
| # Though /tmp can get cleared spuriously or something.... | |
| cat <<'EOF' > /tmp/msan_ignore_list.cfg | |
| [memory] | |
| # Warning: In clang-18 there are breaking changes in how globs/regexes are interpreted. See docs. | |
| # Currently assuming clang-17 or lower. | |
| # | |
| # capnp compiler MSAN failures suppressed: | |
| src:*/kj/filesystem-disk-unix.* | |
| src:*/bits/stl_tree.h | |
| # | |
| # XXX Other suppressions (cmnt, maybe take sep file?). | |
| # boost.chrono duration global initializer => std::string uninit value (TODO: too general but...). | |
| # There is a number of these, including things like operator+() which are tough to specify in | |
| # this format; for now going in for a pound, as they say: the entire file. | |
| src:*/bit/basic_string.h | |
| EOF | |
| - name: Install Flow-IPC dependencies with Conan using the profile | |
| run: | | |
| conan editable add flow flow/1.0 | |
| conan install \ | |
| . \ | |
| --profile:build conan_profile \ | |
| --profile:host conan_profile \ | |
| --build missing | |
| - name: Get the number of processor cores for parallelized work | |
| uses: SimenB/github-actions-cpu-cores@v1 | |
| id: cpu-cores | |
| # TODO: Ideally let our CMake interrogate jemalloc-config to find jemalloc-prefix itself. | |
| # Less stuff for us to worry about that way here. If it has not been tried, try it. | |
| # If it does not work, apply a small effort to see if it can be easily made to work. | |
| # XXX LTO disabled to maybe simplify ASAN situation/get rid of crazy linker errors | |
| - name: Prepare Makefile using CMake | |
| run: | | |
| cmake \ | |
| --preset ${{ matrix.build-type.conan-preset }} \ | |
| -DCFG_ENABLE_TEST_SUITE=ON \ | |
| -DCFG_NO_LTO=ON \ | |
| -DJEMALLOC_PREFIX=je_ \ | |
| -DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }} | |
| - name: Build targets (libraries, demos/tests) with Makefile | |
| run: | | |
| VERBOSE=1 make \ | |
| --keep-going \ | |
| --directory $GITHUB_WORKSPACE/build/${{ matrix.build-type.conan-build-type }} \ | |
| -j${{ steps.cpu-cores.outputs.count }} | |
| - name: Install targets with Makefile | |
| run: | | |
| VERBOSE=1 make install \ | |
| --directory $GITHUB_WORKSPACE/build/${{ matrix.build-type.conan-build-type }} \ | |
| -j${{ steps.cpu-cores.outputs.count }} | |
| # Save runner space: blow away build dir after install. | |
| rm -rf $GITHUB_WORKSPACE/build/${{ matrix.build-type.conan-build-type }} | |
| # From now on use !cancelled() to try to run any test/demo that exists regardless | |
| # of preceding failures if any. Same-ish (always()) with the log-upload at the end. | |
| # Worst-case they'll all fail in super-basic ways and immediately; no extra harm done. | |
| # From now on save logs in install-dir/bin/logs. They will be tarred up and uploaded | |
| # as artifacts at the end. For those tests below that produce separate logs as files | |
| # to begin-with, this is a no-brainer. For those (as of this writing the main one | |
| # is unit_test; but also the link tests produce short logs too) who use stdout/stderr | |
| # the reason to do this is 2-fold. | |
| # - UBSAN in particular produces non-fatal error output about problems it detects. | |
| # We need to analyze them after the fact and fail a step below in that case to | |
| # alert developers who would then fix such new problems. | |
| # - One can also build UBSAN-mode things with -fno-sanitize-recover which would | |
| # cause abnormal program exit on *first* error. We do not do this for these | |
| # reasons: | |
| # - It is more convenient to let it continue and thus show all problems in one | |
| # shot. | |
| # - There is a side problem: At this time compile settings, including | |
| # in UBSAN case `-fsanitize=undefined -fno-sanitize-recover`, are applied not | |
| # just to our code or 3rd party libraries but also any other stuff built | |
| # (due to setting C[XX]FLAGS in Conan profile). Targeting just the exact | |
| # stuff we want with those is hard and a separate project/ticket. | |
| # In the meantime -fno-sanitize-recover causes completely unrealted program | |
| # halts during the very-early step, when building dependencies including | |
| # capnp; some autotools configure.sh fails crazily, and nothing can work | |
| # from that point on due to dependencies-install step failing. So at this | |
| # time -fno-sanitize-recover is a no-go; it affects too much unrelated stuff. | |
| # - It is more consistent/convenient to get all the bulky logs in one place as an | |
| # artifact, rather than some in the pipeline output, others in the artifacts. | |
| # (This preference is subjective, yes.) | |
| - name: Run link test [`ipc_core` - Flow-IPC Core] | |
| if: | | |
| !cancelled() | |
| run: | | |
| ${{ env.before_each_test }} | |
| cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin | |
| mkdir -p logs/ipc_core_link_test | |
| ./ipc_core_link_test.exec > logs/ipc_core_link_test/console.log 2>&1 | |
| - name: Run link test [`ipc_transport_structured` - Flow-IPC Structured Transport] | |
| if: | | |
| !cancelled() | |
| run: | | |
| ${{ env.before_each_test }} | |
| cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin | |
| mkdir -p logs/ipc_transport_structured_link_test | |
| ./ipc_transport_structured_link_test.exec > logs/ipc_transport_structured_link_test/console.log 2>&1 | |
| # Server will exit automatically (same below). | |
| - name: Run link test [`ipc_session` - Flow-IPC Sessions] | |
| if: | | |
| !cancelled() | |
| run: | | |
| ${{ env.before_each_test }} | |
| cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin | |
| mkdir -p logs/ipc_session_link_test | |
| ./ipc_session_link_test_srv.exec > logs/ipc_session_link_test/srv.console.log 2>&1 & | |
| sleep 1 | |
| ./ipc_session_link_test_cli.exec > logs/ipc_session_link_test/cli.console.log 2>&1 | |
| - name: Run link test [`ipc_shm` - Flow-IPC Shared Memory] | |
| if: | | |
| !cancelled() | |
| run: | | |
| ${{ env.before_each_test }} | |
| cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin | |
| mkdir -p logs/ipc_shm_link_test | |
| ./ipc_shm_link_test_srv.exec > logs/ipc_shm_link_test/srv.console.log 2>&1 & | |
| sleep 1 | |
| ./ipc_shm_link_test_cli.exec > logs/ipc_shm_link_test/cli.console.log 2>&1 | |
| - name: Run link test [`ipc_shm_arena_lend` - Flow-IPC SHM-jemalloc] | |
| if: | | |
| !cancelled() | |
| run: | | |
| ${{ env.before_each_test }} | |
| cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin | |
| mkdir -p logs/ipc_shm_arena_lend_link_test | |
| ./ipc_shm_arena_lend_link_test_srv.exec > logs/ipc_shm_arena_lend_link_test/srv.console.log 2>&1 & | |
| sleep 1 | |
| ./ipc_shm_arena_lend_link_test_cli.exec > logs/ipc_shm_arena_lend_link_test/cli.console.log 2>&1 | |
| # As of this writing this runs with high verbosity, so we do not re-run on failure | |
| # like the ones below. | |
| - name: Run unit tests | |
| if: | | |
| !cancelled() | |
| run: | | |
| ${{ env.before_each_test }} | |
| echo $TSAN_OPTIONS | |
| cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin | |
| mkdir -p logs/libipc_unit_test | |
| ./libipc_unit_test.exec ${{ matrix.build-type.gtest-unit-tests-params }} \ | |
| > logs/libipc_unit_test/console.log 2>&1 | |
| # For tests below where on failure it can be very helpful to look at higher-verbosity logs, | |
| # any failing step is repeated with logging hiked up. Note that formally (and practically) | |
| # higher-than-INFO verbosity is allowed to affect performance and thus is *not* right for | |
| # the main integration test run. However on failure all bets are off, and we just want the | |
| # info we can get. (Also higher-verbosity runs can take significantly longer; no one wants that.) | |
| # This follows the instructions in bin/transport_test/README.txt. | |
| - name: Prepare run script for [transport_test - Scripted mode] variations below | |
| if: | | |
| (!cancelled()) && (matrix.build-type.id != 'relwithdebinfo-tsan') | |
| run: | | |
| cat <<'EOF' > $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_sc.sh | |
| # Script created by pipeline during job. | |
| echo "Log level: [$1]." | |
| cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/transport_test | |
| OUT_DIR_NAME=log_level_$1 | |
| OUT_DIR=../logs/transport_test/scripted/$OUT_DIR_NAME | |
| mkdir -p $OUT_DIR | |
| ./transport_test.exec scripted $OUT_DIR/srv.log info $1 \ | |
| < srv-script.txt > $OUT_DIR/srv.console.log 2>&1 & | |
| SRV_PID=$! | |
| sleep 1 | |
| ./transport_test.exec scripted $OUT_DIR/cli.log info $1 \ | |
| < cli-script.txt > $OUT_DIR/cli.console.log 2>&1 & | |
| CLI_PID=$! | |
| if wait $SRV_PID; then SRV_EC=0; else SRV_EC=$?; fi | |
| echo "Server finished with code [$SRV_EC]." | |
| if wait $CLI_PID; then CLI_EC=0; else CLI_EC=$?; fi | |
| echo "Client finished with code [$CLI_EC]." | |
| [ $SRV_EC -eq 0 ] && [ $CLI_EC -eq 0 ] | |
| EOF | |
| - name: Run integration test [transport_test - Scripted mode] | |
| id: transport_test_scripted | |
| if: | | |
| (!cancelled()) && (matrix.build-type.id != 'relwithdebinfo-tsan') | |
| run: | | |
| ${{ env.before_each_test }} | |
| /usr/bin/bash -e \ | |
| $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_sc.sh \ | |
| info | |
| - name: Re-run with increased logging, on failure only | |
| if: | | |
| (!cancelled()) && (steps.transport_test_scripted.outcome == 'failure') | |
| run: | | |
| ${{ env.before_each_test }} | |
| /usr/bin/bash -e \ | |
| $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_sc.sh \ | |
| data | |
| # The following [Exercise mode] tests follow the instructions in bin/transport_test/README.txt. | |
| # Note that the creation of ~/bin/ex_..._run and placement of executables there, plus | |
| # /tmp/var/run for run-time files (PID files and similar), is a necessary consequence of | |
| # the ipc::session safety model for estabshing IPC conversations (sessions). | |
| - name: Prepare IPC-session safety-friendly run-time environment for [transport_test - Exercise mode] | |
| if: | | |
| (!cancelled()) && (matrix.build-type.id != 'relwithdebinfo-tsan') | |
| run: | | |
| mkdir -p ~/bin/ex_srv_run ~/bin/ex_cli_run | |
| mkdir -p /tmp/var/run | |
| cp -v $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/transport_test/transport_test.exec \ | |
| ~/bin/ex_srv.exec | |
| cp -v ~/bin/ex_srv.exec ~/bin/ex_cli.exec | |
| - name: Prepare run script for [transport_test - Exercise mode] variations below | |
| if: | | |
| (!cancelled()) && (matrix.build-type.id != 'relwithdebinfo-tsan') | |
| run: | | |
| cat <<'EOF' > $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_ex.sh | |
| # Script created by pipeline during job. | |
| echo "Log level: [$1]." | |
| echo "Exercise sub-mode: [$2]." | |
| echo "Sub-mode snippet (none or 'shm-?'): [$3]." | |
| OUT_DIR=$GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/logs/transport_test/exercise/$2 | |
| mkdir -p $OUT_DIR | |
| cd ~/bin/ex_srv_run | |
| ~/bin/ex_srv.exec exercise-srv$3 $OUT_DIR/srv.log info $1 \ | |
| > $OUT_DIR/srv.console.log 2>&1 & | |
| SRV_PID=$! | |
| sleep 5 | |
| ~/bin/ex_cli.exec exercise-cli$3 $OUT_DIR/cli.log info $1 \ | |
| > $OUT_DIR/cli.console.log 2>&1 & | |
| CLI_PID=$! | |
| if wait $SRV_PID; then SRV_EC=0; else SRV_EC=$?; fi | |
| echo "Server finished with code [$SRV_EC]." | |
| if wait $CLI_PID; then CLI_EC=0; else CLI_EC=$?; fi | |
| echo "Client finished with code [$CLI_EC]." | |
| [ $SRV_EC -eq 0 ] && [ $CLI_EC -eq 0 ] | |
| EOF | |
| - name: Run integration test [transport_test - Exercise mode - Heap sub-mode] | |
| id: transport_test_ex_heap | |
| if: | | |
| (!cancelled()) && (matrix.build-type.id != 'relwithdebinfo-tsan') | |
| run: | | |
| ${{ env.before_each_test }} | |
| /usr/bin/bash -e \ | |
| $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_ex.sh \ | |
| info heap | |
| - name: Re-run with increased logging, on failure only | |
| if: | | |
| (!cancelled()) && (steps.transport_test_ex_heap.outcome == 'failure') | |
| run: | | |
| ${{ env.before_each_test }} | |
| /usr/bin/bash -e \ | |
| $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_ex.sh \ | |
| data heap_log_level_data | |
| - name: Run integration test [transport_test - Exercise mode - SHM-classic sub-mode] | |
| id: transport_test_ex_shm_c | |
| if: | | |
| (!cancelled()) && (matrix.build-type.id != 'relwithdebinfo-tsan') | |
| run: | | |
| ${{ env.before_each_test }} | |
| /usr/bin/bash -e \ | |
| $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_ex.sh \ | |
| info shm_classic -shm-c | |
| - name: Re-run with increased logging, on failure only | |
| if: | | |
| (!cancelled()) && (steps.transport_test_ex_shm_c.outcome == 'failure') | |
| run: | | |
| ${{ env.before_each_test }} | |
| /usr/bin/bash -e \ | |
| $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_ex.sh \ | |
| data shm_classic_log_level_data -shm-c | |
| - name: Run integration test [transport_test - Exercise mode - SHM-jemalloc sub-mode] | |
| id: transport_test_ex_shm_j | |
| if: | | |
| (!cancelled()) && (matrix.build-type.id != 'relwithdebinfo-tsan') | |
| run: | | |
| ${{ env.before_each_test }} | |
| /usr/bin/bash -e \ | |
| $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_ex.sh \ | |
| info shm_jemalloc -shm-j | |
| - name: Re-run with increased logging, on failure only | |
| if: | | |
| (!cancelled()) && (steps.transport_test_ex_shm_j.outcome == 'failure') | |
| run: | | |
| ${{ env.before_each_test }} | |
| /usr/bin/bash -e \ | |
| $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_ex.sh \ | |
| data shm_jemalloc_log_level_data -shm-j | |
| # See earlier comment block about why we saved all the logs including for console-output-only tests/demos. | |
| - name: Check test/demo logs for non-fatal sanitizer error(s) | |
| if: | | |
| (!cancelled()) && (matrix.build-type.id == 'relwithdebinfo-ubsan') | |
| run: | | |
| cd ${{ github.workspace }}/install/${{ matrix.build-type.conan-build-type }}/bin/logs | |
| # grep returns 0 if 1+ found, 1 if none found, 2+ on error. So check results explicitly instead of -e. | |
| # Namely, for us, the only OK result is an empty stdout and empty stderr. | |
| # Otherwise either grep failed (unlikely) or found 1+ problems; either way redirection target will | |
| # be not-empty, and we will force failure. | |
| grep 'SUMMARY: UndefinedBehaviorSanitizer:' `find . -type f` > san_failure_summaries.txt 2>&1 || true | |
| if [ -s san_failure_summaries.txt ]; then | |
| echo "Error(s) found. Pipeline will fail. Failures summarized below." | |
| echo "Please peruse uploaded log artifacts, resolve the issues, and run pipeline again." | |
| echo "---" | |
| cat san_failure_summaries.txt | |
| false | |
| fi | |
| echo "No errors found in logs." | |
| - name: Package test/demo logs tarball | |
| if: | | |
| always() | |
| run: | | |
| cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin | |
| tar cvzf logs.tgz logs | |
| rm -rf logs # Save runner space. | |
| - name: Upload test/demo logs (please inspect if failure(s) seen above) | |
| if: | | |
| always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ipc-logs-${{ matrix.compiler.id }}-${{ matrix.build-type.id }} | |
| path: | | |
| ${{ github.workspace }}/install/${{ matrix.build-type.conan-build-type }}/bin/logs.tgz | |
| # TODO: Look into the topic of debuggability in case of a crash. Is a core generated? Is it saved? | |
| # Do we need to manually save it as an artifact? For that matter we would then need the binary and | |
| # ideally the source. For now we save any logs that are not printed to console, and where possible | |
| # our tests/demos keep it the console exclusively (but in some cases, such as transport_test, it | |
| # is not practical due to parallel execution and other aspects). In general it is always better that | |
| # logs are sufficient; but look into situations where they are not. | |
| # | |
| # Don't forget situation where program exits due to a sanitizer reporting problem (ASAN, etc.); is | |
| # there a core? Do we need one? Etc. | |
| # | |
| # Possibly this is all handled beautifully automatically; then this should be deleted. |