Merge pull request #138 from BitgesellOfficial/release-0.1.13

Release 0.1.13
BitgesellOfficial · Aug 26, 2024 · 0e278d5 · 0e278d5
2 parents ea2b732 + 2f7d635
commit 0e278d5
Show file tree

Hide file tree

Showing 831 changed files with 25,382 additions and 28,227 deletions.
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -1,35 +1,80 @@
 env:
   ### cirrus config
   CIRRUS_CLONE_DEPTH: 1
-  PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y"
-  MAKEJOBS: "-j10"
-  TEST_RUNNER_PORT_MIN: "14000"  # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache
-  CI_FAILFAST_TEST_LEAVE_DANGLING: "1"  # Cirrus CI does not care about dangling processes and setting this variable avoids killing the CI script itself on error
-  CCACHE_MAXSIZE: "200M"
-  CCACHE_DIR: "/tmp/ccache_dir"
-  CCACHE_NOHASHDIR: "1"  # Debug info might contain a stale path if the build dir changes, but this is fine
+  ### compiler options
+  HOST:
+  WRAPPER_CMD:
+  # Specific warnings can be disabled with -Wno-error=foo.
+  # -pedantic-errors is not equivalent to -Werror=pedantic and thus not implied by -Werror according to the GCC manual.
+  WERROR_CFLAGS: -Werror -pedantic-errors
+  MAKEFLAGS: -j4
+  BUILD: check
+  ### secp256k1 config
+  ECMULTWINDOW: 15
+  ECMULTGENKB: 22
+  ASM: no
+  WIDEMUL: auto
+  WITH_VALGRIND: yes
+  EXTRAFLAGS:
+  ### secp256k1 modules
+  EXPERIMENTAL: no
+  ECDH: no
+  RECOVERY: no
+  SCHNORRSIG: no
+  ELLSWIFT: no
+  ### test options
+  SECP256K1_TEST_ITERS:
+  BENCH: yes
+  SECP256K1_BENCH_ITERS: 2
+  CTIMETESTS: yes
+  # Compile and run the tests
+  EXAMPLES: yes
 
+# A self-hosted machine(s) can be used via Cirrus CI. It can be configured with
+# multiple users to run tasks in parallel. No sudo permission is required.
+#
 # https://cirrus-ci.org/guide/persistent-workers/
 #
-# It is possible to select a specific persistent worker by label. Refer to the
+# Generally, a persistent worker must run Ubuntu 23.04+ or Debian 12+.
+#
+# The following specific types should exist, with the following requirements:
+# - small: For an x86_64 machine, recommended to have 2 CPUs and 8 GB of memory.
+# - medium: For an x86_64 machine, recommended to have 4 CPUs and 16 GB of memory.
+# - arm64: For an aarch64 machine, recommended to have 2 CPUs and 8 GB of memory.
+#
+# CI jobs for the latter configuration can be run on x86_64 hardware
+# by installing qemu-user-static, which works out of the box with
+# podman or docker. Background: https://stackoverflow.com/a/72890225/313633
+#
+# The above machine types are matched to each task by their label. Refer to the
 # Cirrus CI docs for more details.
 #
-# Generally, a persistent worker must run Ubuntu 23.04+ or Debian 12+.
-# Specifically,
+# When a contributor maintains a fork of the repo, any pull request they make
+# to their own fork, or to the main repository, will trigger two CI runs:
+# one for the branch push and one for the pull request.
+# This can be avoided by setting SKIP_BRANCH_PUSH=true as a custom env variable
+# in Cirrus repository settings, accessible from
+# https://cirrus-ci.com/github/my-organization/my-repository
+#
+# On machines that are persisted between CI jobs, RESTART_CI_DOCKER_BEFORE_RUN=1
+# ensures that previous containers and artifacts are cleared before each run.
+# This requires installing Podman instead of Docker.
+#
+# Futhermore:
 # - apt-get is required due to PACKAGE_MANAGER_INSTALL
-# - podman-docker-4.1+ is required due to the use of `podman` when
-#   RESTART_CI_DOCKER_BEFORE_RUN is set and 4.1+ due to the bugfix in 4.1
+# - podman-docker-4.1+ is required due to the bugfix in 4.1
 #   (https://github.com/bitcoin/bitcoin/pull/21652#issuecomment-1657098200)
-# - The ./ci/ depedencies (with cirrus-cli) should be installed:
+# - The ./ci/ dependencies (with cirrus-cli) should be installed. One-liner example
+#   for a single user setup with sudo permission:
 #
 #   ```
 #   apt update && apt install git screen python3 bash podman-docker curl -y && curl -L -o cirrus "https://github.com/cirruslabs/cirrus-cli/releases/latest/download/cirrus-linux-$(dpkg --print-architecture)" && mv cirrus /usr/local/bin/cirrus && chmod +x /usr/local/bin/cirrus
 #   ```
 #
-# - There are no strict requirements on the hardware, because having less CPUs
-#   runs the same CI script (maybe slower). To avoid rare and intermittent OOM
-#   due to short memory usage spikes, it is recommended to add (and persist)
-#   swap:
+# - There are no strict requirements on the hardware. Having fewer CPU threads
+#   than recommended merely causes the CI script to run slower.
+#   To avoid rare and intermittent OOM due to short memory usage spikes,
+#   it is recommended to add (and persist) swap:
 #
 #   ```
 #   fallocate -l 16G /swapfile_ci && chmod 600 /swapfile_ci && mkswap /swapfile_ci && swapon /swapfile_ci && ( echo '/swapfile_ci none swap sw 0 0' | tee -a /etc/fstab )
@@ -40,16 +85,13 @@ env:
 #   ```
 #   RESTART_CI_DOCKER_BEFORE_RUN=1 screen cirrus worker run --labels type=todo_fill_in_type --token todo_fill_in_token
 #   ```
-#
-# The following specific types should exist, with the following requirements:
-# - small: For an x86_64 machine, recommended to have 2 CPUs and 8 GB of memory.
-# - medium: For an x86_64 machine, recommended to have 4 CPUs and 16 GB of memory.
-# - noble: For a machine running the Linux kernel shipped with exaclty Ubuntu Noble 24.04. The machine is recommended to have 4 CPUs and 16 GB of memory.
-# - arm64: For an aarch64 machine, recommended to have 2 CPUs and 8 GB of memory.
 
 # https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks
 filter_template: &FILTER_TEMPLATE
-  skip: $CIRRUS_REPO_FULL_NAME == "BGL-core/gui" && $CIRRUS_PR == ""  # No need to run on the read-only mirror, unless it is a PR. https://cirrus-ci.org/guide/writing-tasks/#conditional-task-execution
+  # Allow forks to specify SKIP_BRANCH_PUSH=true and skip CI runs when a branch is pushed,
+  # but still run CI when a PR is created.
+  # https://cirrus-ci.org/guide/writing-tasks/#conditional-task-execution
+  skip: $SKIP_BRANCH_PUSH == "true" && $CIRRUS_PR == ""
   stateful: false  # https://cirrus-ci.org/guide/writing-tasks/#stateful-tasks
 
 base_template: &BASE_TEMPLATE
@@ -58,43 +100,54 @@ base_template: &BASE_TEMPLATE
     # Unconditionally install git (used in fingerprint_script).
     - git --version || bash -c "$PACKAGE_MANAGER_INSTALL git"
     - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi
-    - git fetch --depth=1 $CIRRUS_REPO_CLONE_URL "pull/${CIRRUS_PR}/merge"
-    - git checkout FETCH_HEAD  # Use merged changes to detect silent merge conflicts
-                               # Also, the merge commit is used to lint COMMIT_RANGE="HEAD~..HEAD"
-
-main_template: &MAIN_TEMPLATE
-  timeout_in: 120m  # https://cirrus-ci.org/faq/#instance-timed-out
-  ci_script:
-    - ./ci/test_run_all.sh
+    - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH
+    - git config --global user.email "[email protected]"
+    - git config --global user.name "ci"
+    - git merge FETCH_HEAD  # Merge base to detect silent merge conflicts
 
-global_task_template: &GLOBAL_TASK_TEMPLATE
-  << : *BASE_TEMPLATE
-  << : *MAIN_TEMPLATE
+env_matrix_snippet: &ENV_MATRIX_VALGRIND
+  - env:
+      ENABLE_FIELDS: "7,32,58"
+  - env:
+      BUILD: distcheck
+  - env:
+      EXEC_CMD: valgrind --error-exitcode=42
+      TESTRUNS: 1
+      BUILD:
 
-compute_credits_template: &CREDITS_TEMPLATE
-  # https://cirrus-ci.org/pricing/#compute-credits
-  # Only use credits for pull requests to the main repo
-  use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'BGL/BGL' && $CIRRUS_PR != ""
+env_matrix_snippet: &ENV_MATRIX_SAN_VALGRIND
+  - env:
+      ENABLE_FIELDS: "11,64,37"
+  - env:
+      BUILD: distcheck
+  - env:
+      EXEC_CMD: valgrind --error-exitcode=42
+      TESTRUNS: 1
+      BUILD:
+  - env:
+      CXXFLAGS:  "-fsanitize=undefined -fno-omit-frame-pointer"
+      LDFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer"
+      UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1"
+      BENCH: no
 
 task:
   name: 'lint'
   << : *BASE_TEMPLATE
   container:
-    image: debian:bookworm
-    cpu: 1
-    memory: 1G
-  # For faster CI feedback, immediately schedule the linters
-  << : *CREDITS_TEMPLATE
-  test_runner_cache:
-    folder: "/lint_test_runner"
-    fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-parse HEAD:test/lint/test_runner)
-  python_cache:
-    folder: "/python_build"
-    fingerprint_script: cat .python-version /etc/os-release
-  unshallow_script:
-    - git fetch --unshallow --no-tags
-  lint_script:
-    - ./ci/lint_run_all.sh
+    dockerfile: ci/linux-debian.Dockerfile
+    memory: 2G
+    cpu: 4
+  matrix:
+    << : *ENV_MATRIX_SAN_VALGRIND
+  matrix:
+    - env:
+        CXX: g++
+    - env:
+        CXX: clang++ -gdwarf-4
+  << : *MERGE_BASE
+  test_script:
+    - ./ci/cirrus.sh
+  << : *CAT_LOGS
 
 task:
   name: 'tidy'
@@ -103,16 +156,54 @@ task:
     labels:
       type: medium
   env:
-    FILE_ENV: "./ci/test/00_setup_env_native_tidy.sh"
+    HOST: i686-linux-gnu
+  matrix:
+    << : *ENV_MATRIX_VALGRIND
+  matrix:
+    - env:
+        CXX: i686-linux-gnu-g++
+    - env:
+        CXX: clang++ --target=i686-linux-gnu -gdwarf-4
+        CXXFLAGS: -g -O2 -isystem /usr/i686-linux-gnu/include -isystem /usr/i686-linux-gnu/include/c++/10/i686-linux-gnu
+  test_script:
+    - ./ci/cirrus.sh
+  << : *CAT_LOGS
 
 task:
-  name: 'ARM, unit tests, no functional tests'
-  << : *GLOBAL_TASK_TEMPLATE
-  persistent_worker:
-    labels:
-      type: arm64  # Use arm64 worker to sidestep qemu and avoid a slow CI: https://github.com/bitcoin/bitcoin/pull/28087#issuecomment-1649399453
+  name: "arm64: macOS Monterey"
+  macos_instance:
+    image: ghcr.io/cirruslabs/macos-monterey-base:latest
   env:
-    FILE_ENV: "./ci/test/00_setup_env_arm.sh"
+    # Cirrus gives us a fixed number of 4 virtual CPUs.
+    MAKEFLAGS: -j5
+  matrix:
+    - env:
+        CXX: g++-11
+      # Homebrew's gcc for arm64 has no libubsan.
+      matrix:
+        - env:
+            ENABLE_FIELDS: 28
+        - env:
+            BUILD: distcheck
+    - env:
+        CXX: clang++
+      matrix:
+        - env:
+            ENABLE_FIELDS: 28
+        - env:
+            BUILD: distcheck
+        - env:
+            CXXFLAGS:  "-fsanitize=undefined -fno-omit-frame-pointer"
+            LDFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer"
+            UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1"
+            BENCH: no
+  brew_script:
+    - brew update
+    - brew install automake libtool gcc@11
+  << : *MERGE_BASE
+  test_script:
+    - ./ci/cirrus.sh
+  << : *CAT_LOGS
 
 task:
   name: 'Win64, unit tests, no gui tests, no functional tests'
@@ -121,7 +212,13 @@ task:
     labels:
       type: small
   env:
-    FILE_ENV: "./ci/test/00_setup_env_win64.sh"
+    EXEC_CMD: qemu-s390x
+    HOST: s390x-linux-gnu
+    BUILD:
+  << : *MERGE_BASE
+  test_script:
+    - ./ci/cirrus.sh
+  << : *CAT_LOGS
 
 task:
   name: '32-bit CentOS, dash, gui'
@@ -160,19 +257,6 @@ task:
   env:
     FILE_ENV: "./ci/test/00_setup_env_native_msan.sh"
 
-task:
-  name: 'ASan + LSan + UBSan + integer, no depends, USDT'
-  enable_bpfcc_script:
-    # In the image build step, no external environment variables are available,
-    # so any settings will need to be written to the settings env file:
-    - sed -i "s|\${CIRRUS_CI}|true|g" ./ci/test/00_setup_env_native_asan.sh
-  << : *GLOBAL_TASK_TEMPLATE
-  persistent_worker:
-    labels:
-      type: noble  # Must use this specific worker (needed for USDT functional tests)
-  env:
-    FILE_ENV: "./ci/test/00_setup_env_native_asan.sh"
-
 task:
   name: 'fuzzer,address,undefined,integer, no depends'
   << : *GLOBAL_TASK_TEMPLATE
@@ -192,20 +276,13 @@ task:
     FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh"
 
 task:
-  name: "ARM64: Linux (Debian stable)"
+  name: 'no wallet, libBGLkernel'
+  << : *GLOBAL_TASK_TEMPLATE
   persistent_worker:
     labels:
-      type: arm64
+      type: small
   env:
-    ECDH: yes
-    RECOVERY: yes
-    SCHNORRSIG: yes
-    ELLSWIFT: yes
-  matrix:
-     # Currently only gcc-snapshot, the other compilers are tested on GHA with QEMU
-     - env: { CC: 'gcc-snapshot' }
-  << : *LINUX_ARM64_CONTAINER
-  << : *CAT_LOGS
+    FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libBGLkernel.sh"
 
 task:
   name: 'macOS-cross, gui, no tests'

diff --git a/.github/actions/install-homebrew-valgrind/action.yml b/.github/actions/install-homebrew-valgrind/action.yml
@@ -16,7 +16,7 @@ runs:
         cat valgrind_fingerprint
       shell: bash
 
-    - uses: actions/cache@v3
+    - uses: actions/cache@v4
       id: cache
       with:
         path: ${{ env.CI_HOMEBREW_CELLAR_VALGRIND }}

diff --git a/.github/actions/run-in-docker-action/action.yml b/.github/actions/run-in-docker-action/action.yml
@@ -36,6 +36,11 @@ runs:
         load: true
         cache-from: type=gha
 
+    - # Workaround for https://github.com/google/sanitizers/issues/1614 .
+      # The underlying issue has been fixed in clang 18.1.3.
+      run: sudo sysctl -w vm.mmap_rnd_bits=28
+      shell: bash
+
     - # Tell Docker to pass environment variables in `env` into the container.
       run: >
         docker run \