diff --git a/.gitignore b/.gitignore index 1f2c0cd540459..38adaf286c6b9 100644 --- a/.gitignore +++ b/.gitignore @@ -135,6 +135,7 @@ win32-build test/config.ini test/cache/* test/.mypy_cache/ +test/lint/test_runner/target/ !src/leveldb*/Makefile diff --git a/Makefile.am b/Makefile.am index 4d4907ead252d..c12f9692efc86 100644 --- a/Makefile.am +++ b/Makefile.am @@ -337,7 +337,7 @@ clean-docs: clean-local: clean-docs rm -rf coverage_percent.txt test_groestlcoin.coverage/ total.coverage/ fuzz.coverage/ test/tmp/ cache/ $(OSX_APP) rm -rf test/functional/__pycache__ test/functional/test_framework/__pycache__ test/cache share/rpcauth/__pycache__ - rm -rf osx_volname dist/ + rm -rf osx_volname dist/ test/lint/test_runner/target/ test/lint/__pycache__ test-security-check: if TARGET_DARWIN diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index e0bb78c70abe3..b27609ffb6619 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -8,21 +8,24 @@ export LC_ALL=C set -ex -if [ -n "$LOCAL_BRANCH" ]; then - # To faithfully recreate CI linting locally, specify all commits on the current - # branch. - COMMIT_RANGE="$(git merge-base HEAD master)..HEAD" -elif [ -n "$CIRRUS_PR" ]; then +if [ -n "$CIRRUS_PR" ]; then COMMIT_RANGE="HEAD~..HEAD" - echo - git log --no-merges --oneline "$COMMIT_RANGE" - echo - test/lint/commit-script-check.sh "$COMMIT_RANGE" + if [ "$(git rev-list -1 HEAD)" != "$(git rev-list -1 --merges HEAD)" ]; then + echo "Error: The top commit must be a merge commit, usually the remote 'pull/${PR_NUMBER}/merge' branch." + false + fi else - COMMIT_RANGE="SKIP_EMPTY_NOT_A_PR" + # Otherwise, assume that a merge commit exists. This merge commit is assumed + # to be the base, after which linting will be done. If the merge commit is + # HEAD, the range will be empty. + COMMIT_RANGE="$( git rev-list --max-count=1 --merges HEAD )..HEAD" fi export COMMIT_RANGE +echo +git log --no-merges --oneline "$COMMIT_RANGE" +echo +test/lint/commit-script-check.sh "$COMMIT_RANGE" RUST_BACKTRACE=1 "${LINT_RUNNER_PATH}/test_runner" if [ "$CIRRUS_REPO_FULL_NAME" = "groestlcoin/groestlcoin" ] && [ "$CIRRUS_PR" = "" ] ; then diff --git a/ci/lint/container-entrypoint.sh b/ci/lint/container-entrypoint.sh index b15c38591570c..df7147a3a8b54 100755 --- a/ci/lint/container-entrypoint.sh +++ b/ci/lint/container-entrypoint.sh @@ -14,7 +14,7 @@ export PATH="/python_build/bin:${PATH}" export LINT_RUNNER_PATH="/lint_test_runner" if [ -z "$1" ]; then - LOCAL_BRANCH=1 bash -ic "./ci/lint/06_script.sh" + bash -ic "./ci/lint/06_script.sh" else exec "$@" fi diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index 6db5827c322d6..1cbcb43652465 100644 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -20,7 +20,7 @@ export CONTAINER_NAME=ci_native_asan export PACKAGES="systemtap-sdt-dev clang-17 llvm-17 libclang-rt-17-dev python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}" export NO_DEPENDS=1 export GOAL="install" -export GROESTLCOIN_CONFIG="--enable-c++20 --enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 \ +export GROESTLCOIN_CONFIG="--enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 \ CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' \ --with-sanitizers=address,float-divide-by-zero,integer,undefined \ CC='clang-17 -ftrivial-auto-var-init=pattern' CXX='clang++-17 -ftrivial-auto-var-init=pattern'" diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index 2967935ebc72f..eec7b9b25ef3f 100644 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04" +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" LIBCXX_DIR="/msan/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index 70c5770a2a083..e3cdfcfc787d5 100644 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04" +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" LIBCXX_DIR="/msan/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" diff --git a/ci/test/00_setup_env_native_previous_releases.sh b/ci/test/00_setup_env_native_previous_releases.sh index 99b60be02373a..fc63de2bd622d 100644 --- a/ci/test/00_setup_env_native_previous_releases.sh +++ b/ci/test/00_setup_env_native_previous_releases.sh @@ -7,10 +7,10 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_previous_releases -export CI_IMAGE_NAME_TAG="docker.io/debian:bullseye" -# Use minimum supported python3.9 and gcc-10, see doc/dependencies.md -export PACKAGES="gcc-10 g++-10 python3-zmq" -export DEP_OPTS="NO_UPNP=1 NO_NATPMP=1 DEBUG=1 CC=gcc-10 CXX=g++-10" +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04" +# Use minimum supported python3.9 (or best effort 3.10) and gcc-11, see doc/dependencies.md +export PACKAGES="gcc-11 g++-11 python3-zmq" +export DEP_OPTS="NO_UPNP=1 NO_NATPMP=1 DEBUG=1 CC=gcc-11 CXX=g++-11" export GOAL="install" export DOWNLOAD_PREVIOUS_RELEASES="true" export GROESTLCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-reduce-exports --enable-debug \ diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh index 319ae843473b8..0732165abf9e3 100644 --- a/ci/test/00_setup_env_native_tidy.sh +++ b/ci/test/00_setup_env_native_tidy.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_tidy -export TIDY_LLVM_V="17" +export TIDY_LLVM_V="18" export PACKAGES="clang-${TIDY_LLVM_V} libclang-${TIDY_LLVM_V}-dev llvm-${TIDY_LLVM_V}-dev libomp-${TIDY_LLVM_V}-dev clang-tidy-${TIDY_LLVM_V} jq bear libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index 99813da106b54..6f1498963ae87 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2022 The Bitcoin Core developers +# Copyright (c) 2018-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -36,7 +36,7 @@ if [ -n "$PIP_PACKAGES" ]; then fi if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-17.0.6 /msan/llvm-project + ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-18.1.1" /msan/llvm-project cmake -G Ninja -B /msan/clang_build/ \ -DLLVM_ENABLE_PROJECTS="clang" \ @@ -53,13 +53,14 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer /msan/clang_build/bin/llvm-symbolizer 100 cmake -G Ninja -B /msan/cxx_build/ \ - -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_USE_SANITIZER=MemoryWithOrigins \ -DCMAKE_C_COMPILER=clang \ -DCMAKE_CXX_COMPILER=clang++ \ -DLLVM_TARGETS_TO_BUILD=Native \ -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \ + -DLIBCXXABI_USE_LLVM_UNWINDER=OFF \ -DLIBCXX_HARDENING_MODE=debug \ -S /msan/llvm-project/runtimes diff --git a/ci/test/03_test_script.sh b/ci/test/03_test_script.sh old mode 100755 new mode 100644 index 242a12de48ba3..a7263c3c2fe07 --- a/ci/test/03_test_script.sh +++ b/ci/test/03_test_script.sh @@ -10,7 +10,7 @@ set -ex export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1" export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan" -export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" +export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1" export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" if [ "$CI_OS_NAME" == "macos" ]; then @@ -53,7 +53,7 @@ export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")} EOF ) -mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" + if [ "$USE_BUSY_BOX" = "true" ]; then echo "Setup to use BusyBox utils" diff --git a/contrib/devtools/bitcoin-tidy/CMakeLists.txt b/contrib/devtools/bitcoin-tidy/CMakeLists.txt index 35e60d1d87e2b..1260c714236ff 100644 --- a/contrib/devtools/bitcoin-tidy/CMakeLists.txt +++ b/contrib/devtools/bitcoin-tidy/CMakeLists.txt @@ -1,14 +1,25 @@ -cmake_minimum_required(VERSION 3.9) +cmake_minimum_required(VERSION 3.22) -project(bitcoin-tidy VERSION 1.0.0 DESCRIPTION "clang-tidy checks for Bitcoin Core") +project(bitcoin-tidy + VERSION + 1.0.0 + DESCRIPTION "clang-tidy checks for Bitcoin Core" + LANGUAGES CXX) include(GNUInstallDirs) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) set(CMAKE_CXX_EXTENSIONS False) -# TODO: Figure out how to avoid the terminfo check +set(CMAKE_DISABLE_FIND_PACKAGE_CURL ON) +set(CMAKE_DISABLE_FIND_PACKAGE_FFI ON) +set(CMAKE_DISABLE_FIND_PACKAGE_LibEdit ON) +set(CMAKE_DISABLE_FIND_PACKAGE_LibXml2 ON) +set(CMAKE_DISABLE_FIND_PACKAGE_Terminfo ON) +set(CMAKE_DISABLE_FIND_PACKAGE_ZLIB ON) +set(CMAKE_DISABLE_FIND_PACKAGE_zstd ON) + find_package(LLVM REQUIRED CONFIG) find_program(CLANG_TIDY_EXE NAMES "clang-tidy-${LLVM_VERSION_MAJOR}" "clang-tidy" HINTS ${LLVM_TOOLS_BINARY_DIR}) message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") diff --git a/contrib/devtools/bitcoin-tidy/README b/contrib/devtools/bitcoin-tidy/README.md similarity index 100% rename from contrib/devtools/bitcoin-tidy/README rename to contrib/devtools/bitcoin-tidy/README.md diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index 194ee411204dd..e86085cabc86d 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -51,7 +51,7 @@ fi time-machine() { # shellcheck disable=SC2086 guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=d5ca4d4fd713a9f7e17e074a1e37dda99bbb09fc \ + --commit=dc4842797bfdc5f9f3f5f725bf189c2b68bd6b5a \ --cores="$JOBS" \ --keep-failed \ --fallback \ diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 6da0baeff1634..7b3ba436f0e39 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -423,6 +423,7 @@ inspecting signatures in Mach-O binaries.") (list "--enable-initfini-array=yes", "--enable-default-ssp=yes", "--enable-default-pie=yes", + "--enable-standard-branch-protection=yes", building-on))) ((#:phases phases) `(modify-phases ,phases diff --git a/depends/funcs.mk b/depends/funcs.mk index 7b5c3d0c59158..494ed5d32457e 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -147,7 +147,7 @@ $(1)_stage_env+=PATH="$(build_prefix)/bin:$(PATH)" # config.guess, which is what we set it too here. This also quells autoconf # warnings, "If you wanted to set the --build type, don't use --host.", # when using versions older than 2.70. -$(1)_autoconf=./configure --build=$(BUILD) --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)" +$(1)_autoconf=./configure --build=$(BUILD) --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) --with-pic $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)" ifneq ($($(1)_nm),) $(1)_autoconf += NM="$$($(1)_nm)" endif @@ -170,12 +170,19 @@ ifneq ($($(1)_ldflags),) $(1)_autoconf += LDFLAGS="$$($(1)_ldflags)" endif +# We hardcode the library install path to "lib" to match the PKG_CONFIG_PATH +# setting in depends/config.site.in, which also hardcodes "lib". +# Without this setting, CMake by default would use the OS library +# directory, which might be "lib64" or something else, not "lib", on multiarch systems. $(1)_cmake=env CC="$$($(1)_cc)" \ CFLAGS="$$($(1)_cppflags) $$($(1)_cflags)" \ CXX="$$($(1)_cxx)" \ CXXFLAGS="$$($(1)_cppflags) $$($(1)_cxxflags)" \ LDFLAGS="$$($(1)_ldflags)" \ - cmake -DCMAKE_INSTALL_PREFIX:PATH="$$($($(1)_type)_prefix)" $$($(1)_config_opts) + cmake -DCMAKE_INSTALL_PREFIX:PATH="$$($($(1)_type)_prefix)" \ + -DCMAKE_INSTALL_LIBDIR=lib/ \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + $$($(1)_config_opts) ifeq ($($(1)_type),build) $(1)_cmake += -DCMAKE_INSTALL_RPATH:PATH="$$($($(1)_type)_prefix)/lib" else diff --git a/depends/packages.md b/depends/packages.md index ad91eaffee47e..0ffdc66d4884f 100644 --- a/depends/packages.md +++ b/depends/packages.md @@ -162,6 +162,9 @@ From the [Gentoo Wiki entry](https://wiki.gentoo.org/wiki/Project:Quality_Assura > creates. This leads to massive overlinking, which is toxic to the Gentoo > ecosystem, as it leads to a massive number of unnecessary rebuilds. +Where possible, packages are built with Position Independant Code. Either using +the Autotools `--with-pic` flag, or `CMAKE_POSITION_INDEPENDENT_CODE` with CMake. + ## Secondary dependencies: Secondary dependency packages relative to the bitcoin binaries/libraries (i.e. diff --git a/depends/packages/bdb.mk b/depends/packages/bdb.mk index 8beb25819bf7e..6b0ff71b2b3cf 100644 --- a/depends/packages/bdb.mk +++ b/depends/packages/bdb.mk @@ -9,11 +9,6 @@ $(package)_patches=clang_cxx_11.patch define $(package)_set_vars $(package)_config_opts=--disable-shared --enable-cxx --disable-replication --enable-option-checking $(package)_config_opts_mingw32=--enable-mingw -$(package)_config_opts_linux=--with-pic -$(package)_config_opts_freebsd=--with-pic -$(package)_config_opts_netbsd=--with-pic -$(package)_config_opts_openbsd=--with-pic -$(package)_config_opts_android=--with-pic $(package)_cflags+=-Wno-error=implicit-function-declaration -Wno-error=format-security -Wno-error=implicit-int $(package)_cppflags_freebsd=-D_XOPEN_SOURCE=600 -D__BSD_VISIBLE=1 $(package)_cppflags_netbsd=-D_XOPEN_SOURCE=600 diff --git a/depends/packages/capnp.mk b/depends/packages/capnp.mk index 2465c8091bbeb..6d792db711364 100644 --- a/depends/packages/capnp.mk +++ b/depends/packages/capnp.mk @@ -5,15 +5,10 @@ $(package)_download_file=$(native_$(package)_download_file) $(package)_file_name=$(native_$(package)_file_name) $(package)_sha256_hash=$(native_$(package)_sha256_hash) -# Hardcode library install path to "lib" to match the PKG_CONFIG_PATH -# setting in depends/config.site.in, which also hardcodes "lib". -# Without this setting, cmake by default would use the OS library -# directory, which might be "lib64" or something else, not "lib", on multiarch systems. define $(package)_set_vars := $(package)_config_opts := -DBUILD_TESTING=OFF $(package)_config_opts += -DWITH_OPENSSL=OFF $(package)_config_opts += -DWITH_ZLIB=OFF - $(package)_config_opts += -DCMAKE_INSTALL_LIBDIR=lib/ endef define $(package)_config_cmds diff --git a/depends/packages/expat.mk b/depends/packages/expat.mk index bb203d06f8442..2db283ef3cf42 100644 --- a/depends/packages/expat.mk +++ b/depends/packages/expat.mk @@ -11,7 +11,6 @@ define $(package)_set_vars $(package)_config_opts=--disable-shared --without-docbook --without-tests --without-examples $(package)_config_opts += --disable-dependency-tracking --enable-option-checking $(package)_config_opts += --without-xmlwf - $(package)_config_opts_linux=--with-pic $(package)_cppflags += -D_DEFAULT_SOURCE endef diff --git a/depends/packages/freetype.mk b/depends/packages/freetype.mk index 6f5dbe0f01377..c28259ed6701c 100644 --- a/depends/packages/freetype.mk +++ b/depends/packages/freetype.mk @@ -7,7 +7,6 @@ $(package)_sha256_hash=8bee39bd3968c4804b70614a0a3ad597299ad0e824bc8aad5ce8aaf48 define $(package)_set_vars $(package)_config_opts=--without-zlib --without-png --without-harfbuzz --without-bzip2 --disable-static $(package)_config_opts += --enable-option-checking --without-brotli - $(package)_config_opts_linux=--with-pic endef define $(package)_config_cmds diff --git a/depends/packages/libXau.mk b/depends/packages/libXau.mk index b7e032c0b2d13..aeb14dcd6e8e7 100644 --- a/depends/packages/libXau.mk +++ b/depends/packages/libXau.mk @@ -10,7 +10,6 @@ $(package)_dependencies=xproto define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-lint-library --without-lint $(package)_config_opts += --disable-dependency-tracking --enable-option-checking - $(package)_config_opts_linux=--with-pic endef define $(package)_preprocess_cmds diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index 9650f77db9207..d764be5d0aeff 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -11,11 +11,6 @@ define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-openssl --disable-libevent-regress --disable-samples $(package)_config_opts += --disable-dependency-tracking --enable-option-checking $(package)_config_opts_release=--disable-debug-mode - $(package)_config_opts_linux=--with-pic - $(package)_config_opts_freebsd=--with-pic - $(package)_config_opts_netbsd=--with-pic - $(package)_config_opts_openbsd=--with-pic - $(package)_config_opts_android=--with-pic $(package)_cppflags_mingw32=-D_WIN32_WINNT=0x0601 ifeq ($(NO_HARDEN),) diff --git a/depends/packages/libmultiprocess.mk b/depends/packages/libmultiprocess.mk index d237f52dbb237..765d6493776bb 100644 --- a/depends/packages/libmultiprocess.mk +++ b/depends/packages/libmultiprocess.mk @@ -8,13 +8,7 @@ ifneq ($(host),$(build)) $(package)_dependencies += native_capnp endif -# Hardcode library install path to "lib" to match the PKG_CONFIG_PATH -# setting in depends/config.site.in, which also hardcodes "lib". -# Without this setting, cmake by default would use the OS library -# directory, which might be "lib64" or something else, not "lib", on multiarch systems. define $(package)_set_vars := -$(package)_config_opts += -DCMAKE_INSTALL_LIBDIR=lib/ -$(package)_config_opts += -DCMAKE_POSITION_INDEPENDENT_CODE=ON ifneq ($(host),$(build)) $(package)_config_opts := -DCAPNP_EXECUTABLE="$$(native_capnp_prefixbin)/capnp" $(package)_config_opts += -DCAPNPC_CXX_EXECUTABLE="$$(native_capnp_prefixbin)/capnpc-c++" diff --git a/depends/packages/libxcb_util.mk b/depends/packages/libxcb_util.mk index 6f1b9cd7c65e0..6e4c7359b20ee 100644 --- a/depends/packages/libxcb_util.mk +++ b/depends/packages/libxcb_util.mk @@ -8,7 +8,6 @@ $(package)_dependencies=libxcb define $(package)_set_vars $(package)_config_opts = --disable-shared --disable-devel-docs --without-doxygen $(package)_config_opts += --disable-dependency-tracking --enable-option-checking -$(package)_config_opts += --with-pic endef define $(package)_preprocess_cmds diff --git a/depends/packages/qrencode.mk b/depends/packages/qrencode.mk index 2afd95d7c4fae..9ebd2dd85a454 100644 --- a/depends/packages/qrencode.mk +++ b/depends/packages/qrencode.mk @@ -8,8 +8,6 @@ define $(package)_set_vars $(package)_config_opts=--disable-shared --without-tools --without-tests --without-png $(package)_config_opts += --disable-gprof --disable-gcov --disable-mudflap $(package)_config_opts += --disable-dependency-tracking --enable-option-checking -$(package)_config_opts_linux=--with-pic -$(package)_config_opts_android=--with-pic $(package)_cflags += -Wno-int-conversion -Wno-implicit-function-declaration endef diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 5608e5f07387d..450d0dbea3d5d 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -18,7 +18,6 @@ $(package)_patches += qtbase-moc-ignore-gcc-macro.patch $(package)_patches += use_android_ndk23.patch $(package)_patches += rcc_hardcode_timestamp.patch $(package)_patches += duplicate_lcqpafonts.patch -$(package)_patches += fast_fixed_dtoa_no_optimize.patch $(package)_patches += guix_cross_lib_path.patch $(package)_patches += fix-macos-linker.patch $(package)_patches += memory_resource.patch @@ -178,6 +177,7 @@ $(package)_config_opts_mingw32 += -xplatform win32-g++ $(package)_config_opts_mingw32 += "QMAKE_CFLAGS = '$($(package)_cflags) $($(package)_cppflags)'" $(package)_config_opts_mingw32 += "QMAKE_CXX = '$($(package)_cxx)'" $(package)_config_opts_mingw32 += "QMAKE_CXXFLAGS = '$($(package)_cxxflags) $($(package)_cppflags)'" +$(package)_config_opts_mingw32 += "QMAKE_LINK = '$($(package)_cxx)'" $(package)_config_opts_mingw32 += "QMAKE_LFLAGS = '$($(package)_ldflags)'" $(package)_config_opts_mingw32 += "QMAKE_LIB = '$($(package)_ar) rc'" $(package)_config_opts_mingw32 += -device-option CROSS_COMPILE="$(host)-" @@ -251,7 +251,6 @@ define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/rcc_hardcode_timestamp.patch && \ patch -p1 -i $($(package)_patch_dir)/duplicate_lcqpafonts.patch && \ patch -p1 -i $($(package)_patch_dir)/utc_from_string_no_optimize.patch && \ - patch -p1 -i $($(package)_patch_dir)/fast_fixed_dtoa_no_optimize.patch && \ patch -p1 -i $($(package)_patch_dir)/guix_cross_lib_path.patch && \ patch -p1 -i $($(package)_patch_dir)/windows_lto.patch && \ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ diff --git a/depends/packages/sqlite.mk b/depends/packages/sqlite.mk index 6809b391139fb..7d175ec4bb3d0 100644 --- a/depends/packages/sqlite.mk +++ b/depends/packages/sqlite.mk @@ -7,10 +7,6 @@ $(package)_sha256_hash=5af07de982ba658fd91a03170c945f99c971f6955bc79df3266544373 define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-readline --disable-dynamic-extensions --enable-option-checking $(package)_config_opts+= --disable-rtree --disable-fts4 --disable-fts5 -$(package)_config_opts_linux=--with-pic -$(package)_config_opts_freebsd=--with-pic -$(package)_config_opts_netbsd=--with-pic -$(package)_config_opts_openbsd=--with-pic # We avoid using `--enable-debug` because it overrides CFLAGS, a behavior we want to prevent. $(package)_cflags_debug += -g $(package)_cppflags_debug += -DSQLITE_DEBUG diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index cc78999dbbeb9..bfa5e97c60498 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -11,11 +11,6 @@ define $(package)_set_vars $(package)_config_opts += --without-libsodium --without-libgssapi_krb5 --without-pgm --without-norm --without-vmci $(package)_config_opts += --disable-libunwind --disable-radix-tree --without-gcov --disable-dependency-tracking $(package)_config_opts += --disable-Werror --disable-drafts --enable-option-checking - $(package)_config_opts_linux=--with-pic - $(package)_config_opts_freebsd=--with-pic - $(package)_config_opts_netbsd=--with-pic - $(package)_config_opts_openbsd=--with-pic - $(package)_config_opts_android=--with-pic endef define $(package)_preprocess_cmds diff --git a/depends/patches/qt/fast_fixed_dtoa_no_optimize.patch b/depends/patches/qt/fast_fixed_dtoa_no_optimize.patch deleted file mode 100644 index d4d6539f56dc4..0000000000000 --- a/depends/patches/qt/fast_fixed_dtoa_no_optimize.patch +++ /dev/null @@ -1,20 +0,0 @@ -Modify the optimisation flags for FastFixedDtoa. -This fixes a non-determinism issue in the asm produced for -this function when cross-compiling on x86_64 and aarch64 for -the arm-linux-gnueabihf HOST. - ---- a/qtbase/src/3rdparty/double-conversion/fixed-dtoa.h -+++ b/qtbase/src/3rdparty/double-conversion/fixed-dtoa.h -@@ -48,9 +48,12 @@ namespace double_conversion { - // - // This method only works for some parameters. If it can't handle the input it - // returns false. The output is null-terminated when the function succeeds. -+#pragma GCC push_options -+#pragma GCC optimize ("-O1") - bool FastFixedDtoa(double v, int fractional_count, - Vector buffer, int* length, int* decimal_point); - -+#pragma GCC pop_options - } // namespace double_conversion - - #endif // DOUBLE_CONVERSION_FIXED_DTOA_H_ diff --git a/doc/dependencies.md b/doc/dependencies.md index 4eff7efd6100d..b749db642a620 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -9,7 +9,7 @@ You can find installation instructions in the `build-*.md` file for your platfor | [Autoconf](https://www.gnu.org/software/autoconf/) | [2.69](https://github.com/bitcoin/bitcoin/pull/17769) | | [Automake](https://www.gnu.org/software/automake/) | [1.13](https://github.com/bitcoin/bitcoin/pull/18290) | | [Clang](https://clang.llvm.org) | [14.0](https://github.com/bitcoin/bitcoin/pull/29208) | -| [GCC](https://gcc.gnu.org) | [10.1](https://github.com/bitcoin/bitcoin/pull/28348) | +| [GCC](https://gcc.gnu.org) | [11.1](https://github.com/bitcoin/bitcoin/pull/29091) | | [Python](https://www.python.org) (scripts, tests) | [3.9](https://github.com/bitcoin/bitcoin/pull/28211) | | [systemtap](https://sourceware.org/systemtap/) ([tracing](tracing.md))| N/A | diff --git a/doc/design/assumeutxo.md b/doc/design/assumeutxo.md index 6fabb53aed5cd..cac0b4da947ac 100644 --- a/doc/design/assumeutxo.md +++ b/doc/design/assumeutxo.md @@ -16,7 +16,7 @@ load it. A pruned node can load a snapshot. To save space, it's possible to delete the snapshot file as soon as `loadtxoutset` finishes. -The minimum `-dbcache` setting is 550 MiB, but this functionality ignores that +The minimum `-prune` setting is 550 MiB, but this functionality ignores that minimum and uses at least 1100 MiB. As the background sync continues there will be temporarily two chainstate @@ -51,18 +51,12 @@ The utility script ## Design notes -- A new block index `nStatus` flag is introduced, `BLOCK_ASSUMED_VALID`, to mark block - index entries that are required to be assumed-valid by a chainstate created - from a UTXO snapshot. This flag is used as a way to modify certain - CheckBlockIndex() logic to account for index entries that are pending validation by a - chainstate running asynchronously in the background. - - The concept of UTXO snapshots is treated as an implementation detail that lives behind the ChainstateManager interface. The external presentation of the changes required to facilitate the use of UTXO snapshots is the understanding that there are - now certain regions of the chain that can be temporarily assumed to be valid (using - the nStatus flag mentioned above). In certain cases, e.g. wallet rescanning, this is - very similar to dealing with a pruned chain. + now certain regions of the chain that can be temporarily assumed to be valid. + In certain cases, e.g. wallet rescanning, this is very similar to dealing with + a pruned chain. Logic outside ChainstateManager should try not to know about snapshots, instead preferring to work in terms of more general states like assumed-valid. diff --git a/src/.clang-tidy b/src/.clang-tidy index bfaa5ab8e7b09..e4b789dcaa908 100644 --- a/src/.clang-tidy +++ b/src/.clang-tidy @@ -12,6 +12,7 @@ modernize-use-noexcept, modernize-use-nullptr, performance-*, -performance-avoid-endl, +-performance-enum-size, -performance-inefficient-string-concatenation, -performance-no-int-to-ptr, -performance-noexcept-move-constructor, diff --git a/src/Makefile.am b/src/Makefile.am index 0f1af37b8094d..bb78414061a83 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -305,6 +305,7 @@ BITCOIN_CORE_H = \ util/error.h \ util/exception.h \ util/fastrange.h \ + util/feefrac.h \ util/fees.h \ util/fs.h \ util/fs_helpers.h \ @@ -747,6 +748,7 @@ libgroestlcoin_util_a_SOURCES = \ util/check.cpp \ util/error.cpp \ util/exception.cpp \ + util/feefrac.cpp \ util/fees.cpp \ util/fs.cpp \ util/fs_helpers.cpp \ @@ -992,6 +994,7 @@ libgroestlcoinkernel_la_SOURCES = \ util/batchpriority.cpp \ util/chaintype.cpp \ util/check.cpp \ + util/feefrac.cpp \ util/fs.cpp \ util/fs_helpers.cpp \ util/hasher.cpp \ diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 4d814bc5dc92b..7ba0111fa686f 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -34,6 +34,7 @@ bench_bench_bitcoin_SOURCES = \ bench/examples.cpp \ bench/gcs_filter.cpp \ bench/hashpadding.cpp \ + bench/index_blockfilter.cpp \ bench/load_external.cpp \ bench/lockedpool.cpp \ bench/logging.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 019101fb35d57..8bcab97562861 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -93,6 +93,7 @@ BITCOIN_TESTS =\ test/denialofservice_tests.cpp \ test/descriptor_tests.cpp \ test/disconnected_transactions.cpp \ + test/feefrac_tests.cpp \ test/flatfile_tests.cpp \ test/fs_tests.cpp \ test/getarg_tests.cpp \ @@ -313,7 +314,9 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/descriptor_parse.cpp \ test/fuzz/deserialize.cpp \ test/fuzz/eval_script.cpp \ + test/fuzz/feefrac.cpp \ test/fuzz/fee_rate.cpp \ + test/fuzz/feeratediagram.cpp \ test/fuzz/fees.cpp \ test/fuzz/flatfile.cpp \ test/fuzz/float.cpp \ diff --git a/src/addrdb.cpp b/src/addrdb.cpp index f8d4240f3fe9b..14dc314c3650f 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -195,7 +195,7 @@ util::Result> LoadAddrman(const NetGroupManager& netgro auto check_addrman = std::clamp(args.GetIntArg("-checkaddrman", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), 0, 1000000); bool deterministic = HasTestOption(args, "addrman"); // use a deterministic addrman only for tests - auto addrman{std::make_unique(netgroupman, /*deterministic=*/deterministic, /*consistency_check_ratio=*/check_addrman)}; + auto addrman{std::make_unique(netgroupman, deterministic, /*consistency_check_ratio=*/check_addrman)}; const auto start{SteadyClock::now()}; const auto path_addr{args.GetDataDirNet() / "peers.dat"}; @@ -204,7 +204,7 @@ util::Result> LoadAddrman(const NetGroupManager& netgro LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman->Size(), Ticks(SteadyClock::now() - start)); } catch (const DbNotFoundError&) { // Addrman can be in an inconsistent state after failure, reset it - addrman = std::make_unique(netgroupman, /*deterministic=*/deterministic, /*consistency_check_ratio=*/check_addrman); + addrman = std::make_unique(netgroupman, deterministic, /*consistency_check_ratio=*/check_addrman); LogPrintf("Creating peers.dat because the file was not found (%s)\n", fs::quoted(fs::PathToString(path_addr))); DumpPeerAddresses(args, *addrman); } catch (const InvalidAddrManVersionError&) { @@ -212,7 +212,7 @@ util::Result> LoadAddrman(const NetGroupManager& netgro return util::Error{strprintf(_("Failed to rename invalid peers.dat file. Please move or delete it and try again."))}; } // Addrman can be in an inconsistent state after failure, reset it - addrman = std::make_unique(netgroupman, /*deterministic=*/deterministic, /*consistency_check_ratio=*/check_addrman); + addrman = std::make_unique(netgroupman, deterministic, /*consistency_check_ratio=*/check_addrman); LogPrintf("Creating new peers.dat because the file version was not compatible (%s). Original backed up to peers.dat.bak\n", fs::quoted(fs::PathToString(path_addr))); DumpPeerAddresses(args, *addrman); } catch (const std::exception& e) { diff --git a/src/bench/index_blockfilter.cpp b/src/bench/index_blockfilter.cpp new file mode 100644 index 0000000000000..5e0bfbfea6b3b --- /dev/null +++ b/src/bench/index_blockfilter.cpp @@ -0,0 +1,43 @@ +// Copyright (c) 2023-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include +#include +#include +#include +#include + +// Very simple block filter index sync benchmark, only using coinbase outputs. +static void BlockFilterIndexSync(benchmark::Bench& bench) +{ + const auto test_setup = MakeNoLogFileContext(); + + // Create more blocks + int CHAIN_SIZE = 600; + CPubKey pubkey{ParseHex("02ed26169896db86ced4cbb7b3ecef9859b5952825adbeab998fb5b307e54949c9")}; + CScript script = GetScriptForDestination(WitnessV0KeyHash(pubkey)); + std::vector noTxns; + for (int i = 0; i < CHAIN_SIZE - 100; i++) { + test_setup->CreateAndProcessBlock(noTxns, script); + SetMockTime(GetTime() + 1); + } + assert(WITH_LOCK(::cs_main, return test_setup->m_node.chainman->ActiveHeight() == CHAIN_SIZE)); + + bench.minEpochIterations(5).run([&] { + BlockFilterIndex filter_index(interfaces::MakeChain(test_setup->m_node), BlockFilterType::BASIC, + /*n_cache_size=*/0, /*f_memory=*/false, /*f_wipe=*/true); + assert(filter_index.Init()); + assert(!filter_index.BlockUntilSyncedToCurrentChain()); + filter_index.Sync(); + + IndexSummary summary = filter_index.GetSummary(); + assert(summary.synced); + assert(summary.best_block_hash == WITH_LOCK(::cs_main, return test_setup->m_node.chainman->ActiveTip()->GetBlockHash())); + }); +} + +BENCHMARK(BlockFilterIndexSync, benchmark::PriorityLevel::HIGH); diff --git a/src/chain.h b/src/chain.h index fa165a4aa732e..bb70dbd8bcd68 100644 --- a/src/chain.h +++ b/src/chain.h @@ -98,16 +98,20 @@ enum BlockStatus : uint32_t { /** * Only first tx is coinbase, 2 <= coinbase input script length <= 100, transactions valid, no duplicate txids, - * sigops, size, merkle root. Implies all parents are at least TREE but not necessarily TRANSACTIONS. When all - * parent blocks also have TRANSACTIONS, CBlockIndex::nChainTx will be set. + * sigops, size, merkle root. Implies all parents are at least TREE but not necessarily TRANSACTIONS. + * + * If a block's validity is at least VALID_TRANSACTIONS, CBlockIndex::nTx will be set. If a block and all previous + * blocks back to the genesis block or an assumeutxo snapshot block are at least VALID_TRANSACTIONS, + * CBlockIndex::nChainTx will be set. */ BLOCK_VALID_TRANSACTIONS = 3, //! Outputs do not overspend inputs, no double spends, coinbase output ok, no immature coinbase spends, BIP30. - //! Implies all parents are either at least VALID_CHAIN, or are ASSUMED_VALID + //! Implies all previous blocks back to the genesis block or an assumeutxo snapshot block are at least VALID_CHAIN. BLOCK_VALID_CHAIN = 4, - //! Scripts & signatures ok. Implies all parents are either at least VALID_SCRIPTS, or are ASSUMED_VALID. + //! Scripts & signatures ok. Implies all previous blocks back to the genesis block or an assumeutxo snapshot block + //! are at least VALID_SCRIPTS. BLOCK_VALID_SCRIPTS = 5, //! All validity bits. @@ -124,21 +128,8 @@ enum BlockStatus : uint32_t { BLOCK_OPT_WITNESS = 128, //!< block data in blk*.dat was received with a witness-enforcing client - /** - * If ASSUMED_VALID is set, it means that this block has not been validated - * and has validity status less than VALID_SCRIPTS. Also that it may have - * descendant blocks with VALID_SCRIPTS set, because they can be validated - * based on an assumeutxo snapshot. - * - * When an assumeutxo snapshot is loaded, the ASSUMED_VALID flag is added to - * unvalidated blocks at the snapshot height and below. Then, as the background - * validation progresses, and these blocks are validated, the ASSUMED_VALID - * flags are removed. See `doc/design/assumeutxo.md` for details. - * - * This flag is only used to implement checks in CheckBlockIndex() and - * should not be used elsewhere. - */ - BLOCK_ASSUMED_VALID = 256, + BLOCK_STATUS_RESERVED = 256, //!< Unused flag that was previously set on assumeutxo snapshot blocks and their + //!< ancestors before they were validated, and unset when they were validated. }; /** The block chain is a tree shaped structure starting with the @@ -173,21 +164,16 @@ class CBlockIndex //! (memory only) Total amount of work (expected number of hashes) in the chain up to and including this block arith_uint256 nChainWork{}; - //! Number of transactions in this block. + //! Number of transactions in this block. This will be nonzero if the block + //! reached the VALID_TRANSACTIONS level, and zero otherwise. //! Note: in a potential headers-first mode, this number cannot be relied upon - //! Note: this value is faked during UTXO snapshot load to ensure that - //! LoadBlockIndex() will load index entries for blocks that we lack data for. - //! @sa ActivateSnapshot unsigned int nTx{0}; //! (memory only) Number of transactions in the chain up to and including this block. - //! This value will be non-zero only if and only if transactions for this block and all its parents are available. + //! This value will be non-zero if this block and all previous blocks back + //! to the genesis block or an assumeutxo snapshot block have reached the + //! VALID_TRANSACTIONS level. //! Change to 64-bit type before 2024 (assuming worst case of 60 byte transactions). - //! - //! Note: this value is faked during use of a UTXO snapshot because we don't - //! have the underlying block data available during snapshot load. - //! @sa AssumeutxoData - //! @sa ActivateSnapshot unsigned int nChainTx{0}; //! Verification status of this block. See enum BlockStatus @@ -262,15 +248,14 @@ class CBlockIndex } /** - * Check whether this block's and all previous blocks' transactions have been - * downloaded (and stored to disk) at some point. + * Check whether this block and all previous blocks back to the genesis block or an assumeutxo snapshot block have + * reached VALID_TRANSACTIONS and had transactions downloaded (and stored to disk) at some point. * * Does not imply the transactions are consensus-valid (ConnectTip might fail) * Does not imply the transactions are still stored on disk. (IsBlockPruned might return true) * - * Note that this will be true for the snapshot base block, if one is loaded (and - * all subsequent assumed-valid blocks) since its nChainTx value will have been set - * manually based on the related AssumeutxoData entry. + * Note that this will be true for the snapshot base block, if one is loaded, since its nChainTx value will have + * been set manually based on the related AssumeutxoData entry. */ bool HaveNumChainTxs() const { return nChainTx != 0; } @@ -318,14 +303,6 @@ class CBlockIndex return ((nStatus & BLOCK_VALID_MASK) >= nUpTo); } - //! @returns true if the block is assumed-valid; this means it is queued to be - //! validated by a background chainstate. - bool IsAssumedValid() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main) - { - AssertLockHeld(::cs_main); - return nStatus & BLOCK_ASSUMED_VALID; - } - //! Raise the validity level of this block index entry. //! Returns true if the validity was changed. bool RaiseValidity(enum BlockStatus nUpTo) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) @@ -335,12 +312,6 @@ class CBlockIndex if (nStatus & BLOCK_FAILED_MASK) return false; if ((nStatus & BLOCK_VALID_MASK) < nUpTo) { - // If this block had been marked assumed-valid and we're raising - // its validity to a certain point, there is no longer an assumption. - if (nStatus & BLOCK_ASSUMED_VALID && nUpTo >= BLOCK_VALID_SCRIPTS) { - nStatus &= ~BLOCK_ASSUMED_VALID; - } - nStatus = (nStatus & ~BLOCK_VALID_MASK) | nUpTo; return true; } diff --git a/src/groestlcoin-chainstate.cpp b/src/groestlcoin-chainstate.cpp index 3eb64aa344b13..642af06e82d2c 100644 --- a/src/groestlcoin-chainstate.cpp +++ b/src/groestlcoin-chainstate.cpp @@ -89,14 +89,13 @@ int main(int argc, char* argv[]) { std::cout << "Warning: " << warning.original << std::endl; } - void flushError(const std::string& debug_message) override + void flushError(const bilingual_str& message) override { - std::cerr << "Error flushing block data to disk: " << debug_message << std::endl; + std::cerr << "Error flushing block data to disk: " << message.original << std::endl; } - void fatalError(const std::string& debug_message, const bilingual_str& user_message) override + void fatalError(const bilingual_str& message) override { - std::cerr << "Error: " << debug_message << std::endl; - std::cerr << (user_message.empty() ? "A fatal internal error occurred." : user_message.original) << std::endl; + std::cerr << "Error: " << message.original << std::endl; } }; auto notifications = std::make_unique(); diff --git a/src/index/base.cpp b/src/index/base.cpp index 036292cd8a11c..13d8ba5a01095 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -31,7 +31,7 @@ template void BaseIndex::FatalErrorf(const char* fmt, const Args&... args) { auto message = tfm::format(fmt, args...); - node::AbortNode(m_chain->context()->shutdown, m_chain->context()->exit_status, message); + node::AbortNode(m_chain->context()->shutdown, m_chain->context()->exit_status, Untranslated(message)); } CBlockLocator GetLocator(interfaces::Chain& chain, const uint256& block_hash) @@ -141,7 +141,7 @@ static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev, CChain& return chain.Next(chain.FindFork(pindex_prev)); } -void BaseIndex::ThreadSync() +void BaseIndex::Sync() { const CBlockIndex* pindex = m_best_block_index.load(); if (!m_synced) { @@ -159,37 +159,20 @@ void BaseIndex::ThreadSync() return; } - { - LOCK(cs_main); - const CBlockIndex* pindex_next = NextSyncBlock(pindex, m_chainstate->m_chain); - if (!pindex_next) { - SetBestBlockIndex(pindex); - m_synced = true; - // No need to handle errors in Commit. See rationale above. - Commit(); - break; - } - if (pindex_next->pprev != pindex && !Rewind(pindex, pindex_next->pprev)) { - FatalErrorf("%s: Failed to rewind index %s to a previous chain tip", - __func__, GetName()); - return; - } - pindex = pindex_next; - } - - auto current_time{std::chrono::steady_clock::now()}; - if (last_log_time + SYNC_LOG_INTERVAL < current_time) { - LogPrintf("Syncing %s with block chain from height %d\n", - GetName(), pindex->nHeight); - last_log_time = current_time; - } - - if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) { - SetBestBlockIndex(pindex->pprev); - last_locator_write_time = current_time; + const CBlockIndex* pindex_next = WITH_LOCK(cs_main, return NextSyncBlock(pindex, m_chainstate->m_chain)); + if (!pindex_next) { + SetBestBlockIndex(pindex); + m_synced = true; // No need to handle errors in Commit. See rationale above. Commit(); + break; } + if (pindex_next->pprev != pindex && !Rewind(pindex, pindex_next->pprev)) { + FatalErrorf("%s: Failed to rewind index %s to a previous chain tip", __func__, GetName()); + return; + } + pindex = pindex_next; + CBlock block; interfaces::BlockInfo block_info = kernel::MakeBlockInfo(pindex); @@ -205,6 +188,20 @@ void BaseIndex::ThreadSync() __func__, pindex->GetBlockHash().ToString()); return; } + + auto current_time{std::chrono::steady_clock::now()}; + if (last_log_time + SYNC_LOG_INTERVAL < current_time) { + LogPrintf("Syncing %s with block chain from height %d\n", + GetName(), pindex->nHeight); + last_log_time = current_time; + } + + if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) { + SetBestBlockIndex(pindex); + last_locator_write_time = current_time; + // No need to handle errors in Commit. See rationale above. + Commit(); + } } } @@ -394,7 +391,7 @@ bool BaseIndex::StartBackgroundSync() { if (!m_init) throw std::logic_error("Error: Cannot start a non-initialized index"); - m_thread_sync = std::thread(&util::TraceThread, GetName(), [this] { ThreadSync(); }); + m_thread_sync = std::thread(&util::TraceThread, GetName(), [this] { Sync(); }); return true; } diff --git a/src/index/base.h b/src/index/base.h index 154061fb19120..0eb1d9ca3b229 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -78,13 +78,6 @@ class BaseIndex : public CValidationInterface std::thread m_thread_sync; CThreadInterrupt m_interrupt; - /// Sync the index with the block index starting from the current best block. - /// Intended to be run in its own thread, m_thread_sync, and can be - /// interrupted with m_interrupt. Once the index gets in sync, the m_synced - /// flag is set and the BlockConnected ValidationInterface callback takes - /// over and the sync thread exits. - void ThreadSync(); - /// Write the current index state (eg. chain block locator and subclass-specific items) to disk. /// /// Recommendations for error handling: @@ -152,9 +145,16 @@ class BaseIndex : public CValidationInterface /// validation interface so that it stays in sync with blockchain updates. [[nodiscard]] bool Init(); - /// Starts the initial sync process. + /// Starts the initial sync process on a background thread. [[nodiscard]] bool StartBackgroundSync(); + /// Sync the index with the block index starting from the current best block. + /// Intended to be run in its own thread, m_thread_sync, and can be + /// interrupted with m_interrupt. Once the index gets in sync, the m_synced + /// flag is set and the BlockConnected ValidationInterface callback takes + /// over and the sync thread exits. + void Sync(); + /// Stops the instance from staying in sync with blockchain updates. void Stop(); diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index 65993e830e54e..41bdca9df562a 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -128,6 +128,16 @@ bool BlockFilterIndex::CustomInit(const std::optional& blo m_next_filter_pos.nFile = 0; m_next_filter_pos.nPos = 0; } + + if (block) { + auto op_last_header = ReadFilterHeader(block->height, block->hash); + if (!op_last_header) { + LogError("Cannot read last block filter header; index may be corrupted\n"); + return false; + } + m_last_header = *op_last_header; + } + return true; } @@ -222,10 +232,25 @@ size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& return data_size; } +std::optional BlockFilterIndex::ReadFilterHeader(int height, const uint256& expected_block_hash) +{ + std::pair read_out; + if (!m_db->Read(DBHeightKey(height), read_out)) { + return std::nullopt; + } + + if (read_out.first != expected_block_hash) { + LogError("%s: previous block header belongs to unexpected block %s; expected %s\n", + __func__, read_out.first.ToString(), expected_block_hash.ToString()); + return std::nullopt; + } + + return read_out.second.header; +} + bool BlockFilterIndex::CustomAppend(const interfaces::BlockInfo& block) { CBlockUndo block_undo; - uint256 prev_header; if (block.height > 0) { // pindex variable gives indexing code access to node internals. It @@ -234,34 +259,28 @@ bool BlockFilterIndex::CustomAppend(const interfaces::BlockInfo& block) if (!m_chainstate->m_blockman.UndoReadFromDisk(block_undo, *pindex)) { return false; } - - std::pair read_out; - if (!m_db->Read(DBHeightKey(block.height - 1), read_out)) { - return false; - } - - uint256 expected_block_hash = *Assert(block.prev_hash); - if (read_out.first != expected_block_hash) { - LogError("%s: previous block header belongs to unexpected block %s; expected %s\n", - __func__, read_out.first.ToString(), expected_block_hash.ToString()); - return false; - } - - prev_header = read_out.second.header; } BlockFilter filter(m_filter_type, *Assert(block.data), block_undo); + const uint256& header = filter.ComputeHeader(m_last_header); + bool res = Write(filter, block.height, header); + if (res) m_last_header = header; // update last header + return res; +} + +bool BlockFilterIndex::Write(const BlockFilter& filter, uint32_t block_height, const uint256& filter_header) +{ size_t bytes_written = WriteFilterToDisk(m_next_filter_pos, filter); if (bytes_written == 0) return false; std::pair value; - value.first = block.hash; + value.first = filter.GetBlockHash(); value.second.hash = filter.GetHash(); - value.second.header = filter.ComputeHeader(prev_header); + value.second.header = filter_header; value.second.pos = m_next_filter_pos; - if (!m_db->Write(DBHeightKey(block.height), value)) { + if (!m_db->Write(DBHeightKey(block_height), value)) { return false; } @@ -315,6 +334,8 @@ bool BlockFilterIndex::CustomRewind(const interfaces::BlockKey& current_tip, con batch.Write(DB_FILTER_POS, m_next_filter_pos); if (!m_db->WriteBatch(batch)) return false; + // Update cached header + m_last_header = *Assert(ReadFilterHeader(new_tip.height, new_tip.hash)); return true; } diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h index 10a1cfd2ee0e5..cdb9563fb8ec8 100644 --- a/src/index/blockfilterindex.h +++ b/src/index/blockfilterindex.h @@ -42,8 +42,15 @@ class BlockFilterIndex final : public BaseIndex /** cache of block hash to filter header, to avoid disk access when responding to getcfcheckpt. */ std::unordered_map m_headers_cache GUARDED_BY(m_cs_headers_cache); + // Last computed header to avoid disk reads on every new block. + uint256 m_last_header{}; + bool AllowPrune() const override { return true; } + bool Write(const BlockFilter& filter, uint32_t block_height, const uint256& filter_header); + + std::optional ReadFilterHeader(int height, const uint256& expected_block_hash); + protected: bool CustomInit(const std::optional& block) override; diff --git a/src/init.cpp b/src/init.cpp index 3215705245a9e..e2a5fa94e91ea 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -256,12 +256,8 @@ void Interrupt(NodeContext& node) InterruptMapPort(); if (node.connman) node.connman->Interrupt(); - if (g_txindex) { - g_txindex->Interrupt(); - } - ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Interrupt(); }); - if (g_coin_stats_index) { - g_coin_stats_index->Interrupt(); + for (auto* index : node.indexes) { + index->Interrupt(); } } @@ -337,16 +333,11 @@ void Shutdown(NodeContext& node) if (node.validation_signals) node.validation_signals->FlushBackgroundCallbacks(); // Stop and delete all indexes only after flushing background callbacks. - if (g_txindex) { - g_txindex->Stop(); - g_txindex.reset(); - } - if (g_coin_stats_index) { - g_coin_stats_index->Stop(); - g_coin_stats_index.reset(); - } - ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Stop(); }); + for (auto* index : node.indexes) index->Stop(); + if (g_txindex) g_txindex.reset(); + if (g_coin_stats_index) g_coin_stats_index.reset(); DestroyAllBlockFilterIndexes(); + node.indexes.clear(); // all instances are nullptr now // Any future callbacks will be dropped. This should absolutely be safe - if // missing a callback results in an unrecoverable situation, unclean shutdown @@ -1040,7 +1031,7 @@ bool AppInitParameterInteraction(const ArgsManager& args) if (args.IsArgSet("-test")) { if (chainparams.GetChainType() != ChainType::REGTEST) { - return InitError(Untranslated("-test=