diff --git a/contrib/devtools/clang-format-diff.py b/contrib/devtools/clang-format-diff.py index 420bf7ff33002..e2b661d65d235 100755 --- a/contrib/devtools/clang-format-diff.py +++ b/contrib/devtools/clang-format-diff.py @@ -1,166 +1,190 @@ #!/usr/bin/env python3 # -#===- clang-format-diff.py - ClangFormat Diff Reformatter ----*- python -*--===# +# ===- clang-format-diff.py - ClangFormat Diff Reformatter ----*- python -*--===# # -# The LLVM Compiler Infrastructure +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # -# This file is distributed under the University of Illinois Open Source -# License. -# -# ============================================================ -# -# University of Illinois/NCSA -# Open Source License -# -# Copyright (c) 2007-2015 University of Illinois at Urbana-Champaign. -# All rights reserved. -# -# Developed by: -# -# LLVM Team -# -# University of Illinois at Urbana-Champaign -# -# http://llvm.org -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of -# this software and associated documentation files (the "Software"), to deal with -# the Software without restriction, including without limitation the rights to -# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -# of the Software, and to permit persons to whom the Software is furnished to do -# so, subject to the following conditions: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimers. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimers in the -# documentation and/or other materials provided with the distribution. -# -# * Neither the names of the LLVM Team, University of Illinois at -# Urbana-Champaign, nor the names of its contributors may be used to -# endorse or promote products derived from this Software without specific -# prior written permission. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE -# SOFTWARE. -# -# ============================================================ -# -#===------------------------------------------------------------------------===# - -r""" -ClangFormat Diff Reformatter -============================ +# ===------------------------------------------------------------------------===# +""" This script reads input from a unified diff and reformats all the changed lines. This is useful to reformat all the lines touched by a specific patch. Example usage for git/svn users: - git diff -U0 HEAD^ | clang-format-diff.py -p1 -i - svn diff --diff-cmd=diff -x-U0 | clang-format-diff.py -i + git diff -U0 --no-color --relative HEAD^ | {clang_format_diff} -p1 -i + svn diff --diff-cmd=diff -x-U0 | {clang_format_diff} -i +It should be noted that the filename contained in the diff is used unmodified +to determine the source file to update. Users calling this script directly +should be careful to ensure that the path in the diff is correct relative to the +current working directory. """ +from __future__ import absolute_import, division, print_function import argparse import difflib -import io import re import subprocess import sys - -# Change this to the full path if clang-format is not on the path. -binary = 'clang-format' +from io import StringIO def main(): - parser = argparse.ArgumentParser(description= - 'Reformat changed lines in diff. Without -i ' - 'option just output the diff that would be ' - 'introduced.') - parser.add_argument('-i', action='store_true', default=False, - help='apply edits to files instead of displaying a diff') - parser.add_argument('-p', metavar='NUM', default=0, - help='strip the smallest prefix containing P slashes') - parser.add_argument('-regex', metavar='PATTERN', default=None, - help='custom pattern selecting file paths to reformat ' - '(case sensitive, overrides -iregex)') - parser.add_argument('-iregex', metavar='PATTERN', default= - r'.*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc|js|ts|proto' - r'|protodevel|java)', - help='custom pattern selecting file paths to reformat ' - '(case insensitive, overridden by -regex)') - parser.add_argument('-sort-includes', action='store_true', default=False, - help='let clang-format sort include blocks') - parser.add_argument('-v', '--verbose', action='store_true', - help='be more verbose, ineffective without -i') - args = parser.parse_args() - - # Extract changed lines for each file. - filename = None - lines_by_file = {} - for line in sys.stdin: - match = re.search(r'^\+\+\+\ (.*?/){%s}(\S*)' % args.p, line) - if match: - filename = match.group(2) - if filename is None: - continue - - if args.regex is not None: - if not re.match('^%s$' % args.regex, filename): - continue - else: - if not re.match('^%s$' % args.iregex, filename, re.IGNORECASE): - continue - - match = re.search(r'^@@.*\+(\d+)(,(\d+))?', line) - if match: - start_line = int(match.group(1)) - line_count = 1 - if match.group(3): - line_count = int(match.group(3)) - if line_count == 0: - continue - end_line = start_line + line_count - 1 - lines_by_file.setdefault(filename, []).extend( - ['-lines', str(start_line) + ':' + str(end_line)]) - - # Reformat files containing changes in place. - for filename, lines in lines_by_file.items(): - if args.i and args.verbose: - print('Formatting {}'.format(filename)) - command = [binary, filename] - if args.i: - command.append('-i') - if args.sort_includes: - command.append('-sort-includes') - command.extend(lines) - command.extend(['-style=file', '-fallback-style=none']) - p = subprocess.Popen(command, - stdout=subprocess.PIPE, - stderr=None, - stdin=subprocess.PIPE, - text=True) - stdout, stderr = p.communicate() - if p.returncode != 0: - sys.exit(p.returncode) - - if not args.i: - with open(filename, encoding="utf8") as f: - code = f.readlines() - formatted_code = io.StringIO(stdout).readlines() - diff = difflib.unified_diff(code, formatted_code, - filename, filename, - '(before formatting)', '(after formatting)') - diff_string = ''.join(diff) - if len(diff_string) > 0: - sys.stdout.write(diff_string) - -if __name__ == '__main__': - main() + parser = argparse.ArgumentParser( + description=__doc__.format(clang_format_diff="%(prog)s"), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "-i", + action="store_true", + default=False, + help="apply edits to files instead of displaying a diff", + ) + parser.add_argument( + "-p", + metavar="NUM", + default=0, + help="strip the smallest prefix containing P slashes", + ) + parser.add_argument( + "-regex", + metavar="PATTERN", + default=None, + help="custom pattern selecting file paths to reformat " + "(case sensitive, overrides -iregex)", + ) + parser.add_argument( + "-iregex", + metavar="PATTERN", + default=r".*\.(?:cpp|cc|c\+\+|cxx|cppm|ccm|cxxm|c\+\+m|c|cl|h|hh|hpp" + r"|hxx|m|mm|inc|js|ts|proto|protodevel|java|cs|json|s?vh?)", + help="custom pattern selecting file paths to reformat " + "(case insensitive, overridden by -regex)", + ) + parser.add_argument( + "-sort-includes", + action="store_true", + default=False, + help="let clang-format sort include blocks", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="be more verbose, ineffective without -i", + ) + parser.add_argument( + "-style", + help="formatting style to apply (LLVM, GNU, Google, Chromium, " + "Microsoft, Mozilla, WebKit)", + ) + parser.add_argument( + "-fallback-style", + help="The name of the predefined style used as a" + "fallback in case clang-format is invoked with" + "-style=file, but can not find the .clang-format" + "file to use.", + ) + parser.add_argument( + "-binary", + default="clang-format", + help="location of binary to use for clang-format", + ) + args = parser.parse_args() + + # Extract changed lines for each file. + filename = None + lines_by_file = {} + for line in sys.stdin: + match = re.search(r"^\+\+\+\ (.*?/){%s}(\S*)" % args.p, line) + if match: + filename = match.group(2) + if filename is None: + continue + + if args.regex is not None: + if not re.match("^%s$" % args.regex, filename): + continue + else: + if not re.match("^%s$" % args.iregex, filename, re.IGNORECASE): + continue + + match = re.search(r"^@@.*\+(\d+)(?:,(\d+))?", line) + if match: + start_line = int(match.group(1)) + line_count = 1 + if match.group(2): + line_count = int(match.group(2)) + # The input is something like + # + # @@ -1, +0,0 @@ + # + # which means no lines were added. + if line_count == 0: + continue + # Also format lines range if line_count is 0 in case of deleting + # surrounding statements. + end_line = start_line + if line_count != 0: + end_line += line_count - 1 + lines_by_file.setdefault(filename, []).extend( + ["-lines", str(start_line) + ":" + str(end_line)] + ) + + # Reformat files containing changes in place. + for filename, lines in lines_by_file.items(): + if args.i and args.verbose: + print("Formatting {}".format(filename)) + command = [args.binary, filename] + if args.i: + command.append("-i") + if args.sort_includes: + command.append("-sort-includes") + command.extend(lines) + if args.style: + command.extend(["-style", args.style]) + if args.fallback_style: + command.extend(["-fallback-style", args.fallback_style]) + + try: + p = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=None, + stdin=subprocess.PIPE, + universal_newlines=True, + ) + except OSError as e: + # Give the user more context when clang-format isn't + # found/isn't executable, etc. + raise RuntimeError( + 'Failed to run "%s" - %s"' % (" ".join(command), e.strerror) + ) + + stdout, stderr = p.communicate() + if p.returncode != 0: + sys.exit(p.returncode) + + if not args.i: + with open(filename, encoding="utf8") as f: + code = f.readlines() + formatted_code = StringIO(stdout).readlines() + diff = difflib.unified_diff( + code, + formatted_code, + filename, + filename, + "(before formatting)", + "(after formatting)", + ) + diff_string = "".join(diff) + if len(diff_string) > 0: + sys.stdout.write(diff_string) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/depends/Makefile b/depends/Makefile index e8842b3f1e73b..88aae7ad81495 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -147,8 +147,8 @@ include packages/packages.mk # 2. Before including packages/*.mk (excluding packages/packages.mk), since # they rely on the build_id variables # -build_id:=$(shell env CC='$(build_CC)' C_STANDARD='$(C_STANDARD)' CXX='$(build_CXX)' CXX_STANDARD='$(CXX_STANDARD)' AR='$(build_AR)' RANLIB='$(build_RANLIB)' STRIP='$(build_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' LTO='$(LTO)' NO_HARDEN='$(NO_HARDEN)' ./gen_id '$(BUILD_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') -$(host_arch)_$(host_os)_id:=$(shell env CC='$(host_CC)' C_STANDARD='$(C_STANDARD)' CXX='$(host_CXX)' CXX_STANDARD='$(CXX_STANDARD)' AR='$(host_AR)' RANLIB='$(host_RANLIB)' STRIP='$(host_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' LTO='$(LTO)' NO_HARDEN='$(NO_HARDEN)' ./gen_id '$(HOST_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') +build_id:=$(shell env CC='$(build_CC)' C_STANDARD='$(C_STANDARD)' CXX='$(build_CXX)' CXX_STANDARD='$(CXX_STANDARD)' AR='$(build_AR) 'NM='$(build_NM)' RANLIB='$(build_RANLIB)' STRIP='$(build_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' LTO='$(LTO)' NO_HARDEN='$(NO_HARDEN)' ./gen_id '$(BUILD_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') +$(host_arch)_$(host_os)_id:=$(shell env CC='$(host_CC)' C_STANDARD='$(C_STANDARD)' CXX='$(host_CXX)' CXX_STANDARD='$(CXX_STANDARD)' AR='$(host_AR)' NM='$(host_NM)' RANLIB='$(host_RANLIB)' STRIP='$(host_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' LTO='$(LTO)' NO_HARDEN='$(NO_HARDEN)' ./gen_id '$(HOST_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') boost_packages_$(NO_BOOST) = $(boost_packages) diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk index 8ed82b276df9c..eb64c97f64780 100644 --- a/depends/builders/darwin.mk +++ b/depends/builders/darwin.mk @@ -11,8 +11,8 @@ build_darwin_SHA256SUM=shasum -a 256 build_darwin_DOWNLOAD=curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o #darwin host on darwin builder. overrides darwin host preferences. -darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION) -isysroot$(shell xcrun --show-sdk-path) -darwin_CXX:=$(shell xcrun -f clang++) -mmacosx-version-min=$(OSX_MIN_VERSION) -stdlib=libc++ -isysroot$(shell xcrun --show-sdk-path) +darwin_CC=$(shell xcrun -f clang) -isysroot$(shell xcrun --show-sdk-path) +darwin_CXX:=$(shell xcrun -f clang++) -stdlib=libc++ -isysroot$(shell xcrun --show-sdk-path) darwin_AR:=$(shell xcrun -f ar) darwin_RANLIB:=$(shell xcrun -f ranlib) darwin_STRIP:=$(shell xcrun -f strip) diff --git a/depends/gen_id b/depends/gen_id index 3341310e460a8..8518b4e67442e 100755 --- a/depends/gen_id +++ b/depends/gen_id @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Usage: env [ CC=... ] [ C_STANDARD=...] [ CXX=... ] [CXX_STANDARD=...] \ -# [ AR=... ] [ RANLIB=... ] [ STRIP=... ] [ DEBUG=... ] \ +# [ AR=... ] [ NM=... ] [ RANLIB=... ] [ STRIP=... ] [ DEBUG=... ] \ # [ LTO=... ] [ NO_HARDEN=... ] ./build-id [ID_SALT]... # # Prints to stdout a SHA256 hash representing the current toolset, used by @@ -56,6 +56,11 @@ echo "ZERO_AR_DATE=${ZERO_AR_DATE}" echo "END AR" + echo "BEGIN NM" + bash -c "${NM} --version" + env | grep '^NM_' + echo "END NM" + echo "BEGIN RANLIB" bash -c "${RANLIB} --version" env | grep '^RANLIB_' diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index a77fd788356be..b94ac7d56fea2 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -79,24 +79,29 @@ $(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$( darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ -u LIBRARY_PATH \ - $(clang_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ - -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ + $(clang_prog) --target=$(host) \ + -B$(build_prefix)/bin \ -isysroot$(OSX_SDK) -nostdlibinc \ -iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ -u LIBRARY_PATH \ - $(clangxx_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ - -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ + $(clangxx_prog) --target=$(host) \ + -B$(build_prefix)/bin \ -isysroot$(OSX_SDK) -nostdlibinc \ -iwithsysroot/usr/include/c++/v1 \ -iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks -darwin_CFLAGS=-pipe -std=$(C_STANDARD) -darwin_CXXFLAGS=-pipe -std=$(CXX_STANDARD) +darwin_CFLAGS=-pipe -std=$(C_STANDARD) -mmacosx-version-min=$(OSX_MIN_VERSION) +darwin_CXXFLAGS=-pipe -std=$(CXX_STANDARD) -mmacosx-version-min=$(OSX_MIN_VERSION) darwin_LDFLAGS=-Wl,-platform_version,macos,$(OSX_MIN_VERSION),$(OSX_SDK_VERSION) +ifneq ($(build_os),darwin) +darwin_CFLAGS += -mlinker-version=$(LD64_VERSION) +darwin_CXXFLAGS += -mlinker-version=$(LD64_VERSION) +endif + darwin_release_CFLAGS=-O2 darwin_release_CXXFLAGS=$(darwin_release_CFLAGS) diff --git a/src/crypto/chacha20.cpp b/src/crypto/chacha20.cpp index a3cc87e81b310..4feed862cbcb1 100644 --- a/src/crypto/chacha20.cpp +++ b/src/crypto/chacha20.cpp @@ -11,15 +11,14 @@ #include #include +#include #include -constexpr static inline uint32_t rotl32(uint32_t v, int c) { return (v << c) | (v >> (32 - c)); } - #define QUARTERROUND(a,b,c,d) \ - a += b; d = rotl32(d ^ a, 16); \ - c += d; b = rotl32(b ^ c, 12); \ - a += b; d = rotl32(d ^ a, 8); \ - c += d; b = rotl32(b ^ c, 7); + a += b; d = std::rotl(d ^ a, 16); \ + c += d; b = std::rotl(b ^ c, 12); \ + a += b; d = std::rotl(d ^ a, 8); \ + c += d; b = std::rotl(b ^ c, 7); #define REPEAT10(a) do { {a}; {a}; {a}; {a}; {a}; {a}; {a}; {a}; {a}; {a}; } while(0) diff --git a/src/crypto/sha3.cpp b/src/crypto/sha3.cpp index 9c0c42fa7743f..770500bfe2781 100644 --- a/src/crypto/sha3.cpp +++ b/src/crypto/sha3.cpp @@ -11,15 +11,10 @@ #include #include // For std::begin and std::end. +#include #include -// Internal implementation code. -namespace -{ -uint64_t Rotl(uint64_t x, int n) { return (x << n) | (x >> (64 - n)); } -} // namespace - void KeccakF(uint64_t (&st)[25]) { static constexpr uint64_t RNDC[24] = { @@ -41,38 +36,38 @@ void KeccakF(uint64_t (&st)[25]) bc2 = st[2] ^ st[7] ^ st[12] ^ st[17] ^ st[22]; bc3 = st[3] ^ st[8] ^ st[13] ^ st[18] ^ st[23]; bc4 = st[4] ^ st[9] ^ st[14] ^ st[19] ^ st[24]; - t = bc4 ^ Rotl(bc1, 1); st[0] ^= t; st[5] ^= t; st[10] ^= t; st[15] ^= t; st[20] ^= t; - t = bc0 ^ Rotl(bc2, 1); st[1] ^= t; st[6] ^= t; st[11] ^= t; st[16] ^= t; st[21] ^= t; - t = bc1 ^ Rotl(bc3, 1); st[2] ^= t; st[7] ^= t; st[12] ^= t; st[17] ^= t; st[22] ^= t; - t = bc2 ^ Rotl(bc4, 1); st[3] ^= t; st[8] ^= t; st[13] ^= t; st[18] ^= t; st[23] ^= t; - t = bc3 ^ Rotl(bc0, 1); st[4] ^= t; st[9] ^= t; st[14] ^= t; st[19] ^= t; st[24] ^= t; + t = bc4 ^ std::rotl(bc1, 1); st[0] ^= t; st[5] ^= t; st[10] ^= t; st[15] ^= t; st[20] ^= t; + t = bc0 ^ std::rotl(bc2, 1); st[1] ^= t; st[6] ^= t; st[11] ^= t; st[16] ^= t; st[21] ^= t; + t = bc1 ^ std::rotl(bc3, 1); st[2] ^= t; st[7] ^= t; st[12] ^= t; st[17] ^= t; st[22] ^= t; + t = bc2 ^ std::rotl(bc4, 1); st[3] ^= t; st[8] ^= t; st[13] ^= t; st[18] ^= t; st[23] ^= t; + t = bc3 ^ std::rotl(bc0, 1); st[4] ^= t; st[9] ^= t; st[14] ^= t; st[19] ^= t; st[24] ^= t; // Rho Pi t = st[1]; - bc0 = st[10]; st[10] = Rotl(t, 1); t = bc0; - bc0 = st[7]; st[7] = Rotl(t, 3); t = bc0; - bc0 = st[11]; st[11] = Rotl(t, 6); t = bc0; - bc0 = st[17]; st[17] = Rotl(t, 10); t = bc0; - bc0 = st[18]; st[18] = Rotl(t, 15); t = bc0; - bc0 = st[3]; st[3] = Rotl(t, 21); t = bc0; - bc0 = st[5]; st[5] = Rotl(t, 28); t = bc0; - bc0 = st[16]; st[16] = Rotl(t, 36); t = bc0; - bc0 = st[8]; st[8] = Rotl(t, 45); t = bc0; - bc0 = st[21]; st[21] = Rotl(t, 55); t = bc0; - bc0 = st[24]; st[24] = Rotl(t, 2); t = bc0; - bc0 = st[4]; st[4] = Rotl(t, 14); t = bc0; - bc0 = st[15]; st[15] = Rotl(t, 27); t = bc0; - bc0 = st[23]; st[23] = Rotl(t, 41); t = bc0; - bc0 = st[19]; st[19] = Rotl(t, 56); t = bc0; - bc0 = st[13]; st[13] = Rotl(t, 8); t = bc0; - bc0 = st[12]; st[12] = Rotl(t, 25); t = bc0; - bc0 = st[2]; st[2] = Rotl(t, 43); t = bc0; - bc0 = st[20]; st[20] = Rotl(t, 62); t = bc0; - bc0 = st[14]; st[14] = Rotl(t, 18); t = bc0; - bc0 = st[22]; st[22] = Rotl(t, 39); t = bc0; - bc0 = st[9]; st[9] = Rotl(t, 61); t = bc0; - bc0 = st[6]; st[6] = Rotl(t, 20); t = bc0; - st[1] = Rotl(t, 44); + bc0 = st[10]; st[10] = std::rotl(t, 1); t = bc0; + bc0 = st[7]; st[7] = std::rotl(t, 3); t = bc0; + bc0 = st[11]; st[11] = std::rotl(t, 6); t = bc0; + bc0 = st[17]; st[17] = std::rotl(t, 10); t = bc0; + bc0 = st[18]; st[18] = std::rotl(t, 15); t = bc0; + bc0 = st[3]; st[3] = std::rotl(t, 21); t = bc0; + bc0 = st[5]; st[5] = std::rotl(t, 28); t = bc0; + bc0 = st[16]; st[16] = std::rotl(t, 36); t = bc0; + bc0 = st[8]; st[8] = std::rotl(t, 45); t = bc0; + bc0 = st[21]; st[21] = std::rotl(t, 55); t = bc0; + bc0 = st[24]; st[24] = std::rotl(t, 2); t = bc0; + bc0 = st[4]; st[4] = std::rotl(t, 14); t = bc0; + bc0 = st[15]; st[15] = std::rotl(t, 27); t = bc0; + bc0 = st[23]; st[23] = std::rotl(t, 41); t = bc0; + bc0 = st[19]; st[19] = std::rotl(t, 56); t = bc0; + bc0 = st[13]; st[13] = std::rotl(t, 8); t = bc0; + bc0 = st[12]; st[12] = std::rotl(t, 25); t = bc0; + bc0 = st[2]; st[2] = std::rotl(t, 43); t = bc0; + bc0 = st[20]; st[20] = std::rotl(t, 62); t = bc0; + bc0 = st[14]; st[14] = std::rotl(t, 18); t = bc0; + bc0 = st[22]; st[22] = std::rotl(t, 39); t = bc0; + bc0 = st[9]; st[9] = std::rotl(t, 61); t = bc0; + bc0 = st[6]; st[6] = std::rotl(t, 20); t = bc0; + st[1] = std::rotl(t, 44); // Chi Iota bc0 = st[0]; bc1 = st[1]; bc2 = st[2]; bc3 = st[3]; bc4 = st[4]; diff --git a/src/crypto/siphash.cpp b/src/crypto/siphash.cpp index 2c328d0840226..8004a0548ecb1 100644 --- a/src/crypto/siphash.cpp +++ b/src/crypto/siphash.cpp @@ -4,15 +4,15 @@ #include -#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b)))) +#include #define SIPROUND do { \ - v0 += v1; v1 = ROTL(v1, 13); v1 ^= v0; \ - v0 = ROTL(v0, 32); \ - v2 += v3; v3 = ROTL(v3, 16); v3 ^= v2; \ - v0 += v3; v3 = ROTL(v3, 21); v3 ^= v0; \ - v2 += v1; v1 = ROTL(v1, 17); v1 ^= v2; \ - v2 = ROTL(v2, 32); \ + v0 += v1; v1 = std::rotl(v1, 13); v1 ^= v0; \ + v0 = std::rotl(v0, 32); \ + v2 += v3; v3 = std::rotl(v3, 16); v3 ^= v2; \ + v0 += v3; v3 = std::rotl(v3, 21); v3 ^= v0; \ + v2 += v1; v1 = std::rotl(v1, 17); v1 ^= v2; \ + v2 = std::rotl(v2, 32); \ } while (0) CSipHasher::CSipHasher(uint64_t k0, uint64_t k1) diff --git a/src/hash.cpp b/src/hash.cpp index 1ece8c5a7922d..78a969ecd3826 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -7,13 +7,9 @@ #include #include +#include #include -inline uint32_t ROTL32(uint32_t x, int8_t r) -{ - return (x << r) | (x >> (32 - r)); -} - unsigned int MurmurHash3(unsigned int nHashSeed, Span vDataToHash) { // The following is MurmurHash3 (x86_32), see https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp @@ -31,11 +27,11 @@ unsigned int MurmurHash3(unsigned int nHashSeed, Span vData uint32_t k1 = ReadLE32(blocks + i*4); k1 *= c1; - k1 = ROTL32(k1, 15); + k1 = std::rotl(k1, 15); k1 *= c2; h1 ^= k1; - h1 = ROTL32(h1, 13); + h1 = std::rotl(h1, 13); h1 = h1 * 5 + 0xe6546b64; } @@ -55,7 +51,7 @@ unsigned int MurmurHash3(unsigned int nHashSeed, Span vData case 1: k1 ^= tail[0]; k1 *= c1; - k1 = ROTL32(k1, 15); + k1 = std::rotl(k1, 15); k1 *= c2; h1 ^= k1; } diff --git a/src/key.cpp b/src/key.cpp index 6c044dbeb0b2b..0bbcc74c82a48 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -179,7 +179,7 @@ CPrivKey CKey::GetPrivKey() const { size_t seckeylen; seckey.resize(SIZE); seckeylen = SIZE; - ret = ec_seckey_export_der(secp256k1_context_sign, seckey.data(), &seckeylen, begin(), fCompressed); + ret = ec_seckey_export_der(secp256k1_context_sign, seckey.data(), &seckeylen, UCharCast(begin()), fCompressed); assert(ret); seckey.resize(seckeylen); return seckey; @@ -190,7 +190,7 @@ CPubKey CKey::GetPubKey() const { secp256k1_pubkey pubkey; size_t clen = CPubKey::SIZE; CPubKey result; - int ret = secp256k1_ec_pubkey_create(secp256k1_context_sign, &pubkey, begin()); + int ret = secp256k1_ec_pubkey_create(secp256k1_context_sign, &pubkey, UCharCast(begin())); assert(ret); secp256k1_ec_pubkey_serialize(secp256k1_context_sign, (unsigned char*)result.begin(), &clen, &pubkey, fCompressed ? SECP256K1_EC_COMPRESSED : SECP256K1_EC_UNCOMPRESSED); assert(result.size() == clen); @@ -220,19 +220,19 @@ bool CKey::Sign(const uint256 &hash, std::vector& vchSig, bool gr WriteLE32(extra_entropy, test_case); secp256k1_ecdsa_signature sig; uint32_t counter = 0; - int ret = secp256k1_ecdsa_sign(secp256k1_context_sign, &sig, hash.begin(), begin(), secp256k1_nonce_function_rfc6979, (!grind && test_case) ? extra_entropy : nullptr); + int ret = secp256k1_ecdsa_sign(secp256k1_context_sign, &sig, hash.begin(), UCharCast(begin()), secp256k1_nonce_function_rfc6979, (!grind && test_case) ? extra_entropy : nullptr); // Grind for low R while (ret && !SigHasLowR(&sig) && grind) { WriteLE32(extra_entropy, ++counter); - ret = secp256k1_ecdsa_sign(secp256k1_context_sign, &sig, hash.begin(), begin(), secp256k1_nonce_function_rfc6979, extra_entropy); + ret = secp256k1_ecdsa_sign(secp256k1_context_sign, &sig, hash.begin(), UCharCast(begin()), secp256k1_nonce_function_rfc6979, extra_entropy); } assert(ret); secp256k1_ecdsa_signature_serialize_der(secp256k1_context_sign, vchSig.data(), &nSigLen, &sig); vchSig.resize(nSigLen); // Additional verification step to prevent using a potentially corrupted signature secp256k1_pubkey pk; - ret = secp256k1_ec_pubkey_create(secp256k1_context_sign, &pk, begin()); + ret = secp256k1_ec_pubkey_create(secp256k1_context_sign, &pk, UCharCast(begin())); assert(ret); ret = secp256k1_ecdsa_verify(secp256k1_context_static, &sig, hash.begin(), &pk); assert(ret); @@ -258,7 +258,7 @@ bool CKey::SignCompact(const uint256 &hash, std::vector& vchSig) vchSig.resize(CPubKey::COMPACT_SIGNATURE_SIZE); int rec = -1; secp256k1_ecdsa_recoverable_signature rsig; - int ret = secp256k1_ecdsa_sign_recoverable(secp256k1_context_sign, &rsig, hash.begin(), begin(), secp256k1_nonce_function_rfc6979, nullptr); + int ret = secp256k1_ecdsa_sign_recoverable(secp256k1_context_sign, &rsig, hash.begin(), UCharCast(begin()), secp256k1_nonce_function_rfc6979, nullptr); assert(ret); ret = secp256k1_ecdsa_recoverable_signature_serialize_compact(secp256k1_context_sign, &vchSig[1], &rec, &rsig); assert(ret); @@ -266,7 +266,7 @@ bool CKey::SignCompact(const uint256 &hash, std::vector& vchSig) vchSig[0] = 27 + rec + (fCompressed ? 4 : 0); // Additional verification step to prevent using a potentially corrupted signature secp256k1_pubkey epk, rpk; - ret = secp256k1_ec_pubkey_create(secp256k1_context_sign, &epk, begin()); + ret = secp256k1_ec_pubkey_create(secp256k1_context_sign, &epk, UCharCast(begin())); assert(ret); ret = secp256k1_ecdsa_recover(secp256k1_context_static, &rpk, &rsig, hash.begin()); assert(ret); @@ -279,7 +279,7 @@ bool CKey::SignSchnorr(const uint256& hash, Span sig, const uint2 { assert(sig.size() == 64); secp256k1_keypair keypair; - if (!secp256k1_keypair_create(secp256k1_context_sign, &keypair, begin())) return false; + if (!secp256k1_keypair_create(secp256k1_context_sign, &keypair, UCharCast(begin()))) return false; if (merkle_root) { secp256k1_xonly_pubkey pubkey; if (!secp256k1_keypair_xonly_pub(secp256k1_context_sign, &pubkey, nullptr, &keypair)) return false; @@ -324,7 +324,7 @@ bool CKey::Derive(CKey& keyChild, ChainCode &ccChild, unsigned int nChild, const BIP32Hash(cc, nChild, *pubkey.begin(), pubkey.begin()+1, vout.data()); } else { assert(size() == 32); - BIP32Hash(cc, nChild, 0, begin(), vout.data()); + BIP32Hash(cc, nChild, 0, UCharCast(begin()), vout.data()); } memcpy(ccChild.begin(), vout.data()+32, 32); keyChild.Set(begin(), begin() + 32, true); diff --git a/src/key.h b/src/key.h index 9cac716ec86b1..d6b26f891d0e4 100644 --- a/src/key.h +++ b/src/key.h @@ -100,7 +100,7 @@ class CKey { if (size_t(pend - pbegin) != std::tuple_size_v) { ClearKeyData(); - } else if (Check(&pbegin[0])) { + } else if (Check(UCharCast(&pbegin[0]))) { MakeKeyData(); memcpy(keydata->data(), (unsigned char*)&pbegin[0], keydata->size()); fCompressed = fCompressedIn; @@ -112,8 +112,8 @@ class CKey //! Simple read-only vector-like interface. unsigned int size() const { return keydata ? keydata->size() : 0; } const std::byte* data() const { return keydata ? reinterpret_cast(keydata->data()) : nullptr; } - const unsigned char* begin() const { return keydata ? keydata->data() : nullptr; } - const unsigned char* end() const { return begin() + size(); } + const std::byte* begin() const { return data(); } + const std::byte* end() const { return data() + size(); } //! Check whether this private key is valid. bool IsValid() const { return !!keydata; } diff --git a/src/key_io.cpp b/src/key_io.cpp index 5bcbb8a069426..a373a2201d9d0 100644 --- a/src/key_io.cpp +++ b/src/key_io.cpp @@ -228,7 +228,7 @@ std::string EncodeSecret(const CKey& key) { assert(key.IsValid()); std::vector data = Params().Base58Prefix(CChainParams::SECRET_KEY); - data.insert(data.end(), key.begin(), key.end()); + data.insert(data.end(), UCharCast(key.begin()), UCharCast(key.end())); if (key.IsCompressed()) { data.push_back(1); } diff --git a/src/pubkey.h b/src/pubkey.h index a06df510e6beb..b46793a659f99 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -287,10 +287,11 @@ class XOnlyPubKey CPubKey GetEvenCorrespondingCPubKey() const; const unsigned char& operator[](int pos) const { return *(m_keydata.begin() + pos); } - const unsigned char* data() const { return m_keydata.begin(); } static constexpr size_t size() { return decltype(m_keydata)::size(); } + const unsigned char* data() const { return m_keydata.begin(); } const unsigned char* begin() const { return m_keydata.begin(); } const unsigned char* end() const { return m_keydata.end(); } + unsigned char* data() { return m_keydata.begin(); } unsigned char* begin() { return m_keydata.begin(); } unsigned char* end() { return m_keydata.end(); } bool operator==(const XOnlyPubKey& other) const { return m_keydata == other.m_keydata; } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f23654e7ebbaf..fef1423d1fe0f 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2785,12 +2785,11 @@ static RPCHelpMan loadtxoutset() if (!chainman.ActivateSnapshot(afile, metadata, false)) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to load UTXO snapshot " + fs::PathToString(path)); } - CBlockIndex* new_tip{WITH_LOCK(::cs_main, return chainman.ActiveTip())}; UniValue result(UniValue::VOBJ); result.pushKV("coins_loaded", metadata.m_coins_count); - result.pushKV("tip_hash", new_tip->GetBlockHash().ToString()); - result.pushKV("base_height", new_tip->nHeight); + result.pushKV("tip_hash", snapshot_start_block->GetBlockHash().ToString()); + result.pushKV("base_height", snapshot_start_block->nHeight); result.pushKV("path", fs::PathToString(path)); return result; }, diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index f6d9d42f0f438..04d2e68939853 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -353,17 +353,15 @@ UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose, bool include_mempoo } return o; } else { + UniValue a(UniValue::VARR); uint64_t mempool_sequence; - std::vector vtxid; { LOCK(pool.cs); - pool.queryHashes(vtxid); + for (const CTxMemPoolEntry& e : pool.entryAll()) { + a.push_back(e.GetTx().GetHash().ToString()); + } mempool_sequence = pool.GetSequence(); } - UniValue a(UniValue::VARR); - for (const uint256& hash : vtxid) - a.push_back(hash.ToString()); - if (!include_mempool_sequence) { return a; } else { diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 12d89e0f8c875..2affa8cbafe1f 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -313,7 +313,7 @@ static RPCHelpMan addnode() { {"node", RPCArg::Type::STR, RPCArg::Optional::NO, "The address of the peer to connect to"}, {"command", RPCArg::Type::STR, RPCArg::Optional::NO, "'add' to add a node to the list, 'remove' to remove a node from the list, 'onetry' to try a connection to the node once"}, - {"v2transport", RPCArg::Type::BOOL, RPCArg::Default{false}, "Attempt to connect using BIP324 v2 transport protocol (ignored for 'remove' command)"}, + {"v2transport", RPCArg::Type::BOOL, RPCArg::DefaultHint{"set by -v2transport"}, "Attempt to connect using BIP324 v2 transport protocol (ignored for 'remove' command)"}, }, RPCResult{RPCResult::Type::NONE, "", ""}, RPCExamples{ @@ -332,9 +332,10 @@ static RPCHelpMan addnode() CConnman& connman = EnsureConnman(node); const std::string node_arg{request.params[0].get_str()}; - bool use_v2transport = self.Arg(2); + bool node_v2transport = connman.GetLocalServices() & NODE_P2P_V2; + bool use_v2transport = self.MaybeArg(2).value_or(node_v2transport); - if (use_v2transport && !(node.connman->GetLocalServices() & NODE_P2P_V2)) { + if (use_v2transport && !node_v2transport) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Error: v2transport requested but not enabled (see -v2transport)"); } diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index 4ad0956201350..8c0b0d702993a 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -101,9 +101,7 @@ void Finish(FuzzedDataProvider& fuzzed_data_provider, MockedTxPool& tx_pool, Cha if (!info_all.empty()) { const auto& tx_to_remove = *PickValue(fuzzed_data_provider, info_all).tx; WITH_LOCK(tx_pool.cs, tx_pool.removeRecursive(tx_to_remove, MemPoolRemovalReason::BLOCK /* dummy */)); - std::vector all_txids; - tx_pool.queryHashes(all_txids); - assert(all_txids.size() < info_all.size()); + assert(tx_pool.size() < info_all.size()); WITH_LOCK(::cs_main, tx_pool.check(chainstate.CoinsTip(), chainstate.m_chain.Height() + 1)); } SyncWithValidationInterfaceQueue(); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index acee56fe783b3..57a86549d9a42 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -803,19 +803,6 @@ std::vector CTxMemPool::Get return iters; } -void CTxMemPool::queryHashes(std::vector& vtxid) const -{ - LOCK(cs); - auto iters = GetSortedDepthAndScore(); - - vtxid.clear(); - vtxid.reserve(mapTx.size()); - - for (auto it : iters) { - vtxid.push_back(it->GetTx().GetHash()); - } -} - static TxMempoolInfo GetInfo(CTxMemPool::indexed_transaction_set::const_iterator it) { return TxMempoolInfo{it->GetSharedTx(), it->GetTime(), it->GetFee(), it->GetTxSize(), it->GetModifiedFee() - it->GetFee()}; } diff --git a/src/txmempool.h b/src/txmempool.h index 9da51756e6084..ca842632dabf4 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -485,7 +485,6 @@ class CTxMemPool void removeForBlock(const std::vector& vtx, unsigned int nBlockHeight) EXCLUSIVE_LOCKS_REQUIRED(cs); bool CompareDepthAndScore(const uint256& hasha, const uint256& hashb, bool wtxid=false); - void queryHashes(std::vector& vtxid) const; bool isSpent(const COutPoint& outpoint) const; unsigned int GetTransactionsUpdated() const; void AddTransactionsUpdated(unsigned int n); diff --git a/src/validation.cpp b/src/validation.cpp index 2adf2226dfb5d..4d1a0ef7b71bd 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -4887,7 +4887,9 @@ void ChainstateManager::CheckBlockIndex() // For testing, allow transaction counts to be completely unset. || (pindex->nChainTx == 0 && pindex->nTx == 0) // For testing, allow this nChainTx to be unset if previous is also unset. - || (pindex->nChainTx == 0 && prev_chain_tx == 0 && pindex->pprev)); + || (pindex->nChainTx == 0 && prev_chain_tx == 0 && pindex->pprev) + // Transaction counts prior to snapshot are unknown. + || pindex->IsAssumedValid()); if (pindexFirstAssumeValid == nullptr && pindex->nStatus & BLOCK_ASSUMED_VALID) pindexFirstAssumeValid = pindex; if (pindexFirstInvalid == nullptr && pindex->nStatus & BLOCK_FAILED_VALID) pindexFirstInvalid = pindex; diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index e9885f921074c..c1c3be0cc4f58 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -280,7 +280,7 @@ bool LegacyScriptPubKeyMan::Encrypt(const CKeyingMaterial& master_key, WalletBat { const CKey &key = mKey.second; CPubKey vchPubKey = key.GetPubKey(); - CKeyingMaterial vchSecret(key.begin(), key.end()); + CKeyingMaterial vchSecret{UCharCast(key.begin()), UCharCast(key.end())}; std::vector vchCryptedSecret; if (!EncryptSecret(master_key, vchSecret, vchPubKey.GetHash(), vchCryptedSecret)) { encrypted_batch = nullptr; @@ -810,7 +810,7 @@ bool LegacyScriptPubKeyMan::AddKeyPubKeyInner(const CKey& key, const CPubKey &pu } std::vector vchCryptedSecret; - CKeyingMaterial vchSecret(key.begin(), key.end()); + CKeyingMaterial vchSecret{UCharCast(key.begin()), UCharCast(key.end())}; if (!EncryptSecret(m_storage.GetEncryptionKey(), vchSecret, pubkey.GetHash(), vchCryptedSecret)) { return false; } @@ -2088,7 +2088,7 @@ bool DescriptorScriptPubKeyMan::Encrypt(const CKeyingMaterial& master_key, Walle { const CKey &key = key_in.second; CPubKey pubkey = key.GetPubKey(); - CKeyingMaterial secret(key.begin(), key.end()); + CKeyingMaterial secret{UCharCast(key.begin()), UCharCast(key.end())}; std::vector crypted_secret; if (!EncryptSecret(master_key, secret, pubkey.GetHash(), crypted_secret)) { return false; @@ -2261,7 +2261,7 @@ bool DescriptorScriptPubKeyMan::AddDescriptorKeyWithDB(WalletBatch& batch, const } std::vector crypted_secret; - CKeyingMaterial secret(key.begin(), key.end()); + CKeyingMaterial secret{UCharCast(key.begin()), UCharCast(key.end())}; if (!EncryptSecret(m_storage.GetEncryptionKey(), secret, pubkey.GetHash(), crypted_secret)) { return false; } diff --git a/test/README.md b/test/README.md index fee8828d343ee..0d1e06c0adec3 100644 --- a/test/README.md +++ b/test/README.md @@ -326,35 +326,7 @@ Use the `-v` option for verbose output. ### Lint tests -#### Dependencies - -| Lint test | Dependency | -|-----------|:----------:| -| [`lint-python.py`](lint/lint-python.py) | [flake8](https://gitlab.com/pycqa/flake8) -| [`lint-python.py`](lint/lint-python.py) | [lief](https://github.com/lief-project/LIEF) -| [`lint-python.py`](lint/lint-python.py) | [mypy](https://github.com/python/mypy) -| [`lint-python.py`](lint/lint-python.py) | [pyzmq](https://github.com/zeromq/pyzmq) -| [`lint-python-dead-code.py`](lint/lint-python-dead-code.py) | [vulture](https://github.com/jendrikseipp/vulture) -| [`lint-shell.py`](lint/lint-shell.py) | [ShellCheck](https://github.com/koalaman/shellcheck) -| [`lint-spelling.py`](lint/lint-spelling.py) | [codespell](https://github.com/codespell-project/codespell) - -In use versions and install instructions are available in the [CI setup](../ci/lint/04_install.sh). - -Please be aware that on Linux distributions all dependencies are usually available as packages, but could be outdated. - -#### Running the tests - -Individual tests can be run by directly calling the test script, e.g.: - -``` -test/lint/lint-files.py -``` - -You can run all the shell-based lint tests by running: - -``` -test/lint/all-lint.py -``` +See the README in [test/lint](/test/lint). # Writing functional tests diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 0ee332b75bf5f..d8ae20981da67 100644 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -602,10 +602,10 @@ def connect_nodes(self, a, b, *, peer_advertises_v2=None, wait_for_connect: bool if peer_advertises_v2 is None: peer_advertises_v2 = from_connection.use_v2transport - if peer_advertises_v2: - from_connection.addnode(node=ip_port, command="onetry", v2transport=True) + if peer_advertises_v2 != from_connection.use_v2transport: + from_connection.addnode(node=ip_port, command="onetry", v2transport=peer_advertises_v2) else: - # skip the optional third argument (default false) for + # skip the optional third argument if it matches the default, for # compatibility with older clients from_connection.addnode(ip_port, "onetry") @@ -1021,5 +1021,4 @@ def is_bdb_compiled(self): return self.config["components"].getboolean("USE_BDB") def has_blockfile(self, node, filenum: str): - blocksdir = node.datadir_path / self.chain / 'blocks' - return (blocksdir / f"blk{filenum}.dat").is_file() + return (node.blocks_path/ f"blk{filenum}.dat").is_file() diff --git a/test/lint/README.md b/test/lint/README.md index cddad0cdd91be..faa96c0bcd576 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -16,12 +16,36 @@ result is cached and it prevents issues when the image changes. test runner =========== -To run the checks in the test runner outside the docker, use: +To run all the lint checks in the test runner outside the docker, use: ```sh ( cd ./test/lint/test_runner/ && cargo fmt && cargo clippy && cargo run ) ``` +#### Dependencies + +| Lint test | Dependency | +|-----------|:----------:| +| [`lint-python.py`](lint/lint-python.py) | [flake8](https://gitlab.com/pycqa/flake8) +| [`lint-python.py`](lint/lint-python.py) | [lief](https://github.com/lief-project/LIEF) +| [`lint-python.py`](lint/lint-python.py) | [mypy](https://github.com/python/mypy) +| [`lint-python.py`](lint/lint-python.py) | [pyzmq](https://github.com/zeromq/pyzmq) +| [`lint-python-dead-code.py`](lint/lint-python-dead-code.py) | [vulture](https://github.com/jendrikseipp/vulture) +| [`lint-shell.py`](lint/lint-shell.py) | [ShellCheck](https://github.com/koalaman/shellcheck) +| [`lint-spelling.py`](lint/lint-spelling.py) | [codespell](https://github.com/codespell-project/codespell) + +In use versions and install instructions are available in the [CI setup](../ci/lint/04_install.sh). + +Please be aware that on Linux distributions all dependencies are usually available as packages, but could be outdated. + +#### Running the tests + +Individual tests can be run by directly calling the test script, e.g.: + +``` +test/lint/lint-files.py +``` + check-doc.py ============ Check for missing documentation of command line options. @@ -59,7 +83,3 @@ To do so, add the upstream repository as remote: ``` git remote add --fetch secp256k1 https://github.com/bitcoin-core/secp256k1.git ``` - -all-lint.py -=========== -Calls other scripts with the `lint-` prefix. diff --git a/test/lint/all-lint.py b/test/lint/all-lint.py deleted file mode 100755 index c7889796c6c1b..0000000000000 --- a/test/lint/all-lint.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (c) 2017-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# This script runs all test/lint/lint-* files, and fails if any exit -# with a non-zero status code. - -from glob import glob -from pathlib import Path -from subprocess import run -from sys import executable - -exit_code = 0 -mod_path = Path(__file__).parent -for lint in glob(f"{mod_path}/lint-*.py"): - result = run([executable, lint]) - if result.returncode != 0: - print(f"^---- failure generated from {lint.split('/')[-1]}") - exit_code |= result.returncode - -exit(exit_code) diff --git a/test/lint/test_runner/src/main.rs b/test/lint/test_runner/src/main.rs index ce94c3b628789..1dc79e97bdd15 100644 --- a/test/lint/test_runner/src/main.rs +++ b/test/lint/test_runner/src/main.rs @@ -3,6 +3,7 @@ // file COPYING or https://opensource.org/license/mit/. use std::env; +use std::fs; use std::path::PathBuf; use std::process::Command; use std::process::ExitCode; @@ -29,8 +30,8 @@ fn check_output(cmd: &mut std::process::Command) -> Result { } /// Return the git root as utf8, or panic -fn get_git_root() -> String { - check_output(git().args(["rev-parse", "--show-toplevel"])).unwrap() +fn get_git_root() -> PathBuf { + PathBuf::from(check_output(git().args(["rev-parse", "--show-toplevel"])).unwrap()) } fn lint_subtree() -> LintResult { @@ -94,11 +95,24 @@ fn lint_doc() -> LintResult { } fn lint_all() -> LintResult { - if Command::new("test/lint/all-lint.py") - .status() - .expect("command error") - .success() - { + let mut good = true; + let lint_dir = get_git_root().join("test/lint"); + for entry in fs::read_dir(lint_dir).unwrap() { + let entry = entry.unwrap(); + let entry_fn = entry.file_name().into_string().unwrap(); + if entry_fn.starts_with("lint-") + && entry_fn.ends_with(".py") + && !Command::new("python3") + .arg(entry.path()) + .status() + .expect("command error") + .success() + { + good = false; + println!("^---- failure generated from {}", entry_fn); + } + } + if good { Ok(()) } else { Err("".to_string()) @@ -110,10 +124,10 @@ fn main() -> ExitCode { ("subtree check", lint_subtree), ("std::filesystem check", lint_std_filesystem), ("-help=1 documentation check", lint_doc), - ("all-lint.py script", lint_all), + ("lint-*.py scripts", lint_all), ]; - let git_root = PathBuf::from(get_git_root()); + let git_root = get_git_root(); let mut test_failed = false; for (lint_name, lint_fn) in test_list { diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index f0ee698909f46..b22caa15a53be 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -69,7 +69,6 @@ implicit-signed-integer-truncation:crypto/ implicit-unsigned-integer-truncation:crypto/ shift-base:arith_uint256.cpp shift-base:crypto/ -shift-base:ROTL32 shift-base:streams.h shift-base:FormatHDKeypath shift-base:xoroshiro128plusplus.h