From ae36ff4f0679df21233c2e829f9c840c6a016b71 Mon Sep 17 00:00:00 2001 From: Stephen Gallagher Date: Wed, 13 Jan 2021 15:16:28 -0500 Subject: [PATCH 1/2] Simplify autoformatters in CI We don't need to stand up a heavy container image just to run the autoformatters. We'll just use the ones available in the host environment. Also switch to https://github.com/marketplace/actions/add-commit for handling the git commit. Signed-off-by: Stephen Gallagher --- .ci/autoformat/Dockerfile.tmpl | 12 -------- .ci/autoformat/ci-tasks.sh | 33 ---------------------- .ci/git_push_if_changes.sh | 46 ------------------------------- .github/workflows/formatters.yaml | 30 ++++++++++++++++++++ .github/workflows/upstreamed.yaml | 17 ------------ 5 files changed, 30 insertions(+), 108 deletions(-) delete mode 100644 .ci/autoformat/Dockerfile.tmpl delete mode 100755 .ci/autoformat/ci-tasks.sh delete mode 100755 .ci/git_push_if_changes.sh create mode 100644 .github/workflows/formatters.yaml diff --git a/.ci/autoformat/Dockerfile.tmpl b/.ci/autoformat/Dockerfile.tmpl deleted file mode 100644 index f4fec9dc9..000000000 --- a/.ci/autoformat/Dockerfile.tmpl +++ /dev/null @@ -1,12 +0,0 @@ -FROM __DEPS_IMAGE__ - -MAINTAINER Stephen Gallagher - -RUN dnf -y --setopt=install_weak_deps=False --nogpgcheck install rsync \ - && dnf -y clean all - -ARG TARBALL - -ADD $TARBALL /builddir/ - -ENTRYPOINT /builddir/.ci/autoformat/ci-tasks.sh diff --git a/.ci/autoformat/ci-tasks.sh b/.ci/autoformat/ci-tasks.sh deleted file mode 100755 index 879ed940b..000000000 --- a/.ci/autoformat/ci-tasks.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -# Exit on failures -set -e -set -x - -: ${FORMAT_DEST_DIR:?parameter must be set} - -pushd /builddir/ - -# Build the code under GCC and run code auto-formatting -meson setup \ - --buildtype=debugoptimized \ - -Dverbose_tests=false \ - $COMMON_MESON_ARGS \ - autoformat - -set +e -meson test \ - -C autoformat \ - --suite formatters \ - --print-errorlogs \ - -t 5 -err=$? -if [ $err != 0 ]; then - cat autoformat/meson-logs/testlog.txt - exit $err -fi -set -e - -rsync -avh --existing --no-perms --no-times /builddir/modulemd/ $FORMAT_DEST_DIR - -popd #builddir diff --git a/.ci/git_push_if_changes.sh b/.ci/git_push_if_changes.sh deleted file mode 100755 index 2378fe681..000000000 --- a/.ci/git_push_if_changes.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash - -usage="Usage: ${FUNCNAME[0]} [-u git_user] [-e git_email] [-m commit_message]" - -git_user="github-actions" -git_email="github-actions@github.com" -git_msg="generated" - -while getopts "e:m:u:" OPTION; do -case "${OPTION}" in -e) - git_email=${OPTARG} - ;; -m) - git_msg=${OPTARG} - ;; -u) - git_user=${OPTARG} - ;; -*) - echo "$usage" 1>&2 - exit 1 - ;; -esac -done -shift $((OPTIND-1)) - -#Exit on failures -set -e -set -x - -# Check to see if there are any changes -set +e -git commit -m "$git_msg" --dry-run -err=$? -if [ $err = 0 ]; then - echo committing and pushing changes - set -e - git config user.name "$git_user" - git config user.email "$git_email" - git commit -m "$git_msg" - git show - git push -fi -set -e - diff --git a/.github/workflows/formatters.yaml b/.github/workflows/formatters.yaml new file mode 100644 index 000000000..6e3d50886 --- /dev/null +++ b/.github/workflows/formatters.yaml @@ -0,0 +1,30 @@ +name: Automatic Code Formatters +on: push + +jobs: + autoformat: + name: Auto-format code + runs-on: ubuntu-20.04 + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Install formatters + run: | + sudo apt-get update + sudo apt-get install clang-format black + + - name: Format code + run: | + clang-format -i --verbose $(find . -name "*.[ch]") + black --target-version py36 --verbose --line-length 79 . + + - name: Commit changes + uses: EndBug/add-and-commit@v6 + with: + author_name: Libmodulemd CI + author_email: github-actions@github.com + message: 'Code auto-formatting' + add: -u + signoff: true + push: true diff --git a/.github/workflows/upstreamed.yaml b/.github/workflows/upstreamed.yaml index 4ce99ef2d..d80b22c43 100644 --- a/.github/workflows/upstreamed.yaml +++ b/.github/workflows/upstreamed.yaml @@ -3,7 +3,6 @@ on: push: branches: - main - jobs: static_analysis: name: Static Analysis @@ -30,19 +29,3 @@ jobs: DOC_TOKEN: ${{ secrets.DOC_TOKEN }} run: | ./.ci/ci-docs.sh - - autoformat: - name: Auto-format code - runs-on: ubuntu-20.04 - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Format code - run: | - ./.ci/ci-autoformat.sh - - - name: Push any formatting changes - run: | - git add -u - $GITHUB_WORKSPACE/.ci/git_push_if_changes.sh -m "Code auto-formatting" From 1096cda67bbb465ebca2781916c39b784a1b3097 Mon Sep 17 00:00:00 2001 From: Stephen Gallagher Date: Wed, 13 Jan 2021 15:51:16 -0500 Subject: [PATCH 2/2] Drop formatters from codebase They are now handled by automatic CI runs. Signed-off-by: Stephen Gallagher --- .ci/archlinux/ci-tasks.sh | 2 - .ci/centos/ci-tasks.sh | 8 +-- .ci/coverity/ci-tasks.sh | 4 +- .ci/docs/ci-tasks.sh | 1 - .ci/fedora/ci-tasks.sh | 11 ++-- .ci/mageia/ci-tasks.sh | 8 +-- .ci/openmandriva/ci-tasks.sh | 11 ++-- .ci/opensuse/ci-tasks.sh | 2 +- contrib/release-tools/release.sh | 2 +- fedora/libmodulemd.spec | 3 +- fedora/tests.yml | 2 +- meson.build | 5 +- meson_options.txt | 9 ---- modulemd/meson.build | 86 -------------------------------- modulemd/tests/test-dirty.py | 38 -------------- modulemd/tests/test-version.sh | 36 ------------- 16 files changed, 26 insertions(+), 202 deletions(-) delete mode 100644 modulemd/tests/test-dirty.py delete mode 100755 modulemd/tests/test-version.sh diff --git a/.ci/archlinux/ci-tasks.sh b/.ci/archlinux/ci-tasks.sh index c11e6ce56..f98faca2a 100755 --- a/.ci/archlinux/ci-tasks.sh +++ b/.ci/archlinux/ci-tasks.sh @@ -9,8 +9,6 @@ pushd /builddir/ # Build the code under GCC and run standard tests meson --buildtype=debugoptimized \ - -Dtest_dirty_git=false \ - -Ddeveloper_build=false \ -Dpython_name=python3 \ -Drpmio=disabled \ ci diff --git a/.ci/centos/ci-tasks.sh b/.ci/centos/ci-tasks.sh index b54e6ed24..b63c78378 100755 --- a/.ci/centos/ci-tasks.sh +++ b/.ci/centos/ci-tasks.sh @@ -4,17 +4,11 @@ set -e set -x - -# CentOS 7 doesn't have autopep8, so we'll drop the requirement for it -# This implementation will still allow it to occur if autopep8 still shows -# up later. -COMMON_MESON_ARGS="-Dtest_dirty_git=false -Ddeveloper_build=false -Dpython_name=python3.6" - pushd /builddir/ # Build the code under GCC and run standard tests meson --buildtype=debugoptimized \ - $COMMON_MESON_ARGS \ + -Dpython_name=python3.6 \ ci ninja-build -C ci test diff --git a/.ci/coverity/ci-tasks.sh b/.ci/coverity/ci-tasks.sh index 0f1e4d6c1..84cb3653e 100755 --- a/.ci/coverity/ci-tasks.sh +++ b/.ci/coverity/ci-tasks.sh @@ -8,10 +8,10 @@ if [ -e /usr/lib/os-release ]; then source /usr/lib/os-release case "${ID-unknown} ${VERSION_ID-unknown}" in "centos 7") - COMMON_MESON_ARGS="-Dtest_dirty_git=false -Ddeveloper_build=false -Dpython_name=python3.6" + COMMON_MESON_ARGS="-Dpython_name=python3.6" ;; "centos "*) - COMMON_MESON_ARGS="-Dtest_dirty_git=false -Ddeveloper_build=false" + COMMON_MESON_ARGS="" ;; esac fi diff --git a/.ci/docs/ci-tasks.sh b/.ci/docs/ci-tasks.sh index 7f0fc996d..46921692a 100755 --- a/.ci/docs/ci-tasks.sh +++ b/.ci/docs/ci-tasks.sh @@ -10,7 +10,6 @@ pushd /builddir/ # Build the code under GCC and run documentation generation meson --buildtype=debug \ - $COMMON_MESON_ARGS \ doc-generation set +e diff --git a/.ci/fedora/ci-tasks.sh b/.ci/fedora/ci-tasks.sh index 91b3e2696..32a1779d6 100755 --- a/.ci/fedora/ci-tasks.sh +++ b/.ci/fedora/ci-tasks.sh @@ -5,7 +5,6 @@ set -e set -x PROCESSORS=$(/usr/bin/getconf _NPROCESSORS_ONLN) -MESON_DIRTY_REPO_ARGS="-Dtest_dirty_git=${DIRTY_REPO_CHECK:-false}" RETRY_CMD=/builddir/.ci/retry-command.sh WITH_RPM_TESTS=${WITH_RPM_TESTS:-true} @@ -18,8 +17,6 @@ sed -i -e 's/test -r/test -f/g' -e 's/test -x/test -f/g' /bin/ldd # Build the code under GCC and run standard tests meson --buildtype=debugoptimized \ - -Dverbose_tests=false \ - $MESON_DIRTY_REPO_ARGS \ ci meson test --suite formatters \ @@ -34,9 +31,14 @@ meson test --suite ci \ --print-errorlogs \ -t 5 +# Disable the verbose tests and run the supported tests through valgrind +meson --buildtype=debugoptimized \ + -Dverbose_tests=false \ + ci-valgrind + meson test --suite ci_valgrind \ --wrap=/builddir/contrib/valgrind/valgrind_wrapper.sh \ - -C ci \ + -C ci-valgrind \ --num-processes=$PROCESSORS \ --print-errorlogs \ -t 10 @@ -53,7 +55,6 @@ else set -e meson --buildtype=debug \ -Dskip_introspection=true \ - $COMMON_MESON_ARGS \ ci_scanbuild pushd ci_scanbuild diff --git a/.ci/mageia/ci-tasks.sh b/.ci/mageia/ci-tasks.sh index ca2a79873..00ca1b853 100755 --- a/.ci/mageia/ci-tasks.sh +++ b/.ci/mageia/ci-tasks.sh @@ -5,15 +5,12 @@ set -e set -x PROCESSORS=$(/usr/bin/getconf _NPROCESSORS_ONLN) -COMMON_MESON_ARGS="-Dtest_dirty_git=${DIRTY_REPO_CHECK:-false} -Ddeveloper_build=false" pushd /builddir/ # Build the code under GCC and run standard tests meson --buildtype=debugoptimized \ - -Dverbose_tests=false \ - $COMMON_MESON_ARGS \ ci meson test --suite ci \ @@ -22,9 +19,12 @@ meson test --suite ci \ --print-errorlogs \ -t 5 +meson --buildtype=debugoptimized \ + -Dverbose_tests=false \ + ci-valgrind meson test --suite ci_valgrind \ --wrap=/builddir/contrib/valgrind/valgrind_wrapper.sh \ - -C ci \ + -C ci-valgrind \ --num-processes=$PROCESSORS \ --print-errorlogs \ -t 10 diff --git a/.ci/openmandriva/ci-tasks.sh b/.ci/openmandriva/ci-tasks.sh index 7905c35e8..c089dd4ef 100755 --- a/.ci/openmandriva/ci-tasks.sh +++ b/.ci/openmandriva/ci-tasks.sh @@ -5,13 +5,12 @@ set -e set -x PROCESSORS=$(/usr/bin/getconf _NPROCESSORS_ONLN) -COMMON_MESON_ARGS="-Dtest_dirty_git=false -Ddeveloper_build=false -Dwith_docs=false" +COMMON_MESON_ARGS="-Dwith_docs=false" pushd /builddir/ # Build the code under LLVM/clang and run standard tests CC=clang CXX=clang++ meson --buildtype=debugoptimized \ - -Dverbose_tests=false \ $COMMON_MESON_ARGS \ ci @@ -21,9 +20,15 @@ meson test --suite ci \ --print-errorlogs \ -t 5 +# Disable the verbose tests and run the supported tests through valgrind +CC=clang CXX=clang++ meson --buildtype=debugoptimized \ + -Dverbose_tests=false \ + $COMMON_MESON_ARGS \ + ci-valgrind + meson test --suite ci_valgrind \ --wrap=/builddir/contrib/valgrind/valgrind_wrapper.sh \ - -C ci \ + -C ci-valgrind \ --num-processes=$PROCESSORS \ --print-errorlogs \ -t 10 diff --git a/.ci/opensuse/ci-tasks.sh b/.ci/opensuse/ci-tasks.sh index 2382c7bf7..7a943c5e0 100755 --- a/.ci/opensuse/ci-tasks.sh +++ b/.ci/opensuse/ci-tasks.sh @@ -5,7 +5,7 @@ set -e set -x -COMMON_MESON_ARGS="-Dtest_dirty_git=false -Ddeveloper_build=false -Dpython_name=python3" +COMMON_MESON_ARGS="-Dpython_name=python3" pushd /builddir/ diff --git a/contrib/release-tools/release.sh b/contrib/release-tools/release.sh index e118505ae..f59f43bbf 100755 --- a/contrib/release-tools/release.sh +++ b/contrib/release-tools/release.sh @@ -41,7 +41,7 @@ hub api "https://api.github.com/user" || error_out code=1 message="Invalid crede OLDTAG=$(git describe --first-parent --abbrev=0) # Configure the build directory -meson --buildtype=release -Dskip_formatters=true -Ddeveloper_build=false $TMPDIR +meson --buildtype=release $TMPDIR MMD_SKIP_VALGRIND=True ninja -C $TMPDIR dist # Get the version that will be tagged diff --git a/fedora/libmodulemd.spec b/fedora/libmodulemd.spec index 3a1f10706..36c08330e 100644 --- a/fedora/libmodulemd.spec +++ b/fedora/libmodulemd.spec @@ -99,8 +99,7 @@ Development files for libmodulemd. %build -%meson -Ddeveloper_build=false \ - %{meson_python_flags} +%meson %{meson_python_flags} %meson_build diff --git a/fedora/tests.yml b/fedora/tests.yml index afbcca60a..f736549da 100644 --- a/fedora/tests.yml +++ b/fedora/tests.yml @@ -26,7 +26,7 @@ tests: - build: dir: . - run: meson -Dtest_installed_lib=true -Ddeveloper_build=false source + run: meson -Dtest_installed_lib=true source - unittests: dir: . run: ninja test diff --git a/meson.build b/meson.build index 955dfe1fd..10baca3b3 100644 --- a/meson.build +++ b/meson.build @@ -196,16 +196,13 @@ if meson.version().version_compare('>=0.53') 'Python 3 GObject Overrides': gobject_overrides_dir_py3 }, section: 'Directories') - summary({'Developer Build': get_option('developer_build'), - 'libmagic Support': magic_status, + summary({'libmagic Support': magic_status, 'Custom Python': get_option('python_name'), 'RPMIO Support': rpmio_status, 'Generate Manpages': manpages_status, 'Generate HTML Documentation': get_option('with_docs'), 'Python 2 Support': get_option('with_py2'), - 'Skip Formatters': skip_formatters, 'Skip Introspection': get_option('skip_introspection'), - 'Test Dirty Git': get_option('test_dirty_git'), 'Test Installed Library': get_option('test_installed_lib'), }, section: 'Build Configuration') endif diff --git a/meson_options.txt b/meson_options.txt index 6f6a47cea..db7445b29 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -11,9 +11,6 @@ # # REMEMBER TO UPDATE THE SUMMARY() IN meson.build when adding options here -option('developer_build', type : 'boolean', value : true, - description : 'Enables automatic code formatters and memory leak checks. This option should be set to FALSE for release builds.') - option('verbose_tests', type : 'boolean', value : true, description : 'Tests that are run under the "debug" configuration will print all debug messages. Disable this option for valgrind checks, as it speeds it up substantially.') @@ -26,15 +23,9 @@ option('python_name', type : 'string', option('rpmio', type : 'feature', value : 'enabled', description : 'Use the rpmio library to automatically decompress gzip, bzip2 and xz YAML streams.') -option('skip_formatters', type : 'boolean', value : false, - description : 'Do not do any automatic formatting of source code.') - option('skip_introspection', type : 'boolean', value : false, description : 'Do not generate GObject Introspection data.') -option('test_dirty_git', type : 'boolean', value : false, - description : 'Check whether the automatic formatting tools have made changes. Used during CI to verify coding style compliance.') - option('test_installed_lib', type : 'boolean', value : false, description : 'Build only the test suite and run it against a copy of libmodulemd installed on the local system.') diff --git a/modulemd/meson.build b/modulemd/meson.build index a8db7cd3c..d85a23325 100644 --- a/modulemd/meson.build +++ b/modulemd/meson.build @@ -11,41 +11,11 @@ # -- Configuration Options -- # -test_dirty_git = get_option('test_dirty_git') test_installed_lib = get_option('test_installed_lib') -skip_formatters = get_option('skip_formatters') skip_introspection = get_option('skip_introspection') -developer_build = get_option('developer_build') verbose_tests = get_option('verbose_tests') - -if not developer_build - skip_formatters = true -endif - clang_simple_version_script = find_program ('clang_simple_version.sh') -if skip_formatters - clang_format = disabler() - python_formatter = disabler() - pycodestyle = disabler() -else - clang_format = find_program('clang-format', required: developer_build) - if not clang_format.found() - clang_format = disabler() - endif - - python_formatter = find_program('black', required : developer_build) - if not python_formatter.found() - python_formatter = disabler() - endif - - pycodestyle = find_program('pycodestyle-3', 'pycodestyle', - required : developer_build) - if not pycodestyle.found() - pycodestyle = disabler() - endif -endif - # -- Library -- # @@ -280,15 +250,6 @@ py_test_release_env = py_test_env py_test_release_env.set('G_DEBUG', 'fatal-warnings,fatal-criticals') -# Make sure we remembered to bump the version number since the last release -version_test = find_program('tests/test-version.sh') -if developer_build - test('version', - version_test, - env : test_env, - suite : ['smoketest', 'ci']) -endif - # --- Test utility library --- # test_utils_lib = library( 'modulemd_tests', @@ -397,53 +358,6 @@ foreach name, script : python_tests endforeach -# --- Formatting Helpers -- # -# Run these after the functional tests so we get those results more quickly - - -# Fake test to ensure that all sources and headers are formatted properly -clang_files = modulemd_srcs + modulemd_hdrs + modulemd_priv_hdrs + modulemd_validator_srcs + test_srcs + test_priv_hdrs - -clang_args = [ '-i' ] -test('clang_format', clang_format, - args : clang_args + clang_files, - suite : 'formatters') - - -# Fake test to ensure that the python tests are formatted according to PEP8 -python_formatter_args = [ '--verbose', '--line-length', '79' ] -test('python_formatter', python_formatter, - args : python_formatter_args + [ meson.source_root() ], - suite : 'formatters') - - -# Test all python files for compliance with pycodestyle -# Ignore line-length as there are a number of places where this is unavoidable -python_scripts = [ files('tests/test-dirty.py', - '../bindings/python/gi/overrides/Modulemd.py') + - test_python_scripts ] - -pycodestyle_args = [ - '--show-pep8', - '--ignore=E121,E123,E126,E226,E24,E501,E704,W503,W504' -] -test('pycodestyle', pycodestyle, - args : pycodestyle_args + python_scripts, - suite : 'formatters') - - -# Fake test to ensure that the autoformatters didn't end up making changes -# when run in CI (such as during a pull-request). This test is only run if -# meson was configured with `-Dtest_dirty_git=true` - -if test_dirty_git - dirty_repo_scripts = [ files('tests/test-dirty.py') ] - test('test_dirty_repo', python3, - args : dirty_repo_scripts, - suite : ['ci', 'dirty']) -endif - - # -- C/C++ Header test -- # # Ensures that all public headers can be imported by consumers # This test takes a while, so run it near the end so that the functional test diff --git a/modulemd/tests/test-dirty.py b/modulemd/tests/test-dirty.py deleted file mode 100644 index 45f9fe4eb..000000000 --- a/modulemd/tests/test-dirty.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/python3 - -# This file is part of libmodulemd -# Copyright (C) 2017-2018 Stephen Gallagher -# -# Fedora-License-Identifier: MIT -# SPDX-2.0-License-Identifier: MIT -# SPDX-3.0-License-Identifier: MIT -# -# This program is free software. -# For more information on the license, see COPYING. -# For more information on free software, see -# . - -import os -import sys -import git -import subprocess - -script_dir = os.path.dirname(os.path.realpath(__file__)) - -# Get the repo we're running in -repo = git.Repo(script_dir, search_parent_directories=True) - -# When running in CI, the only reason the git repo could -# become "dirty" (files differ from their checkout) is if -# the autoformatter made changes. This should be reported -# back so the submitter can fix this. -if repo.is_dirty(): - print( - "Autoformatter was not run before submitting. Please run " - "`ninja test`, amend the commit and resubmit this pull request." - ) - res = subprocess.run(["git", "diff"], capture_output=True, text=True) - print(res.stdout, file=sys.stderr) - sys.exit(os.EX_USAGE) - -sys.exit(os.EX_OK) diff --git a/modulemd/tests/test-version.sh b/modulemd/tests/test-version.sh deleted file mode 100755 index c51852421..000000000 --- a/modulemd/tests/test-version.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/bash - -set -x - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -pushd $MESON_BUILD_ROOT - -function error_out { - local code message - local "${@}" - - echo $message 1>&2 - exit $code -} - -function common_finalize { - exitcode=$? - - return $exitcode -} - -trap common_finalize EXIT - -jq --version >/dev/null || error_out code=127 message="Install 'jq' to use this script" - -# Get the previous tag for this branch -OLDTAG=$(git describe --first-parent --abbrev=0) - -# Get the version that will be tagged -NEWVERSION=$(meson introspect $MESON_BUILD_ROOT --projectinfo |jq -r .version) -NEWTAG=libmodulemd-$NEWVERSION - -if [ "$NEWTAG" = "$OLDTAG" ]; then - error_out code=2 message="Version is already tagged. Update meson.build with the new version." -fi -