From 070534afbf5c6e96652b9551c90d3403561988fb Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 13 Oct 2022 13:03:17 +0200 Subject: [PATCH 1/3] Don't download the whole Boost.CI Repo in functions.star Only download the required file(s). Move the command commands of all unix pipelines into a function to implement this only once --- ci/drone/functions.star | 97 ++++++++++++++++++----------------------- 1 file changed, 43 insertions(+), 54 deletions(-) diff --git a/ci/drone/functions.star b/ci/drone/functions.star index 17dbd0a6..fc30395e 100644 --- a/ci/drone/functions.star +++ b/ci/drone/functions.star @@ -7,6 +7,37 @@ # For Drone CI we use the Starlark scripting language to reduce duplication. # As the yaml syntax for Drone CI is rather limited. +# Common steps for unix systems +# Takes the install script (inside the Boost.CI "ci/drone" folder) and the build script (relative to the root .drone folder) +def unix_common(install_script, buildscript_to_run): + # Commands to download scripts + # Factored out for ease of writing the commands + download_scripts = [ + # Only when not testing Boost.CI + 'if [ "$(basename "$DRONE_REPO")" != "boost-ci" ]; then', + + # Install script + 'wget "https://github.com/boostorg/boost-ci/raw/master/ci/drone/%s" -P ci/drone' % install_script, + # Default build script (if not exists) + '[ ! -e .drone/drone.sh ] || wget "https://github.com/boostorg/boost-ci/raw/master/.drone/drone.sh" -P .drone', + # Chosen build script inside .drone (if a filename/stem and does not exist) + '{{ [ "$(basename "{0}")" == "{0}" ] && [ ! -e .drone/{0}.sh ]; }} || wget "https://github.com/boostorg/boost-ci/raw/master/.drone/{0}.sh" -P .drone'.format(buildscript_to_run), + + 'fi' + ] + return [ + "echo '==================================> SETUP'", + "uname -a", + "export PATH=/usr/local/bin:$PATH", + '\n'.join(download_scripts), + + "echo '==================================> PACKAGES'", + "ci/drone/" + install_script, + + "echo '==================================> INSTALL AND TEST'", + ".drone/%s.sh" % buildscript_to_run, + ] + # Generate pipeline for Linux platform compilers. def linux_cxx(name, cxx, cxxflags="", packages="", sources="", llvm_os="", llvm_ver="", arch="amd64", image="cppalliance/ubuntu16.04:1", buildtype="boost", buildscript="", environment={}, globalenv={}, triggers={ "branch": [ "master", "develop", "drone*", "bugfix/*", "feature/*", "fix/*", "pr/*" ] }, node={}, privileged=False): environment_global = { @@ -24,10 +55,8 @@ def linux_cxx(name, cxx, cxxflags="", packages="", sources="", llvm_os="", llvm_ environment_current=environment_global environment_current.update(environment) - if buildscript: - buildscript_to_run = buildscript - else: - buildscript_to_run = buildtype + if not buildscript: + buildscript = buildtype return { "name": "Linux %s" % name, @@ -49,23 +78,10 @@ def linux_cxx(name, cxx, cxxflags="", packages="", sources="", llvm_os="", llvm_ "pull": "if-not-exists", "privileged" : privileged, "environment": environment_current, - "commands": [ - - "echo '==================================> SETUP'", - "uname -a", - # Moved to Docker: - # "apt-get -o Acquire::Retries=3 update && DEBIAN_FRONTEND=noninteractive apt-get -y install tzdata && apt-get -o Acquire::Retries=3 install -y sudo software-properties-common wget curl apt-transport-https git make cmake apt-file sudo unzip libssl-dev build-essential autotools-dev autoconf libc++-helpers automake g++", - # "for i in {1..3}; do apt-add-repository ppa:git-core/ppa && break || sleep 10; done", - # "apt-get -o Acquire::Retries=3 update && apt-get -o Acquire::Retries=3 -y install git", - # - "BOOST_CI_ORG=boostorg BOOST_CI_BRANCH=master && curl -s -S --retry 10 -L -o $BOOST_CI_BRANCH.tar.gz https://github.com/$BOOST_CI_ORG/boost-ci/archive/$BOOST_CI_BRANCH.tar.gz && tar -xvf $BOOST_CI_BRANCH.tar.gz && mv boost-ci-$BOOST_CI_BRANCH .drone/boost-ci && rm $BOOST_CI_BRANCH.tar.gz", - "echo '==================================> PACKAGES'", - # "./.drone/linux-cxx-install.sh", - "./.drone/boost-ci/ci/drone/linux-cxx-install.sh", - - "echo '==================================> INSTALL AND COMPILE'", - "./.drone/%s.sh" % buildscript_to_run, - ] + # Installed in Docker: + # - ppa:git-core/ppa + # - tzdata sudo software-properties-common wget curl apt-transport-https git make cmake apt-file sudo unzip libssl-dev build-essential autotools-dev autoconf libc++-helpers automake g++ git + "commands": unix_common("linux-cxx-install.sh", buildscript) } ] } @@ -137,10 +153,8 @@ def osx_cxx(name, cxx, cxxflags="", packages="", sources="", llvm_os="", llvm_ve environment_current=environment_global environment_current.update(environment) - if buildscript: - buildscript_to_run = buildscript - else: - buildscript_to_run = buildtype + if not buildscript: + buildscript = buildtype if xcode_version: environment_current.update({"DEVELOPER_DIR": "/Applications/Xcode-" + xcode_version + ".app/Contents/Developer"}) @@ -175,18 +189,7 @@ def osx_cxx(name, cxx, cxxflags="", packages="", sources="", llvm_os="", llvm_ve # "pull": "if-not-exists", "privileged" : privileged, "environment": environment_current, - "commands": [ - - "echo '==================================> SETUP'", - "uname -a", - "BOOST_CI_ORG=boostorg BOOST_CI_BRANCH=master && curl -s -S --retry 10 -L -o $BOOST_CI_BRANCH.tar.gz https://github.com/$BOOST_CI_ORG/boost-ci/archive/$BOOST_CI_BRANCH.tar.gz && tar -xvf $BOOST_CI_BRANCH.tar.gz && mv boost-ci-$BOOST_CI_BRANCH .drone/boost-ci && rm $BOOST_CI_BRANCH.tar.gz", - - "echo '==================================> PACKAGES'", - "./.drone/boost-ci/ci/drone/osx-cxx-install.sh", - - "echo '==================================> INSTALL AND COMPILE'", - "./.drone/%s.sh" % buildscript_to_run, - ] + "commands": unix_common("osx-cxx-install.sh", buildscript) } ] } @@ -207,10 +210,8 @@ def freebsd_cxx(name, cxx, cxxflags="", packages="", sources="", llvm_os="", llv environment_current=environment_global environment_current.update(environment) - if buildscript: - buildscript_to_run = buildscript - else: - buildscript_to_run = buildtype + if not buildscript: + buildscript = buildtype return { "name": "FreeBSD %s" % name, @@ -231,19 +232,7 @@ def freebsd_cxx(name, cxx, cxxflags="", packages="", sources="", llvm_os="", llv # "pull": "if-not-exists", "privileged" : privileged, "environment": environment_current, - "commands": [ - - "echo '==================================> SETUP'", - "uname -a", - "export PATH=/usr/local/bin:$PATH", - "BOOST_CI_ORG=boostorg BOOST_CI_BRANCH=master && curl -s -S --retry 10 -L -o $BOOST_CI_BRANCH.tar.gz https://github.com/$BOOST_CI_ORG/boost-ci/archive/$BOOST_CI_BRANCH.tar.gz && tar -xvf $BOOST_CI_BRANCH.tar.gz && mv boost-ci-$BOOST_CI_BRANCH .drone/boost-ci && rm $BOOST_CI_BRANCH.tar.gz", - - "echo '==================================> PACKAGES'", - "./.drone/boost-ci/ci/drone/freebsd-cxx-install.sh", - - "echo '==================================> INSTALL AND COMPILE'", - "./.drone/%s.sh" % buildscript_to_run, - ] + "commands": unix_common("freebsd-cxx-install.sh", buildscript) } ] } From 20841cdcbf904f4b9ec09381629b1417233b3f9f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 13 Oct 2022 18:58:35 +0200 Subject: [PATCH 2/3] Use curl instead of wget for downloading Factor it out into a helper function to allow changing it later --- ci/drone/functions.star | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/ci/drone/functions.star b/ci/drone/functions.star index fc30395e..21d27fed 100644 --- a/ci/drone/functions.star +++ b/ci/drone/functions.star @@ -7,6 +7,13 @@ # For Drone CI we use the Starlark scripting language to reduce duplication. # As the yaml syntax for Drone CI is rather limited. +# Helper function to compose a download command from BoostCI +# Downloads the file inside the directory @boostCI_dir from the master branch of BoostCI into @out_dir (defaults to @boostCI_dir) +def download_from_boostCI(filename, boostCI_dir, out_dir=None): + if out_dir == None: + out_dir = boostCI_dir + return 'curl -s -S --retry 10 --create-dirs -L "https://github.com/boostorg/boost-ci/raw/master/{1}/{0}" -o "{2}/{0}" && chmod 755 {2}/{0}'.format(filename, boostCI_dir, out_dir) + # Common steps for unix systems # Takes the install script (inside the Boost.CI "ci/drone" folder) and the build script (relative to the root .drone folder) def unix_common(install_script, buildscript_to_run): @@ -15,15 +22,14 @@ def unix_common(install_script, buildscript_to_run): download_scripts = [ # Only when not testing Boost.CI 'if [ "$(basename "$DRONE_REPO")" != "boost-ci" ]; then', - # Install script - 'wget "https://github.com/boostorg/boost-ci/raw/master/ci/drone/%s" -P ci/drone' % install_script, + download_from_boostCI(install_script, 'ci/drone'), # Default build script (if not exists) - '[ ! -e .drone/drone.sh ] || wget "https://github.com/boostorg/boost-ci/raw/master/.drone/drone.sh" -P .drone', + '[ ! -e .drone/drone.sh ] || ' + download_from_boostCI('drone.sh', '.drone'), # Chosen build script inside .drone (if a filename/stem and does not exist) - '{{ [ "$(basename "{0}")" == "{0}" ] && [ ! -e .drone/{0}.sh ]; }} || wget "https://github.com/boostorg/boost-ci/raw/master/.drone/{0}.sh" -P .drone'.format(buildscript_to_run), - - 'fi' + '{{ [ "$(basename "{0}")" = "{0}" ] && [ ! -e .drone/{0}.sh ]; }} || '.format(buildscript_to_run) + download_from_boostCI(buildscript_to_run + ".sh", '.drone'), + # Done + 'fi', ] return [ "echo '==================================> SETUP'", From ab1fa82ae28c73ec863aa956eeebcee84b8e8122 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 14 Oct 2022 13:35:11 +0200 Subject: [PATCH 3/3] Fix wrong exists-condition and echo when downloading The conditions were inverted due to combination via `&&`. Use plain `if` to make it easier to read and echo when downloading anything. Also reformat to make it easier to read and append the extension to the script at the start to avoid forgetting it. [skip ci] --- ci/drone/functions.star | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/ci/drone/functions.star b/ci/drone/functions.star index 21d27fed..dd610254 100644 --- a/ci/drone/functions.star +++ b/ci/drone/functions.star @@ -12,36 +12,37 @@ def download_from_boostCI(filename, boostCI_dir, out_dir=None): if out_dir == None: out_dir = boostCI_dir - return 'curl -s -S --retry 10 --create-dirs -L "https://github.com/boostorg/boost-ci/raw/master/{1}/{0}" -o "{2}/{0}" && chmod 755 {2}/{0}'.format(filename, boostCI_dir, out_dir) + url = 'https://github.com/boostorg/boost-ci/raw/master/%s/%s' % (boostCI_dir, filename) + target_path = '%s/%s' % (out_dir, filename) + return 'echo "Downloading {0} to {1}"; curl -s -S --retry 10 --create-dirs -L "{0}" -o "{1}" && chmod 755 {1}'.format(url, target_path) # Common steps for unix systems # Takes the install script (inside the Boost.CI "ci/drone" folder) and the build script (relative to the root .drone folder) def unix_common(install_script, buildscript_to_run): - # Commands to download scripts - # Factored out for ease of writing the commands - download_scripts = [ - # Only when not testing Boost.CI - 'if [ "$(basename "$DRONE_REPO")" != "boost-ci" ]; then', - # Install script - download_from_boostCI(install_script, 'ci/drone'), - # Default build script (if not exists) - '[ ! -e .drone/drone.sh ] || ' + download_from_boostCI('drone.sh', '.drone'), - # Chosen build script inside .drone (if a filename/stem and does not exist) - '{{ [ "$(basename "{0}")" = "{0}" ] && [ ! -e .drone/{0}.sh ]; }} || '.format(buildscript_to_run) + download_from_boostCI(buildscript_to_run + ".sh", '.drone'), - # Done - 'fi', - ] + if not buildscript_to_run.endswith('.sh'): + buildscript_to_run += '.sh' return [ "echo '==================================> SETUP'", "uname -a", "export PATH=/usr/local/bin:$PATH", - '\n'.join(download_scripts), + '\n'.join([ + # Only when not testing Boost.CI + 'if [ "$(basename "$DRONE_REPO")" != "boost-ci" ]; then', + # Install script + download_from_boostCI(install_script, 'ci/drone'), + # Default build script (if not exists) + 'if [ ! -e .drone/drone.sh ]; then %s; fi' % download_from_boostCI('drone.sh', '.drone'), + # Chosen build script inside .drone (if a filename and does not exist) + 'if [ "$(basename "{0}")" = "{0}" ] && [ ! -e .drone/{0} ]; then {1}; fi'.format(buildscript_to_run, download_from_boostCI(buildscript_to_run, '.drone')), + # Done + 'fi', + ]), "echo '==================================> PACKAGES'", "ci/drone/" + install_script, "echo '==================================> INSTALL AND TEST'", - ".drone/%s.sh" % buildscript_to_run, + ".drone/" + buildscript_to_run, ] # Generate pipeline for Linux platform compilers.