From 38030c13260f13937349e0436450d8cfd53e288a Mon Sep 17 00:00:00 2001 From: "Marco A. Gutierrez" Date: Wed, 17 Jan 2024 16:21:34 +0100 Subject: [PATCH 1/8] Add os code name mapping for Ubuntu Noble (#1017) Signed-off-by: Marco A. Gutierrez --- ros_buildfarm/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ros_buildfarm/common.py b/ros_buildfarm/common.py index ea1a05e09..0deb7acfc 100644 --- a/ros_buildfarm/common.py +++ b/ros_buildfarm/common.py @@ -318,6 +318,7 @@ def get_short_os_code_name(os_code_name): 'focal': 'F', 'jammy': 'J', 'jessie': 'J', + 'noble': 'N', 'saucy': 'S', 'stretch': 'S', 'trusty': 'T', From bf2bd689455ef1142b89e83236739aeebd90c37e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Crist=C3=B3bal=20Arroyo?= Date: Wed, 31 Jan 2024 14:49:03 -0500 Subject: [PATCH 2/8] Fix `pytest-rerunfailures` installation by using apt instead of pip (#1020) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Change installation of pytest-rerunfailures to use apt Signed-off-by: Cristóbal Arroyo * Initial commit * Remove os_code_name Signed-off-by: Cristóbal Arroyo --------- Signed-off-by: Cristóbal Arroyo --- ros_buildfarm/templates/devel/devel_task.Dockerfile.em | 6 +++++- .../snippet/install_pytest-rerunfailures.Dockerfile.em | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 ros_buildfarm/templates/snippet/install_pytest-rerunfailures.Dockerfile.em diff --git a/ros_buildfarm/templates/devel/devel_task.Dockerfile.em b/ros_buildfarm/templates/devel/devel_task.Dockerfile.em index 682df198c..95d1a79b3 100644 --- a/ros_buildfarm/templates/devel/devel_task.Dockerfile.em +++ b/ros_buildfarm/templates/devel/devel_task.Dockerfile.em @@ -66,7 +66,11 @@ RUN python3 -u /tmp/wrapper_scripts/apt.py update-install-clean -q -y git python RUN python3 -u /tmp/wrapper_scripts/apt.py update-install-clean -q -y python3-pip @# colcon-core.package_identification.python needs at least setuptools 30.3.0 @# pytest-rerunfailures enables usage of --retest-until-pass -RUN pip3 install -U setuptools==59.6.0 pytest-rerunfailures +@(TEMPLATE( + 'snippet/install_pytest-rerunfailures.Dockerfile.em', + os_name=os_name, +))@ +RUN pip3 install -U setuptools==59.6.0 @[end if]@ RUN python3 -u /tmp/wrapper_scripts/apt.py update-install-clean -q -y ccache diff --git a/ros_buildfarm/templates/snippet/install_pytest-rerunfailures.Dockerfile.em b/ros_buildfarm/templates/snippet/install_pytest-rerunfailures.Dockerfile.em new file mode 100644 index 000000000..727ebf0da --- /dev/null +++ b/ros_buildfarm/templates/snippet/install_pytest-rerunfailures.Dockerfile.em @@ -0,0 +1,4 @@ +@[if os_name == 'debian' or os_name == 'ubuntu']@ +@# python3-pytest-rerunfailures is supported since Ubuntu jammy +RUN for i in 1 2 3; do apt-get update && apt-get install -q -y python3-pytest-rerunfailures && apt-get clean && break || if [ $i -lt 3 ]; then sleep 5; else false; fi; done +@[end if]@ From 09a78f439aa21565a885bc863445f35b9eaa618e Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Fri, 8 Mar 2024 16:16:52 -0600 Subject: [PATCH 3/8] Handle virtual deb packages when querying apt cache (#1023) These virtual packages are supported by the rosdep tool, so should be acceptable for use here as well. --- ros_buildfarm/common.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ros_buildfarm/common.py b/ros_buildfarm/common.py index 0deb7acfc..1347818ca 100644 --- a/ros_buildfarm/common.py +++ b/ros_buildfarm/common.py @@ -172,7 +172,11 @@ def get_distribution_repository_keys(urls, key_files): def get_binary_package_versions(apt_cache, debian_pkg_names): versions = {} for debian_pkg_name in debian_pkg_names: - pkg = apt_cache[debian_pkg_name] + pkg = apt_cache.get(debian_pkg_name) + if not pkg: + prov = apt_cache.get_providing_packages(debian_pkg_name) + assert len(prov) == 1 + pkg = apt_cache[prov[0]] versions[debian_pkg_name] = max(pkg.versions).version return versions From 7c12df582301a99f3d2e877f3c63e69581da5f8e Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Mon, 11 Mar 2024 16:06:00 -0500 Subject: [PATCH 4/8] Fix error message when a package isn't available (#1024) The original behavior was to raise a KeyError with the missing package name, but this was regressed to an assert with no actionable error message by a previous change. Fixes 09a78f4 --- ros_buildfarm/common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ros_buildfarm/common.py b/ros_buildfarm/common.py index 1347818ca..319d524af 100644 --- a/ros_buildfarm/common.py +++ b/ros_buildfarm/common.py @@ -175,6 +175,8 @@ def get_binary_package_versions(apt_cache, debian_pkg_names): pkg = apt_cache.get(debian_pkg_name) if not pkg: prov = apt_cache.get_providing_packages(debian_pkg_name) + if not prov: + raise KeyError("No packages available for '%s'" % (debian_pkg_name,)) assert len(prov) == 1 pkg = apt_cache[prov[0]] versions[debian_pkg_name] = max(pkg.versions).version From 050666fe16f0fbe61f26da0780e08cd74d0bd087 Mon Sep 17 00:00:00 2001 From: "Marco A. Gutierrez" Date: Thu, 14 Mar 2024 09:21:46 +0800 Subject: [PATCH 5/8] Adding break-system-packages pip option for noble in devel_task (#1026) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Remove pip-installed setuptools. colcon requires setuptools 30.3.0[1] and we had previously needed to install it from pypi to get that version on older Ubuntu distros, but since Ubuntu Focal had python3-setuptools 45.2.0 and Debian Buster has 40.8.0 so we've not needed this for most of the last few years. [1]: https://github.com/colcon/colcon-core/blob/master/setup.cfg#L41 --------- Signed-off-by: Marco A. Gutierrez Co-authored-by: Steven! Ragnarök --- ros_buildfarm/templates/devel/devel_task.Dockerfile.em | 3 --- 1 file changed, 3 deletions(-) diff --git a/ros_buildfarm/templates/devel/devel_task.Dockerfile.em b/ros_buildfarm/templates/devel/devel_task.Dockerfile.em index 95d1a79b3..b25ca4722 100644 --- a/ros_buildfarm/templates/devel/devel_task.Dockerfile.em +++ b/ros_buildfarm/templates/devel/devel_task.Dockerfile.em @@ -63,14 +63,11 @@ RUN echo "@today_str" RUN python3 -u /tmp/wrapper_scripts/apt.py update-install-clean -q -y git python3-yaml @[if build_tool == 'colcon']@ -RUN python3 -u /tmp/wrapper_scripts/apt.py update-install-clean -q -y python3-pip -@# colcon-core.package_identification.python needs at least setuptools 30.3.0 @# pytest-rerunfailures enables usage of --retest-until-pass @(TEMPLATE( 'snippet/install_pytest-rerunfailures.Dockerfile.em', os_name=os_name, ))@ -RUN pip3 install -U setuptools==59.6.0 @[end if]@ RUN python3 -u /tmp/wrapper_scripts/apt.py update-install-clean -q -y ccache From f8d8219b7b7566dcccd6b95a7cb880962cdd816e Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Fri, 15 Mar 2024 11:33:26 -0700 Subject: [PATCH 6/8] Working to bring CI back to green (#1015) * drop python 2.7 no longer supported on github actions. * move to noetic and focal from melodic and bionic * pin empy to less than 4 * move from foxy to humble * focal to jammy for humble only * Use Ubuntu Noble for Rolling builds Co-authored-by: Scott K Logan --- .github/workflows/ci.yaml | 105 ++++++++++++++------------------------ setup.py | 2 +- 2 files changed, 40 insertions(+), 67 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 93479b019..01e5de971 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python: ['2.7', '3.6'] + python: ['3.6'] steps: - name: Check out project uses: actions/checkout@v2 @@ -38,8 +38,8 @@ jobs: - name: Run job uses: ./.github/actions/audit with: - ros_distro: melodic - os_code_name: bionic + ros_distro: noetic + os_code_name: focal ros1_config: name: ROS 1 Config Validation @@ -52,39 +52,12 @@ jobs: - name: Validate configration run: validate_config_index.py https://raw.githubusercontent.com/ros-infrastructure/ros_buildfarm_config/production/index.yaml - ros1_devel: - name: ROS 1 Devel - runs-on: ubuntu-20.04 - strategy: - matrix: - python: ['2.7', '3.5', '3.6'] - build_tool: [null] - include: - - python: '3.6' - build_tool: colcon - steps: - - name: Check out project - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: ${{matrix.python}} - - name: Install dependencies - uses: ./.github/actions/setup - - name: Run job - uses: ./.github/actions/devel - with: - ros_distro: melodic - os_code_name: bionic - build_tool: ${{matrix.build_tool}} - repo: roscpp_core - ros1_doc: name: ROS 1 Doc runs-on: ubuntu-20.04 strategy: matrix: - python: ['2.7', '3.5', '3.6'] + python: ['3.5', '3.6'] steps: - name: Check out project uses: actions/checkout@v2 @@ -97,8 +70,8 @@ jobs: - name: Run job uses: ./.github/actions/doc with: - ros_distro: melodic - os_code_name: bionic + ros_distro: noetic + os_code_name: focal repo: roscpp_core ros1_prerelease: @@ -106,7 +79,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python: ['2.7', '3.5', '3.6'] + python: ['3.5', '3.6'] steps: - name: Check out project uses: actions/checkout@v2 @@ -119,8 +92,8 @@ jobs: - name: Run job uses: ./.github/actions/prerelease with: - ros_distro: melodic - os_code_name: bionic + ros_distro: noetic + os_code_name: focal overlay_pkg: roscpp underlay_repos: roscpp_core @@ -129,7 +102,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python: ['2.7', '3.5', '3.6'] + python: ['3.5', '3.6'] steps: - name: Check out project uses: actions/checkout@v2 @@ -148,8 +121,8 @@ jobs: - name: Run job uses: ./.github/actions/prerelease with: - ros_distro: melodic - os_code_name: bionic + ros_distro: noetic + os_code_name: focal source_dir: ${{github.workspace}}/dummy_package ros1_release: @@ -157,7 +130,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python: ['2.7', '3.5', '3.6'] + python: ['3.5', '3.6'] steps: - name: Check out project uses: actions/checkout@v2 @@ -170,8 +143,8 @@ jobs: - name: Run job uses: ./.github/actions/release with: - ros_distro: melodic - os_code_name: bionic + ros_distro: noetic + os_code_name: focal pkg_name: rostime ros1_release_reconfigure: @@ -179,7 +152,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python: ['2.7', '3.5', '3.6'] + python: ['3.5', '3.6'] steps: - name: Check out project uses: actions/checkout@v2 @@ -192,7 +165,7 @@ jobs: - name: Run job uses: ./.github/actions/release_reconfigure with: - ros_distro: melodic + ros_distro: noetic pkg_names: rostime ros1_status_pages: @@ -200,7 +173,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python: ['2.7', '3.5', '3.6'] + python: ['3.5', '3.6'] steps: - name: Check out project uses: actions/checkout@v2 @@ -213,14 +186,14 @@ jobs: - name: Run job uses: ./.github/actions/status_pages with: - ros_distro: melodic + ros_distro: noetic ros1_sync_criteria_check: name: ROS 1 Sync Criteria Check runs-on: ubuntu-20.04 strategy: matrix: - python: ['2.7', '3.5', '3.6'] + python: ['3.5', '3.6'] steps: - name: Check out project uses: actions/checkout@v2 @@ -229,15 +202,15 @@ jobs: - name: Run job uses: ./.github/actions/sync_criteria_check with: - ros_distro: melodic - os_code_name: bionic + ros_distro: noetic + os_code_name: focal ros1_trigger: name: ROS 1 Trigger runs-on: ubuntu-20.04 strategy: matrix: - python: ['2.7', '3.5', '3.6'] + python: ['3.5', '3.6'] steps: - name: Check out project uses: actions/checkout@v2 @@ -246,7 +219,7 @@ jobs: - name: Run job uses: ./.github/actions/trigger with: - ros_distro: melodic + ros_distro: noetic ros2_audit: name: ROS 2 Audit @@ -260,8 +233,8 @@ jobs: uses: ./.github/actions/audit with: config_url: https://raw.githubusercontent.com/ros2/ros_buildfarm_config/ros2/index.yaml - ros_distro: foxy - os_code_name: focal + ros_distro: humble + os_code_name: jammy ros2_ci: name: ROS 2 CI @@ -276,24 +249,24 @@ jobs: uses: ./.github/actions/ci with: config_url: https://raw.githubusercontent.com/ros2/ros_buildfarm_config/ros2/index.yaml - ros_distro: foxy - os_code_name: focal + ros_distro: humble + os_code_name: jammy package_selection_args: --packages-up-to ament_flake8 - name: Run job 2 id: underlay2 uses: ./.github/actions/ci with: config_url: https://raw.githubusercontent.com/ros2/ros_buildfarm_config/ros2/index.yaml - ros_distro: foxy - os_code_name: focal + ros_distro: humble + os_code_name: jammy underlay_dirs: ${{steps.underlay1.outputs.install_dir}} package_selection_args: --packages-skip-up-to ament_flake8 --packages-up-to ament_pep257 - name: Run job 3 uses: ./.github/actions/ci with: config_url: https://raw.githubusercontent.com/ros2/ros_buildfarm_config/ros2/index.yaml - ros_distro: foxy - os_code_name: focal + ros_distro: humble + os_code_name: jammy underlay_dirs: ${{steps.underlay1.outputs.install_dir}} ${{steps.underlay2.outputs.install_dir}} package_selection_args: --packages-skip-up-to ament_flake8 ament_pep257 --packages-up-to ament_cmake_ros @@ -320,8 +293,8 @@ jobs: uses: ./.github/actions/devel with: config_url: https://raw.githubusercontent.com/ros2/ros_buildfarm_config/ros2/index.yaml - ros_distro: foxy - os_code_name: focal + ros_distro: humble + os_code_name: jammy repo: rcutils ros2_doc: @@ -337,7 +310,7 @@ jobs: with: config_url: https://raw.githubusercontent.com/ros2/ros_buildfarm_config/ros2/index.yaml ros_distro: rolling - os_code_name: jammy + os_code_name: noble repo: rcl output_directory: ws/docs_output @@ -353,8 +326,8 @@ jobs: uses: ./.github/actions/prerelease with: config_url: https://raw.githubusercontent.com/ros2/ros_buildfarm_config/ros2/index.yaml - ros_distro: foxy - os_code_name: focal + ros_distro: humble + os_code_name: jammy overlay_pkg: rcutils underlay_repos: ament_cmake_ros @@ -371,7 +344,7 @@ jobs: with: config_url: https://raw.githubusercontent.com/ros2/ros_buildfarm_config/ros2/index.yaml ros_distro: rolling - os_code_name: jammy + os_code_name: noble pkg_name: rcutils ros2_release_reconfigure: @@ -421,7 +394,7 @@ jobs: with: config_url: https://raw.githubusercontent.com/ros2/ros_buildfarm_config/ros2/index.yaml ros_distro: rolling - os_code_name: jammy + os_code_name: noble ros2_sync_criteria_check_rpm: name: ROS 2 Sync Criteria Check (RPM) diff --git a/setup.py b/setup.py index 82a41f3a2..a357f93e3 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ 'include_package_data': True, 'zip_safe': False, 'install_requires': [ - 'empy', + 'empy<4', 'PyYAML'], 'extras_require': { 'test': [ From bde772137570645b1d027d3016df7f38c1fcbdcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Mon, 18 Mar 2024 11:24:17 -0700 Subject: [PATCH 7/8] Mount rosdoc2 source directory read-write. (#1031) Our deprecated out-of-tree build option has been removed which means that we now need to host our rosdoc2 sources in a read-write location to allow build artifacts to be generated in-tree. Mounting read-write shouldn't matter overmuch because we delete and re-clone rosdoc2 each time the job runs[1]. [1]: https://github.com/ros-infrastructure/ros_buildfarm/blob/f8d8219b7b7566dcccd6b95a7cb880962cdd816e/ros_buildfarm/templates/doc/rosdoc2_job.xml.em#L94 --- ros_buildfarm/scripts/doc/build_rosdoc2.py | 3 +-- ros_buildfarm/templates/doc/rosdoc2_job.xml.em | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ros_buildfarm/scripts/doc/build_rosdoc2.py b/ros_buildfarm/scripts/doc/build_rosdoc2.py index a66af9bad..25bb7cee5 100644 --- a/ros_buildfarm/scripts/doc/build_rosdoc2.py +++ b/ros_buildfarm/scripts/doc/build_rosdoc2.py @@ -49,8 +49,7 @@ def main(argv=sys.argv[1:]): '-m', 'pip', 'install', - '--no-warn-script-location', - '--use-deprecated=out-of-tree-build', + '--break-system-packages', '.'], cwd=args.rosdoc2_dir) if pip_rc: diff --git a/ros_buildfarm/templates/doc/rosdoc2_job.xml.em b/ros_buildfarm/templates/doc/rosdoc2_job.xml.em index d6654b91d..87ccede9b 100644 --- a/ros_buildfarm/templates/doc/rosdoc2_job.xml.em +++ b/ros_buildfarm/templates/doc/rosdoc2_job.xml.em @@ -178,7 +178,7 @@ else: ' --rm ' + ' --cidfile=$WORKSPACE/docker_doc/docker.cid' + ' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' + - ' -v $WORKSPACE/rosdoc2:/tmp/rosdoc2:ro' + + ' -v $WORKSPACE/rosdoc2:/tmp/rosdoc2' + ' -v $WORKSPACE/ws:/tmp/ws' + ' rosdoc2.%s_%s' % (rosdistro_name, doc_repo_spec.name.lower()), 'echo "# END SECTION"', From 3634e05948836b6e72a13db9e1dd2a4132a940a9 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Mon, 18 Mar 2024 16:28:32 -0500 Subject: [PATCH 8/8] Set a sane HOME for binarypkg jobs (#1013) The comment above this invocation indicates that apt needs HOME to be set to something, but the variable is set to an empty string. This is even more critical for running the buildfarm using rootless Podman, because `/` is no longer writable and seems to be the default of HOME is empty. --- ros_buildfarm/templates/release/deb/binarypkg_job.xml.em | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ros_buildfarm/templates/release/deb/binarypkg_job.xml.em b/ros_buildfarm/templates/release/deb/binarypkg_job.xml.em index 70e364ea3..41e1ee362 100644 --- a/ros_buildfarm/templates/release/deb/binarypkg_job.xml.em +++ b/ros_buildfarm/templates/release/deb/binarypkg_job.xml.em @@ -168,7 +168,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config 'docker run' + ' --rm ' + ' --cidfile=$WORKSPACE/docker_build_binarydeb/docker.cid' + - ' -e=HOME=' + + ' -e=HOME=/home/buildfarm' + ' -e=TRAVIS=$TRAVIS' + ' --net=host' + ' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' +