From c5d6e3a92d3e7f2cb883e09dc5bebdcb3eb9e9d4 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Thu, 12 Dec 2024 15:29:33 -0600 Subject: [PATCH 1/2] Plumb 'build_environment_variables' into doc jobs This buildfarm configuration variable has always been included for doc job configurations, but has been left unused. With this change, those variables are now used in 'doc' and 'rosdoc2' jobs in the same way that they're used for devel, CI, and release jobs. --- ros_buildfarm/doc_job.py | 7 +++++++ ros_buildfarm/scripts/doc/create_doc_task_generator.py | 10 ++++++++-- .../scripts/doc/create_rosdoc2_task_generator.py | 4 ++++ ros_buildfarm/scripts/doc/run_doc_job.py | 2 ++ ros_buildfarm/scripts/doc/run_rosdoc2_job.py | 2 ++ .../templates/doc/doc_create_task.Dockerfile.em | 1 + ros_buildfarm/templates/doc/doc_job.xml.em | 1 + .../templates/doc/rosdoc2_create_task.Dockerfile.em | 1 + ros_buildfarm/templates/doc/rosdoc2_job.xml.em | 1 + ros_buildfarm/templates/doc/rosdoc2_task.Dockerfile.em | 5 +++++ 10 files changed, 32 insertions(+), 2 deletions(-) diff --git a/ros_buildfarm/doc_job.py b/ros_buildfarm/doc_job.py index ced8f85ad..cddb42d36 100644 --- a/ros_buildfarm/doc_job.py +++ b/ros_buildfarm/doc_job.py @@ -269,6 +269,12 @@ def _get_doc_job_config( repository_args, script_generating_key_files = \ get_repositories_and_script_generating_key_files(build_file=build_file) + build_environment_variables = [] + if build_file.build_environment_variables: + build_environment_variables = [ + '%s=%s' % (var, value) + for var, value in sorted(build_file.build_environment_variables.items())] + maintainer_emails = set([]) if build_file.notify_maintainers and dist_cache and repo_name and \ repo_name in dist_cache.distribution_file.repositories: @@ -311,6 +317,7 @@ def _get_doc_job_config( 'arch': arch, 'build_tool': build_file.build_tool, 'repository_args': repository_args, + 'build_environment_variables': build_environment_variables, 'upload_user': build_file.upload_user, 'upload_host': build_file.upload_host, diff --git a/ros_buildfarm/scripts/doc/create_doc_task_generator.py b/ros_buildfarm/scripts/doc/create_doc_task_generator.py index a681f6bdc..d4a1daeb0 100644 --- a/ros_buildfarm/scripts/doc/create_doc_task_generator.py +++ b/ros_buildfarm/scripts/doc/create_doc_task_generator.py @@ -31,6 +31,7 @@ add_argument_distribution_repository_key_files from ros_buildfarm.argument import add_argument_distribution_repository_urls from ros_buildfarm.argument import add_argument_dockerfile_dir +from ros_buildfarm.argument import add_argument_env_vars from ros_buildfarm.argument import add_argument_force from ros_buildfarm.argument import add_argument_output_dir from ros_buildfarm.argument import add_argument_repository_name @@ -108,6 +109,7 @@ def main(argv=sys.argv[1:]): add_argument_vcs_information(parser) add_argument_distribution_repository_urls(parser) add_argument_distribution_repository_key_files(parser) + add_argument_env_vars(parser) add_argument_force(parser) add_argument_output_dir(parser, required=True) add_argument_dockerfile_dir(parser) @@ -578,6 +580,11 @@ def main(argv=sys.argv[1:]): debian_pkg_names.remove(debian_pkg_name) print('# END SUBSECTION') + env_vars = { + **args.env_vars, + 'ROS_PYTHON_VERSION': condition_context['ROS_PYTHON_VERSION'], + } + # generate Dockerfile data = { 'os_name': args.os_name, @@ -590,8 +597,7 @@ def main(argv=sys.argv[1:]): args.distribution_repository_urls, args.distribution_repository_key_files), - 'environment_variables': [ - 'ROS_PYTHON_VERSION={}'.format(condition_context['ROS_PYTHON_VERSION'])], + 'environment_variables': ['%s=%s' % key_value for key_value in env_vars.items()], 'rosdistro_name': args.rosdistro_name, diff --git a/ros_buildfarm/scripts/doc/create_rosdoc2_task_generator.py b/ros_buildfarm/scripts/doc/create_rosdoc2_task_generator.py index 498e028a4..dab3a3126 100644 --- a/ros_buildfarm/scripts/doc/create_rosdoc2_task_generator.py +++ b/ros_buildfarm/scripts/doc/create_rosdoc2_task_generator.py @@ -19,6 +19,7 @@ add_argument_distribution_repository_key_files from ros_buildfarm.argument import add_argument_distribution_repository_urls from ros_buildfarm.argument import add_argument_dockerfile_dir +from ros_buildfarm.argument import add_argument_env_vars from ros_buildfarm.common import get_distribution_repository_keys from ros_buildfarm.common import get_user_id from ros_buildfarm.templates import create_dockerfile @@ -45,6 +46,7 @@ def main(argv=sys.argv[1:]): help="The architecture (e.g. 'amd64')") add_argument_distribution_repository_urls(parser) add_argument_distribution_repository_key_files(parser) + add_argument_env_vars(parser) add_argument_dockerfile_dir(parser) args = parser.parse_args(argv) @@ -60,6 +62,8 @@ def main(argv=sys.argv[1:]): args.distribution_repository_urls, args.distribution_repository_key_files), + 'environment_variables': ['%s=%s' % key_value for key_value in args.env_vars.items()], + 'uid': get_user_id(), } create_dockerfile('doc/rosdoc2_task.Dockerfile.em', data, args.dockerfile_dir) diff --git a/ros_buildfarm/scripts/doc/run_doc_job.py b/ros_buildfarm/scripts/doc/run_doc_job.py index 02194c5f2..7950cf4df 100644 --- a/ros_buildfarm/scripts/doc/run_doc_job.py +++ b/ros_buildfarm/scripts/doc/run_doc_job.py @@ -26,6 +26,7 @@ add_argument_distribution_repository_key_files from ros_buildfarm.argument import add_argument_distribution_repository_urls from ros_buildfarm.argument import add_argument_dockerfile_dir +from ros_buildfarm.argument import add_argument_env_vars from ros_buildfarm.argument import add_argument_force from ros_buildfarm.argument import add_argument_os_code_name from ros_buildfarm.argument import add_argument_os_name @@ -56,6 +57,7 @@ def main(argv=sys.argv[1:]): add_argument_custom_rosdep_urls(parser) add_argument_custom_rosdep_update_options(parser) add_argument_force(parser) + add_argument_env_vars(parser) add_argument_dockerfile_dir(parser) args = parser.parse_args(argv) diff --git a/ros_buildfarm/scripts/doc/run_rosdoc2_job.py b/ros_buildfarm/scripts/doc/run_rosdoc2_job.py index 44d001857..064e4bdf1 100644 --- a/ros_buildfarm/scripts/doc/run_rosdoc2_job.py +++ b/ros_buildfarm/scripts/doc/run_rosdoc2_job.py @@ -21,6 +21,7 @@ add_argument_distribution_repository_key_files from ros_buildfarm.argument import add_argument_distribution_repository_urls from ros_buildfarm.argument import add_argument_dockerfile_dir +from ros_buildfarm.argument import add_argument_env_vars from ros_buildfarm.argument import add_argument_os_code_name from ros_buildfarm.argument import add_argument_os_name from ros_buildfarm.common import get_distribution_repository_keys @@ -36,6 +37,7 @@ def main(argv=sys.argv[1:]): add_argument_arch(parser) add_argument_distribution_repository_urls(parser) add_argument_distribution_repository_key_files(parser) + add_argument_env_vars(parser) add_argument_dockerfile_dir(parser) args = parser.parse_args(argv) diff --git a/ros_buildfarm/templates/doc/doc_create_task.Dockerfile.em b/ros_buildfarm/templates/doc/doc_create_task.Dockerfile.em index 385d59e23..a0ec51bb9 100644 --- a/ros_buildfarm/templates/doc/doc_create_task.Dockerfile.em +++ b/ros_buildfarm/templates/doc/doc_create_task.Dockerfile.em @@ -76,6 +76,7 @@ cmds = [ ' --vcs-info "%s"' % vcs_info + \ ' --distribution-repository-urls ' + ' '.join(distribution_repository_urls) + \ ' --distribution-repository-key-files ' + ' ' .join(['/tmp/keys/%d.key' % i for i in range(len(distribution_repository_keys))]) + \ + ' --env-vars ' + ' ' .join(['%s=%s' % key_value for key_value in env_vars.items() + \ (' --force' if force else '') + \ ' --output-dir /tmp/generated_documentation' + \ ' --dockerfile-dir /tmp/docker_doc', diff --git a/ros_buildfarm/templates/doc/doc_job.xml.em b/ros_buildfarm/templates/doc/doc_job.xml.em index f70a9b687..3a001f788 100644 --- a/ros_buildfarm/templates/doc/doc_job.xml.em +++ b/ros_buildfarm/templates/doc/doc_job.xml.em @@ -168,6 +168,7 @@ else: ' ' + os_code_name + ' ' + arch + ' --build-tool ' + build_tool + + ' --env-vars ' + ' '.join([v.replace('$', '\\$',) for v in build_environment_variables]) + ' --vcs-info "%s %s %s"' % (doc_repo_spec.type, doc_repo_spec.version if doc_repo_spec.version is not None else '', doc_repo_spec.url) + ' ' + ' '.join(repository_args) + ' $FORCE_FLAG' + diff --git a/ros_buildfarm/templates/doc/rosdoc2_create_task.Dockerfile.em b/ros_buildfarm/templates/doc/rosdoc2_create_task.Dockerfile.em index b4f53d3e2..e7fd1903c 100644 --- a/ros_buildfarm/templates/doc/rosdoc2_create_task.Dockerfile.em +++ b/ros_buildfarm/templates/doc/rosdoc2_create_task.Dockerfile.em @@ -58,6 +58,7 @@ cmds = [ ' --arch ' + arch + \ ' --distribution-repository-urls ' + ' '.join(distribution_repository_urls) + \ ' --distribution-repository-key-files ' + ' ' .join(['/tmp/keys/%d.key' % i for i in range(len(distribution_repository_keys))]) + \ + ' --env-vars ' + ' ' .join(['%s=%s' % key_value for key_value in env_vars.items() + \ ' --dockerfile-dir /tmp/docker_doc', ] }@ diff --git a/ros_buildfarm/templates/doc/rosdoc2_job.xml.em b/ros_buildfarm/templates/doc/rosdoc2_job.xml.em index 4105c162f..96f7668a2 100644 --- a/ros_buildfarm/templates/doc/rosdoc2_job.xml.em +++ b/ros_buildfarm/templates/doc/rosdoc2_job.xml.em @@ -129,6 +129,7 @@ else: ' ' + os_code_name + ' ' + arch + ' ' + ' '.join(repository_args) + + ' --env-vars ' + ' '.join([v.replace('$', '\\$',) for v in build_environment_variables]) + ' --dockerfile-dir $WORKSPACE/docker_generating_docker', 'echo "# END SECTION"', '', diff --git a/ros_buildfarm/templates/doc/rosdoc2_task.Dockerfile.em b/ros_buildfarm/templates/doc/rosdoc2_task.Dockerfile.em index c68c88111..cf9558b64 100644 --- a/ros_buildfarm/templates/doc/rosdoc2_task.Dockerfile.em +++ b/ros_buildfarm/templates/doc/rosdoc2_task.Dockerfile.em @@ -21,6 +21,11 @@ ENV DEBIAN_FRONTEND noninteractive uid=uid, ))@ +@(TEMPLATE( + 'snippet/set_environment_variables.Dockerfile.em', + environment_variables=environment_variables, +))@ + @(TEMPLATE( 'snippet/add_distribution_repositories.Dockerfile.em', distribution_repository_keys=distribution_repository_keys, From b5fc817c9fc410bffbc4afdcb1bed10154da035e Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Thu, 12 Dec 2024 15:38:11 -0600 Subject: [PATCH 2/2] fixup! Plumb 'build_environment_variables' into doc jobs --- ros_buildfarm/templates/doc/doc_create_task.Dockerfile.em | 2 +- ros_buildfarm/templates/doc/rosdoc2_create_task.Dockerfile.em | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ros_buildfarm/templates/doc/doc_create_task.Dockerfile.em b/ros_buildfarm/templates/doc/doc_create_task.Dockerfile.em index a0ec51bb9..51f283116 100644 --- a/ros_buildfarm/templates/doc/doc_create_task.Dockerfile.em +++ b/ros_buildfarm/templates/doc/doc_create_task.Dockerfile.em @@ -76,7 +76,7 @@ cmds = [ ' --vcs-info "%s"' % vcs_info + \ ' --distribution-repository-urls ' + ' '.join(distribution_repository_urls) + \ ' --distribution-repository-key-files ' + ' ' .join(['/tmp/keys/%d.key' % i for i in range(len(distribution_repository_keys))]) + \ - ' --env-vars ' + ' ' .join(['%s=%s' % key_value for key_value in env_vars.items() + \ + ' --env-vars ' + ' ' .join(['%s=%s' % key_value for key_value in env_vars.items()]) + \ (' --force' if force else '') + \ ' --output-dir /tmp/generated_documentation' + \ ' --dockerfile-dir /tmp/docker_doc', diff --git a/ros_buildfarm/templates/doc/rosdoc2_create_task.Dockerfile.em b/ros_buildfarm/templates/doc/rosdoc2_create_task.Dockerfile.em index e7fd1903c..47ba72aac 100644 --- a/ros_buildfarm/templates/doc/rosdoc2_create_task.Dockerfile.em +++ b/ros_buildfarm/templates/doc/rosdoc2_create_task.Dockerfile.em @@ -58,7 +58,7 @@ cmds = [ ' --arch ' + arch + \ ' --distribution-repository-urls ' + ' '.join(distribution_repository_urls) + \ ' --distribution-repository-key-files ' + ' ' .join(['/tmp/keys/%d.key' % i for i in range(len(distribution_repository_keys))]) + \ - ' --env-vars ' + ' ' .join(['%s=%s' % key_value for key_value in env_vars.items() + \ + ' --env-vars ' + ' ' .join(['%s=%s' % key_value for key_value in env_vars.items()]) + \ ' --dockerfile-dir /tmp/docker_doc', ] }@