Change homebrew -ci- daily jobs to use gz-collections.yaml (#1058)

* Change homebrew -ci- daily jobs to use gz-collections.yaml * Remove brew CI daily jobs from ignition.dsl * Remove relevant code generated in sdformat.dsl * Add a test to check for github orgs expected --------- Signed-off-by: Jose Luis Rivero <[email protected]>
gazebo-tooling · Nov 9, 2023 · d273cae · d273cae
1 parent da4fd32
commit d273cae
Show file tree

Hide file tree

Showing 6 changed files with 64 additions and 89 deletions.
diff --git a/jenkins-scripts/dsl/dsl_checks.bash b/jenkins-scripts/dsl/dsl_checks.bash
@@ -3,6 +3,7 @@
 if [[ -z $(ls -- *.xml) ]]; then
   echo "No .xml file founds. Generate them using:"
   echo "https://github.com/gazebo-tooling/release-tools/blob/master/jenkins-scripts/README.md#L11"
+  exit 1
 fi
 
 not_null=$(grep -3 'null' -- *.xml || true)
@@ -28,6 +29,18 @@ if [[ -n ${abichecker_main} ]]; then
   exit 1
 fi
 
+# Check authorized/expected github organizations and/or repositories
+non_github_orgs=$(grep -R "https?://github.com/" -- *.xml | \
+  grep -E -v '/gazebosim/|/gazebo-tooling/|/gazebo-release/|/gazebo-forks/' | \
+  grep -E -v '/osrf/|/ros-simulation/|/ros2?-gbp/' | \
+  grep -E -v '/j-rivero/ratt/' || true)
+if [[ -n ${non_github_orgs} ]]; then
+  echo "Unexpected github orgs in XML files:"
+  echo "${non_github_orgs}"
+  echo "either update this test or fix the DSL code"
+  exit 1
+fi
+
 # Filter out the previous auto jobs
 filtered_dir=$(mktemp -d)
 cp -- *-abichecker-*.xml "${filtered_dir}"

diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl
@@ -151,6 +151,28 @@ void generate_ci_job(gz_ci_job, lib_name, branch, ci_config,
   }
 }
 
+void generate_brew_ci_job(gz_brew_ci_job, lib_name, branch, ci_config)
+{
+  def script_name_prefix = cleanup_library_name(lib_name)
+  OSRFBrewCompilation.create(gz_brew_ci_job,
+                             is_testing_enabled(lib_name, ci_config),
+                             are_cmake_warnings_enabled(lib_name, ci_config))
+  OSRFGitHub.create(gz_brew_ci_job,
+                    "gazebosim/${lib_name}",
+                    branch,
+                    lib_name)
+  gz_brew_ci_job.with
+  {
+    steps {
+      shell("""\
+            #!/bin/bash -xe
+
+            /bin/bash -xe ./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash "${lib_name}"
+            """.stripIndent())
+      }
+  }
+}
+
 def ciconf_per_lib_index = [:].withDefault { [:] }
 def pkgconf_per_src_index = [:].withDefault { [:] }
 generate_ciconfigs_by_lib(gz_collections_yaml, ciconf_per_lib_index, pkgconf_per_src_index)
@@ -164,15 +186,39 @@ ciconf_per_lib_index.each { lib_name, lib_configs ->
     def branch_names = branches_with_collections.collect { it.branch }.unique()
     def script_name_prefix = cleanup_library_name(lib_name)
     def gz_job_name_prefix = lib_name.replaceAll('-','_')
+    def distro = ci_config.system.version
+    def arch = ci_config.system.arch
     if (ci_config.exclude.all?.contains(lib_name))
       return
     assert(lib_name)
     assert(branch_names)
     assert(ci_config)
 
+    // CI branch jobs (-ci-$branch-) (pulling check every 5 minutes)
+    branches_with_collections.each { branch_and_collection ->
+      def gz_ci_job
+      branch_name = branch_and_collection.branch
+      if (ci_config.system.so == 'linux') {
+        gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-${distro}-${arch}")
+        generate_ci_job(gz_ci_job, lib_name, branch_name, ci_config)
+      } else if (ci_config.system.so == 'darwin') {
+        gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-homebrew-${arch}")
+        generate_brew_ci_job(gz_ci_job, lib_name, branch_name, ci_config)
+      }
+
+      gz_ci_job.with
+      {
+        triggers {
+          scm('@daily')
+        }
+      }
+
+      logging_list['branch_ci'].add(
+        [collection: branch_and_collection.collection,
+         job_name: gz_ci_job.name])
+    } // end_of_branch
+
     if (ci_config.system.so == 'linux') {
-      def distro = ci_config.system.version
-      def arch = ci_config.system.arch
       def pre_setup_script = ci_config.pre_setup_script_hook?.get(lib_name)?.join('\n')
       def extra_cmd = pre_setup_script ?: ""
 
@@ -233,23 +279,6 @@ ciconf_per_lib_index.each { lib_name, lib_configs ->
           } // end of steps
         }  // end of with
       }
-
-      // CI branch jobs (-ci-$branch-) (pulling check every 5 minutes)
-      branches_with_collections.each { branch_and_collection ->
-        branch_name = branch_and_collection.branch
-        def gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-${distro}-${arch}")
-        generate_ci_job(gz_ci_job, lib_name, branch_name, ci_config)
-        gz_ci_job.with
-        {
-          triggers {
-            scm('@daily')
-          }
-        }
-
-        logging_list['branch_ci'].add(
-          [collection: branch_and_collection.collection,
-           job_name: gz_ci_job.name])
-      } // end_of_branch
     } else if (ci_config.system.so == 'darwin') {
       // --------------------------------------------------------------
       def gz_brew_ci_any_job_name = "${gz_job_name_prefix}-ci-pr_any-homebrew-amd64"

diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml
@@ -522,6 +522,7 @@ ci_configs:
       distribution: macOSX
       version: all
       arch: amd64
+    requirements:
     exclude:
       all:
         - ign-citadel

diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl
@@ -599,33 +599,6 @@ gz_software.each { gz_sw ->
     description 'Automatic generated job by DSL jenkins. Stub job for migration, not doing any check'
   }
 
-  // 2. main, release branches
-  all_branches("${software_name}").each { branch ->
-    def gz_brew_ci_job = job("ignition_${software_name}-ci-${branch}-homebrew-amd64")
-    OSRFBrewCompilation.create(gz_brew_ci_job,
-                               enable_testing(software_name),
-                               enable_cmake_warnings(software_name))
-    OSRFGitHub.create(gz_brew_ci_job,
-                              "gazebosim/ign-${software_name}",
-                              "${branch}", "ign-${software_name}")
-    gz_brew_ci_job.with
-    {
-        triggers {
-          scm('@daily')
-        }
-
-        steps {
-          shell("""\
-                #!/bin/bash -xe
-
-                software_name="gz-${software_name}"
-                [[ ${software_name} == 'gazebo' ]] && software_name="gz-sim"
-                /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "\${software_name}"
-                """.stripIndent())
-        }
-    }
-  }
-
   // 3. install jobs to test bottles
   supported_install_pkg_branches(gz_sw).each { major_version, supported_distros ->
     def install_default_job = job("ignition_${gz_sw}${major_version}-install_bottle-homebrew-amd64")

diff --git a/jenkins-scripts/dsl/ros_gz_bridge.dsl b/jenkins-scripts/dsl/ros_gz_bridge.dsl
@@ -57,7 +57,7 @@ bridge_packages.each { pkg ->
         stringParam("ARCH", "amd64", "Architecture to build packages for")
         stringParam('ROS_DISTRO', null,'ROS DISTRO to build pakcages for')
         stringParam("UPLOAD_TO_REPO", 'stable', "OSRF repo name to upload the package to")
-        stringParam('UPSTREAM_RELEASE_REPO', 'https://github.com/j-rivero/ros_ign-release', 'Release repository url')
+        stringParam('UPSTREAM_RELEASE_REPO', 'https://github.com/ros2-gbp/ros_ign-release', 'Release repository url')
     }
 
     // Blocks to control dependencies

diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl
@@ -182,47 +182,7 @@ all_debbuild_versions.each { version ->
 // --------------------------------------------------------------
 // BREW: CI jobs
 
-// 1. ANY job @ SCM/5min
-String ci_build_any_job_name_brew = "sdformat-ci-pr_any-homebrew-amd64"
-def sdformat_brew_ci_any_job = job(ci_build_any_job_name_brew)
-OSRFBrewCompilationAnyGitHub.create(sdformat_brew_ci_any_job,
-                                    "gazebosim/sdformat")
-sdformat_brew_ci_any_job.with
-{
-    steps {
-      shell("""\
-            #!/bin/bash -xe
-
-            /bin/bash -xe ./scripts/jenkins-scripts/sdformat-default-devel-homebrew-amd64.bash
-            """.stripIndent())
-    }
-}
-
-// 2. main in all branches @SCM/daily
-all_versions = sdformat_supported_versions + 'main'
-all_versions.each { version ->
-  def sdformat_brew_ci_job = job("sdformat-ci-${version}-homebrew-amd64")
-  OSRFBrewCompilation.create(sdformat_brew_ci_job)
-  OSRFGitHub.create(sdformat_brew_ci_job, "gazebosim/sdformat",
-                         get_sdformat_branch_name(version))
-
-  sdformat_brew_ci_job.with
-  {
-      triggers {
-        scm('@daily')
-      }
-
-      steps {
-        shell("""\
-              #!/bin/bash -xe
-
-              /bin/bash -xe ./scripts/jenkins-scripts/sdformat-default-devel-homebrew-amd64.bash
-              """.stripIndent())
-      }
-  }
-}
-
-// 3. install jobs to test bottles
+// install jobs to test bottles
 sdformat_supported_versions.each { version ->
   def install_default_job = job("${version}-install_bottle-homebrew-amd64")
   OSRFBrewInstall.create(install_default_job)
@@ -299,5 +259,4 @@ all_versions.each { version ->
 def sdformat_ci_main = pipelineJob("sdformat-ci-manual_any")
 OSRFCIWorkFlowMultiAnyGitHub.create(sdformat_ci_main,
                                     [ci_build_any_job_name_linux,
-                                     ci_build_any_job_name_brew,
                                      ci_build_any_job_name_win7])