Merge branch 'master' of github.com:adoptium/temurin-build into gaction

andrew-m-leonard · Apr 26, 2024 · 7fd22ff · 7fd22ff
2 parents 6399253 + fae0d1a
commit 7fd22ff
Show file tree

Hide file tree

Showing 26 changed files with 120 additions and 72 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -409,7 +409,7 @@ jobs:
          path: workspace/target/*
 
      - name: Restore build artifacts
-       uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4.1.6
+       uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
        with:
          name: ${{matrix.version}}-${{matrix.os}}-${{matrix.variant}}
          path: ~/${{matrix.version}}-${{matrix.os}}-${{matrix.variant}}

diff --git a/.licenserc.yaml b/.licenserc.yaml
@@ -46,5 +46,6 @@ header:
     - 'sbin/*.template'
     - '.github/linters/*'
     - 'cyclonedx-lib/getDependencies'
+    - 'cyclonedx-lib/dependency_data/**'
     - 'makejdk-any-platform.1'
     - 'serverTimestamp.properties'
diff --git a/cyclonedx-lib/README.md b/cyclonedx-lib/README.md
@@ -34,13 +34,12 @@ formulation section (mostly because the author of this added such a section
 recently)
 
 1. Check that the version of CycloneDX you are using supports the
-    functionality you want.  If not, it will need to be updated. To do this
-    you need to do three steps [Sample PR](https://github.com/adoptium/temurin-build/pull/3558/files):
-    - Update [getDependencies](https://github.com/adoptium/temurin-build/blob/master/cyclonedx-lib/getDependencies) with the new version and SHA
+    functionality you want. If not, it will need to be updated. To do this
+    you need to do two steps:
+    - Update the [sha and version files](https://github.com/adoptium/temurin-build/blob/master/cyclonedx-lib/dependency_data) for each jar you plan to change.
     - Ensure [build.getDependency](https://ci.adoptium.net/job/build.getDependency/) is run to pick up the new version
-    - Update [build.xml](https://github.com/adoptium/temurin-build/blob/master/cyclonedx-lib/build.xml) with the new SHA to allow it to be downloaded successfully during the build (This change can be made along with the getDependencies one if     you co--ordinate the update appopriately
-2. If the build and java code does not already have support for the CycloneDX functionality that you need the follow ([Sample PR](https://github.com/adoptium/temurin-build/blob/master/cyclonedx-lib/build.xml))
-    - Updates to [cyclonedx-lib/TemurinGenSBOM.java])https://github.com/adoptium/temurin-build/blob/master/sbin/common/sbom.sh) to add a new parameter, a new function to implement it, the call to that function from the `switch` functionality in the `main` function
+2. If the build and java code does not already have support for the CycloneDX functionality that you need, then follow these steps ([Sample PR](https://github.com/adoptium/temurin-build/pull/3538))
+    - Updates to [cyclonedx-lib/TemurinGenSBOM.java](https://github.com/adoptium/temurin-build/blob/master/sbin/common/sbom.sh) to add a new parameter, a new function to implement it, the call to that function from the `switch` functionality in the `main` function
     - Update [cyclonedx-lib/build.xml](https://github.com/adoptium/temurin-build/blob/master/cyclonedx-lib/build.xml) to add tests for the new functionality
     - Add a new function to [sbin/common/sbom.sh](https://github.com/adoptium/temurin-build/blob/master/sbin/common/sbom.sh) to add the fields you need
     - Updates to [sbin/build.sh](https://github.com/adoptium/temurin-build/blob/master/sbin/build.sh) to invoke the new function(s) in sbom.sh

diff --git a/cyclonedx-lib/build.xml b/cyclonedx-lib/build.xml
@@ -20,9 +20,12 @@
 // jscpd:ignore-start
 -->
 
-        <!-- All other dependency versions are now listed in the ./getDependencies file -->
+        <!-- All other dependency versions are listed in the relevant ./dependency_data/versions files -->
         <property name="openkeystore-version" value="1.0.0"/>
 
+        <!-- When retrieving sbom dependency jars with default settings, we will look here. -->
+        <property name="sbom_dependency_default_location" value="https://ci.adoptium.net/view/all/job/build.getDependency/lastSuccessfulBuild/artifact/sbom_dependencies"/>
+
         <!-- classpath for running application -->
         <property name="classpath" value="build/jar/temurin-gen-sbom.jar:build/jar/cyclonedx-core-java.jar:build/jar/jackson-core.jar:build/jar/jackson-dataformat-xml.jar:build/jar/jackson-databind.jar:build/jar/jackson-annotations.jar:build/jar/json-schema.jar:build/jar/commons-codec.jar:build/jar/commons-io.jar:build/jar/github-package-url.jar:build/jar/webpki.org-libext-1.00.jar:build/jar/temurin-sign-sbom.jar"/>
 
@@ -41,20 +44,12 @@
 
         <target name="download-cyclonedx" unless="cyclonedx_available">
                 <echo message="Downloading cyclonedx-core-java"/>
-                <download-file
-                  destfile="cyclonedx-core-java.jar"
-                  checksum="ecc371d12808dfe76047f87f8235665d74dd6cf8ec12c41d052715a3fd79e0b5"
-                  srcurl="https://ci.adoptium.net/view/all/job/build.getDependency/lastSuccessfulBuild/artifact/sbom_dependencies/cyclonedx-core-java.jar"
-                  />
-	      </target>
+                <download-file-with-default-options destfile="cyclonedx-core-java.jar"/>
+	    </target>
 
         <target name="download-jackson-core" unless="jackson-core_available">
                 <echo message="Downloading jackson-core"/>
-                <download-file
-                  checksum="b5d37a77c88277b97e3593c8740925216c06df8e4172bbde058528df04ad3e7a"
-                  destfile="jackson-core.jar"
-                  srcurl="https://ci.adoptium.net/view/all/job/build.getDependency/lastSuccessfulBuild/artifact/sbom_dependencies/jackson-core.jar"
-                  />
+                <download-file-with-default-options destfile="jackson-core.jar"/>
         </target>
 
         <target name="clone-and-build-openkeystore" unless="openkeystore_available">
@@ -91,58 +86,37 @@
 
         <target name="download-jackson-dataformat-xml" unless="jackson-dataformat_available">
                 <echo message="Downloading jackson-dataformat-xml"/>
-                <download-file
-                  checksum="edbda6c775a36049cf0088b111ab958cca0dc70cb9326918d6cf153cb3fa426b"
-                  destfile="jackson-dataformat-xml.jar"
-                  srcurl="https://ci.adoptium.net/view/all/job/build.getDependency/lastSuccessfulBuild/artifact/sbom_dependencies/jackson-dataformat-xml.jar"/>
+                <download-file-with-default-options destfile="jackson-dataformat-xml.jar"/>
         </target>
 
         <target name="download-jackson-databind" unless="jackson-databind_available">
                 <echo message="Downloading jackson-databind"/>
-                <download-file
-                  checksum="501d3abce4d18dcc381058ec593c5b94477906bba6efbac14dae40a642f77424"
-                  destfile="jackson-databind.jar"
-                  srcurl="https://ci.adoptium.net/view/all/job/build.getDependency/lastSuccessfulBuild/artifact/sbom_dependencies/jackson-databind.jar"/>
+                <download-file-with-default-options destfile="jackson-databind.jar"/>
         </target>
 
         <target name="download-jackson-annotations" unless="jackson-annotations_available">
                 <echo message="Downloading jackson-annotations"/>
-                <download-file
-                  checksum="2c6869d505cf60dc066734b7d50339f975bd3adc635e26a78abb71acb4473c0d"
-                  destfile="jackson-annotations.jar"
-                  srcurl="https://ci.adoptium.net/view/all/job/build.getDependency/lastSuccessfulBuild/artifact/sbom_dependencies/jackson-annotations.jar"/>
+                <download-file-with-default-options destfile="jackson-annotations.jar"/>
         </target>
 
         <target name="download-json-schema" unless="json-schema_available">
                 <echo message="Downloading json-schema"/>
-                <download-file
-                  checksum="968991e5718520cdd7b224770f790cf2c241cddf64d10a36c21f9f8b4a15e79c"
-                  destfile="json-schema.jar"
-                  srcurl="https://ci.adoptium.net/view/all/job/build.getDependency/lastSuccessfulBuild/artifact/sbom_dependencies/json-schema.jar"/>
+                <download-file-with-default-options destfile="json-schema.jar"/>
         </target>
 
         <target name="download-commons-codec" unless="commons-codec_available">
                 <echo message="Downloading commons-codec"/>
-                <download-file
-                  checksum="b3e9f6d63a790109bf0d056611fbed1cf69055826defeb9894a71369d246ed63"
-                  destfile="commons-codec.jar"
-                  srcurl="https://ci.adoptium.net/view/all/job/build.getDependency/lastSuccessfulBuild/artifact/sbom_dependencies/commons-codec.jar"/>
+                <download-file-with-default-options destfile="commons-codec.jar"/>
         </target>
 
         <target name="download-commons-io" unless="commons-io_available">
                 <echo message="Downloading commons-io"/>
-                <download-file
-                  checksum="961b2f6d87dbacc5d54abf45ab7a6e2495f89b75598962d8c723cea9bc210908"
-                  destfile="commons-io.jar"
-                  srcurl="https://ci.adoptium.net/view/all/job/build.getDependency/lastSuccessfulBuild/artifact/sbom_dependencies/commons-io.jar"/>
+                <download-file-with-default-options destfile="commons-io.jar"/>
         </target>
 
         <target name="download-github-package-url" unless="github-package-url_available">
                 <echo message="Downloading github-package-url"/>
-                <download-file
-                  checksum="8e23280221afd1e6561d433dfb133252cd287167acb0eca5a991667118ff10a2"
-                  destfile="github-package-url.jar"
-                  srcurl="https://ci.adoptium.net/view/all/job/build.getDependency/lastSuccessfulBuild/artifact/sbom_dependencies/github-package-url.jar"/>
+                <download-file-with-default-options destfile="github-package-url.jar"/>
         </target>
 
 	      <target name="build" depends="dep-checks, download-cyclonedx, download-jackson-core, download-jackson-dataformat-xml, download-jackson-databind, download-jackson-annotations, download-json-schema, download-commons-codec, download-commons-io, download-github-package-url, compile, jar">
@@ -505,6 +479,20 @@
 
         </target>
 
+	<macrodef name="download-file-with-default-options" description="Use curl to download a file and retrieve its checksum from default locations.">
+		<attribute name="destfile" description="File name of the downloaded file"/>
+		<sequential>
+			<echo message="Executing macro download-file-with-default-options"/>
+			<echo message="Fetching checksum prior to downloading file."/>
+			<loadfile property="checksum-for-@{destfile}" srcFile="dependency_data/shas/@{destfile}.sha256"/>
+			<echo message="Checksum found: ${checksum-for-@{destfile}}"/>
+			<download-file
+				checksum="${checksum-for-@{destfile}}"
+				destfile="@{destfile}"
+				srcurl="${sbom_dependency_default_location}/@{destfile}"/>
+		</sequential>
+	</macrodef>
+
 	<macrodef name="download-file" description="Use curl to download a file">
 		<attribute name="srcurl" description="URL of file to download"/>
 		<attribute name="destdir" default="build/jar" description="Directory in which to place the downloaded file"/>

diff --git a/cyclonedx-lib/dependency_data/shas/commons-codec.jar.sha256 b/cyclonedx-lib/dependency_data/shas/commons-codec.jar.sha256
@@ -0,0 +1 @@
+b3e9f6d63a790109bf0d056611fbed1cf69055826defeb9894a71369d246ed63
diff --git a/cyclonedx-lib/dependency_data/shas/commons-io.jar.sha256 b/cyclonedx-lib/dependency_data/shas/commons-io.jar.sha256
@@ -0,0 +1 @@
+961b2f6d87dbacc5d54abf45ab7a6e2495f89b75598962d8c723cea9bc210908
diff --git a/cyclonedx-lib/dependency_data/shas/cyclonedx-core-java.jar.sha256 b/cyclonedx-lib/dependency_data/shas/cyclonedx-core-java.jar.sha256
@@ -0,0 +1 @@
+ecc371d12808dfe76047f87f8235665d74dd6cf8ec12c41d052715a3fd79e0b5
diff --git a/cyclonedx-lib/dependency_data/shas/github-package-url.jar.sha256 b/cyclonedx-lib/dependency_data/shas/github-package-url.jar.sha256
@@ -0,0 +1 @@
+8e23280221afd1e6561d433dfb133252cd287167acb0eca5a991667118ff10a2
diff --git a/cyclonedx-lib/dependency_data/shas/jackson-annotations.jar.sha256 b/cyclonedx-lib/dependency_data/shas/jackson-annotations.jar.sha256
@@ -0,0 +1 @@
+2c6869d505cf60dc066734b7d50339f975bd3adc635e26a78abb71acb4473c0d
diff --git a/cyclonedx-lib/dependency_data/shas/jackson-core.jar.sha256 b/cyclonedx-lib/dependency_data/shas/jackson-core.jar.sha256
@@ -0,0 +1 @@
+b5d37a77c88277b97e3593c8740925216c06df8e4172bbde058528df04ad3e7a
diff --git a/cyclonedx-lib/dependency_data/shas/jackson-databind.jar.sha256 b/cyclonedx-lib/dependency_data/shas/jackson-databind.jar.sha256
@@ -0,0 +1 @@
+501d3abce4d18dcc381058ec593c5b94477906bba6efbac14dae40a642f77424
diff --git a/cyclonedx-lib/dependency_data/shas/jackson-dataformat-xml.jar.sha256 b/cyclonedx-lib/dependency_data/shas/jackson-dataformat-xml.jar.sha256
@@ -0,0 +1 @@
+edbda6c775a36049cf0088b111ab958cca0dc70cb9326918d6cf153cb3fa426b
diff --git a/cyclonedx-lib/dependency_data/shas/json-schema.jar.sha256 b/cyclonedx-lib/dependency_data/shas/json-schema.jar.sha256
@@ -0,0 +1 @@
+968991e5718520cdd7b224770f790cf2c241cddf64d10a36c21f9f8b4a15e79c
diff --git a/cyclonedx-lib/dependency_data/versions/commons-codec.jar.version b/cyclonedx-lib/dependency_data/versions/commons-codec.jar.version
@@ -0,0 +1 @@
+1.15
diff --git a/cyclonedx-lib/dependency_data/versions/commons-io.jar.version b/cyclonedx-lib/dependency_data/versions/commons-io.jar.version
@@ -0,0 +1 @@
+2.11.0
diff --git a/cyclonedx-lib/dependency_data/versions/cyclonedx-core-java.jar.version b/cyclonedx-lib/dependency_data/versions/cyclonedx-core-java.jar.version
@@ -0,0 +1 @@
+8.0.3
diff --git a/cyclonedx-lib/dependency_data/versions/github-package-url.jar.version b/cyclonedx-lib/dependency_data/versions/github-package-url.jar.version
@@ -0,0 +1 @@
+1.4.1
diff --git a/cyclonedx-lib/dependency_data/versions/jackson-annotations.jar.version b/cyclonedx-lib/dependency_data/versions/jackson-annotations.jar.version
@@ -0,0 +1 @@
+2.14.2
diff --git a/cyclonedx-lib/dependency_data/versions/jackson-core.jar.version b/cyclonedx-lib/dependency_data/versions/jackson-core.jar.version
@@ -0,0 +1 @@
+2.14.2
diff --git a/cyclonedx-lib/dependency_data/versions/jackson-databind.jar.version b/cyclonedx-lib/dependency_data/versions/jackson-databind.jar.version
@@ -0,0 +1 @@
+2.14.2
diff --git a/cyclonedx-lib/dependency_data/versions/jackson-dataformat-xml.jar.version b/cyclonedx-lib/dependency_data/versions/jackson-dataformat-xml.jar.version
@@ -0,0 +1 @@
+2.14.2
diff --git a/cyclonedx-lib/dependency_data/versions/json-schema.jar.version b/cyclonedx-lib/dependency_data/versions/json-schema.jar.version
@@ -0,0 +1 @@
+1.0.77
diff --git a/cyclonedx-lib/getDependencies b/cyclonedx-lib/getDependencies
@@ -2,16 +2,27 @@
 
 LABEL=params.LABEL ? params.LABEL : 'ci.role.test&&hw.arch.x86&&sw.os.linux'
 
+TEMURIN_BUILD_REPO="https://github.com/adamfarley/temurin-build"
+TEMURIN_BUILD_BRANCH="add_versions_and_shas_to_build_getdependencies"
+
 stage('Queue') {
     node("$LABEL") {
        cleanWs()
        fetchDeps()
     }
 }
 
-def fetchSingleFile(String jarFile, String sha, String mavenURL) {
-    sh 'echo "' + sha + '  sbom_dependencies/' + jarFile + '" >> sbom_dep_shas.txt'
+// Fetch a single file from a location under Maven.
+// Args:
+//   - jarFile:  The name of the file after we've downloaded it.
+//   - mavenURL: The file and location of the jarfile to be downloaded (under Maven).
+def fetchSingleFile(String jarFile, String mavenURL) {
+    // Downloads the requested jarfile
     sh 'curl -L -o "sbom_dependencies/' + jarFile + '" "https://search.maven.org/classic/remotecontent?filepath=' + mavenURL + '"'
+
+    // Puts the matching sha for the jarfile into the list of shas to check.
+    def jarSHA = readFile(file : './temurin-build/cyclonedx-lib/dependency_data/shas/' + jarFile + '.sha256').replaceAll("\\s","")
+    sh 'echo "' + jarSHA + '  sbom_dependencies/' + jarFile + '" >> sbom_dependency_shas.txt'
 }
 
 def fetchDeps() {
@@ -21,38 +32,45 @@ def fetchDeps() {
     }
     timeout(time: time_limit, unit: 'HOURS') {
         try {
+            // This allows us to easily specify which of our workspace files are to be archived later.
             sh 'mkdir sbom_dependencies'
-
-
+
             // These versions come from https://github.com/CycloneDX/cyclonedx-core-java/tags
             // Version->spec mappings are in https://github.com/CycloneDX/cyclonedx-core-java#cyclonedx-schema-support
-            def cyclonedx_core_java_version = "8.0.3"
-            def jackson_core_version = "2.14.2"
-            def jackson_annotations_version = "2.14.2"
-            def jackson_databind_version = "2.14.2"
-            def json_schema_version = "1.0.77"
-            def commons_codec_version = "1.15"
-            def commons_io_version = "2.11.0"
-            def github_package_url_version = "1.4.1"
-
-            fetchSingleFile("cyclonedx-core-java.jar", "ecc371d12808dfe76047f87f8235665d74dd6cf8ec12c41d052715a3fd79e0b5", "org/cyclonedx/cyclonedx-core-java/${cyclonedx_core_java_version}/cyclonedx-core-java-${cyclonedx_core_java_version}.jar")
-            fetchSingleFile("jackson-core.jar", "b5d37a77c88277b97e3593c8740925216c06df8e4172bbde058528df04ad3e7a", "com/fasterxml/jackson/core/jackson-core/${jackson_core_version}/jackson-core-${jackson_core_version}.jar")
-            fetchSingleFile("jackson-dataformat-xml.jar", "edbda6c775a36049cf0088b111ab958cca0dc70cb9326918d6cf153cb3fa426b", "com/fasterxml/jackson/dataformat/jackson-dataformat-xml/${jackson_databind_version}/jackson-dataformat-xml-${jackson_databind_version}.jar")
-            fetchSingleFile("jackson-databind.jar", "501d3abce4d18dcc381058ec593c5b94477906bba6efbac14dae40a642f77424", "com/fasterxml/jackson/core/jackson-databind/${jackson_databind_version}/jackson-databind-${jackson_databind_version}.jar")
-            fetchSingleFile("jackson-annotations.jar", "2c6869d505cf60dc066734b7d50339f975bd3adc635e26a78abb71acb4473c0d", "com/fasterxml/jackson/core/jackson-annotations/${jackson_annotations_version}/jackson-annotations-${jackson_annotations_version}.jar")
-            fetchSingleFile("json-schema.jar", "968991e5718520cdd7b224770f790cf2c241cddf64d10a36c21f9f8b4a15e79c", "com/networknt/json-schema-validator/${json_schema_version}/json-schema-validator-${json_schema_version}.jar")
-            fetchSingleFile("commons-codec.jar", "b3e9f6d63a790109bf0d056611fbed1cf69055826defeb9894a71369d246ed63", "commons-codec/commons-codec/${commons_codec_version}/commons-codec-${commons_codec_version}.jar")
-            fetchSingleFile("github-package-url.jar", "8e23280221afd1e6561d433dfb133252cd287167acb0eca5a991667118ff10a2", "com/github/package-url/packageurl-java/${github_package_url_version}/packageurl-java-${github_package_url_version}.jar")
-            fetchSingleFile("commons-io.jar", "961b2f6d87dbacc5d54abf45ab7a6e2495f89b75598962d8c723cea9bc210908", "commons-io/commons-io/${commons_io_version}/commons-io-${commons_io_version}.jar")
-
-            sh 'sha256sum -c sbom_dep_shas.txt'
+            sh 'git clone -b ' + TEMURIN_BUILD_BRANCH + ' ' + TEMURIN_BUILD_REPO + ' temurin-build'
+            def dep_data = WORKSPACE + "/temurin-build/cyclonedx-lib/dependency_data/"
+
+            def cyclonedx_core_java_version = readFile(file : dep_data + 'versions/cyclonedx-core-java.jar.version').replaceAll("\\s","")
+            def jackson_core_version = readFile(file : dep_data + 'versions/jackson-core.jar.version').replaceAll("\\s","")
+            def jackson_annotations_version = readFile(file : dep_data + 'versions/jackson-annotations.jar.version').replaceAll("\\s","")
+            def jackson_databind_version = readFile(file : dep_data + 'versions/jackson-databind.jar.version').replaceAll("\\s","")
+            def json_schema_version = readFile(file : dep_data + 'versions/json-schema.jar.version').replaceAll("\\s","")
+            def commons_codec_version = readFile(file : dep_data + 'versions/commons-codec.jar.version').replaceAll("\\s","")
+            def commons_io_version = readFile(file : dep_data + 'versions/commons-io.jar.version').replaceAll("\\s","")
+            def github_package_url_version = readFile(file : dep_data + 'versions/github-package-url.jar.version').replaceAll("\\s","")
 
+            // Each of these fetches a jar in the format: fetchSingleFile(jar name post-download, current jar location under Maven)
+            fetchSingleFile("cyclonedx-core-java.jar", "org/cyclonedx/cyclonedx-core-java/${cyclonedx_core_java_version}/cyclonedx-core-java-${cyclonedx_core_java_version}.jar")
+            fetchSingleFile("jackson-core.jar", "com/fasterxml/jackson/core/jackson-core/${jackson_core_version}/jackson-core-${jackson_core_version}.jar")
+            fetchSingleFile("jackson-dataformat-xml.jar", "com/fasterxml/jackson/dataformat/jackson-dataformat-xml/${jackson_databind_version}/jackson-dataformat-xml-${jackson_databind_version}.jar")
+            fetchSingleFile("jackson-databind.jar", "com/fasterxml/jackson/core/jackson-databind/${jackson_databind_version}/jackson-databind-${jackson_databind_version}.jar")
+            fetchSingleFile("jackson-annotations.jar", "com/fasterxml/jackson/core/jackson-annotations/${jackson_annotations_version}/jackson-annotations-${jackson_annotations_version}.jar")
+            fetchSingleFile("json-schema.jar", "com/networknt/json-schema-validator/${json_schema_version}/json-schema-validator-${json_schema_version}.jar")
+            fetchSingleFile("commons-codec.jar", "commons-codec/commons-codec/${commons_codec_version}/commons-codec-${commons_codec_version}.jar")
+            fetchSingleFile("github-package-url.jar", "com/github/package-url/packageurl-java/${github_package_url_version}/packageurl-java-${github_package_url_version}.jar")
+            fetchSingleFile("commons-io.jar", "commons-io/commons-io/${commons_io_version}/commons-io-${commons_io_version}.jar")
+
+
+            // Check that every file matches the sha of the file we expected.
+            sh 'sha256sum -c sbom_dependency_shas.txt'
+
+            // Store all retrieved files on Jenkins, for use during sbom creation.
             archiveArtifacts '**/sbom_dependencies/*'
         } finally {
+            // Clean up workspace.
             cleanWs()
         }
     }
 }
 
-
 return this