Skip to content

Commit

Permalink
[Build] Simplify bootstrapping of I-build tests and fetch files from git
Browse files Browse the repository at this point in the history 
Instead of fetching the test-configuration files from eclipse.downloads
and push it there in each I-build, simply get the test-configuration
files directly from the eclipse.platform.releng.aggregator git
repository, using partial-clones for improved performance:
- https://git-scm.com/docs/partial-clone
- https://github.blog/open-source/git/get-up-to-speed-with-partial-clone-and-shallow-clone/

This avoids the need to pack and upload the test-configuration zip-file
and also simplifies fetching the test-configuration for other branches,
like maintenance branches or to test changes.

This also removes from the getEBuilder.xml ant script the possibility to
download the test-configuration zip file and to use a local zip.

Furthermore this removes from the I-/Y-builds the creation and
publication of the test-configuration zip-file.
  • Loading branch information
@HannesWell
HannesWell committed Nov 30, 2024
1 parent 0e4fb78 commit a8bbcec
Show file tree
Hide file tree
Showing 4 changed files with 37 additions and 250 deletions.
17 changes: 0 additions & 17 deletions JenkinsJobs/Builds/I_build.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,23 +227,6 @@ spec:
}
}
}
stage('Create Source Bundles'){
steps {
container('jnlp') {
sh \'\'\'
cd ${WORKSPACE}/eclipse.platform.releng.aggregator/eclipse.platform.releng.aggregator/cje-production/mbscripts
unset JAVA_TOOL_OPTIONS
unset _JAVA_OPTIONS
./mb200_createSourceBundles.sh $CJE_ROOT/buildproperties.shsource 2>&1 | tee $logDir/mb200_createSourceBundles.sh.log
if [[ ${PIPESTATUS[0]} -ne 0 ]]
then
echo "Failed in Create Source Bundles stage"
exit 1
fi
\'\'\'
}
}
}
stage('Aggregator maven build'){
environment {
KEYRING = credentials('secret-subkeys-releng.asc')
Expand Down
17 changes: 0 additions & 17 deletions JenkinsJobs/YBuilds/Y_build.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,23 +238,6 @@ spec:
}
}
}
stage('Create Source Bundles'){
steps {
container('jnlp') {
sh \'\'\'
cd ${WORKSPACE}/eclipse.platform.releng.aggregator/eclipse.platform.releng.aggregator/cje-production/mbscripts
unset JAVA_TOOL_OPTIONS
unset _JAVA_OPTIONS
./mb200_createSourceBundles.sh $CJE_ROOT/buildproperties.shsource 2>&1 | tee $logDir/mb200_createSourceBundles.sh.log
if [[ ${PIPESTATUS[0]} -ne 0 ]]
then
echo "Failed in Create Source Bundles stage"
exit 1
fi
\'\'\'
}
}
}
stage('Aggregator maven build'){
environment {
KEYRING_PASSPHRASE = credentials('secret-subkeys-releng.acs-passphrase')
Expand Down
31 changes: 0 additions & 31 deletions cje-production/mbscripts/mb200_createSourceBundles.sh
View file

This file was deleted.

222 changes: 37 additions & 185 deletions production/testScripts/hudsonBootstrap/getEBuilder.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
Then use ant to invoke the functionality. Example:
ant -f getEBuilder.xml -EBUILDER_HASH=910fca82df3ea649ba8ee758259561582e13bf8b
ant -f getEBuilder.xml
-->
<project
Expand All @@ -22,61 +22,42 @@

<target
name="init"
depends="initWorkspace"
unless="getEBuilderInitialized">

<!-- assign defaults if not on hudson, for local testing if nothing else -->
<property
name="WORKSPACE"
value="${basedir}" />
<property environment="env" />
<condition
property="WORKSPACE"
value="${env.WORKSPACE}"
else="${basedir}">
<isset property="env.WORKSPACE" />
</condition>

<condition
property="gitHost"
value="${env.GIT_HOST}"
else="git.eclipse.org">
else="https://github.com/eclipse-platform">
<isset property="env.GIT_HOST" />
</condition>

<!-- normally provided by caller, but if not, assume master -->
<property
name="EBUILDER_HASH"
value="master" />

<!-- git is fall back, if no method set explicitly -->
<condition
property="fetchSource"
value="git">
<and>
<not>
<isset property="EBuilderlocal" />
</not>
<not>
<isset property="downloadURL" />
</not>
</and>
</condition>
<!-- Can be provided by caller to use another branch, e.g. for testing changes -->
<property
name="scmProject"
value="eclipse.platform.releng.aggregator" />
name="testConfigurationBranch"
value="${buildId}"/>

<property
name="folderName"
name="projectName"
value="eclipse.platform.releng.aggregator" />

<property
name="eScriptLocation"
value="${folderName}/production/testScripts" />
value="${projectName}/production/testScripts" />

<echo message="=== Properties in 'getEBuilder.xml'" />
<echo message=" WORKSPACE: ${WORKSPACE}" />
<echo message=" folderName: ${folderName}" />
<echo message=" projectName: ${projectName}" />
<echo message=" eScriptLocation: ${eScriptLocation}" />
<echo message=" scmProject: ${scmProject}" />
<echo message=" EBUILDER_HASH: ${EBUILDER_HASH}" />
<echo message=" The following are used, in order (if defined), to get 'production scripts'" />
<echo message=" (fetchDirect from) downloadURL: ${downloadURL}" />
<echo message=" (copy from local cache directory) EBuilderlocal: ${EBuilderlocal}" />
<echo message=" fetchSource (from Git): ${fetchSource}" />
<echo message=" test-configuration source git branch: ${testConfigurationBranch}" />

<property
name="getEBuilderInitialized"
Expand All @@ -91,165 +72,36 @@
<!-- remove old one, if there, and make sure directory exists for
eventual checkout/copy/globmapper
-->
<delete dir="${WORKSPACE}/${folderName}" />
<mkdir dir="${WORKSPACE}/${folderName}" />
<delete dir="${WORKSPACE}/${projectName}" />
<mkdir dir="${WORKSPACE}/${projectName}" />

<antcall target="fetchEBuilder" />
<antcall target="EBuildercleanup" />
<antcall target="fetchEBuilderFromGit" />

<ant antfile="${WORKSPACE}/${eScriptLocation}/runTests2.xml" />

</target>

<target
name="fetchEBuilderFromGit"
depends="init,fetchEBuilderLocal"
unless="EBuilderfetched">
<echo message="Fetching EBuilder from Git via CGit" />
<get
src="https://${gitHost}/c/platform/${scmProject}.git/snapshot/${scmProject}-${EBUILDER_HASH}.zip"
dest="${WORKSPACE}/${scmProject}-${EBUILDER_HASH}.zip"
httpusecaches="false"
verbose="true" />
<unzip
src="${WORKSPACE}/${scmProject}-${EBUILDER_HASH}.zip"
dest="${WORKSPACE}/tempEBuilder/"
overwrite="true" />

<echo message="preparing to move ${WORKSPACE}/tempEBuilder" />
<echo message=" into ${WORKSPACE}/${folderName}" />
<echo message=" using regexp mapper" />
<!-- We need only the 'production/testscripts' directory. And input can be of different "depths" depending on where it comes from. -->
<move
verbose="true"
todir="${WORKSPACE}/${folderName}">
<fileset dir="${WORKSPACE}/tempEBuilder" />
<mapper
type="regexp"
from=".*production([/\\])testScripts(.*)"
to="production\1testScripts\2" />
</move>

<echo message="Fetched EBuilder/production/testscrips to ${WORKSPACE}/${folderName}" />
<property
name="EBuilderfetched"
value="true" />

</target>

<!-- specify -DnoEBuildercleanup=true on command line, to have zip and folder left in place,
such as for debugging or desired to copy the zip to your local cache.
-->
<target
name="EBuildercleanup"
unless="noEBuildercleanup">

<!-- we've moved the files, now to remove the empty dir -->
<delete
dir="${WORKSPACE}/${scmProject}-${EBUILDER_HASH}"
failonerror="false" />

<!-- cleanup (may eventually want to "cache" this zip file, so no need to re-fetch if already exists -->
<delete
file="${WORKSPACE}/${scmProject}-${EBUILDER_HASH}.zip"
failonerror="false" />

<delete
dir="${WORKSPACE}/tempEBuilder"
failonerror="false" />
<echo message="Cleaned temporary working copies" />

</target>

<!-- Will leave here for now, though doesn't add much. We had two choices: local, git.
-->
<target
name="fetchEBuilder"
depends="init, fetchEBuilderFromGit"
unless="EBuilderfetched">
<fail message="EBuilder was not fetched by any available method. Is a local variable not set?" />
</target>

<target
name="fetchEBuilderLocal"
depends="init,fetchEBuilderDirect"
if="EBuilderlocal"
unless="EBuilderfetched">
<echo message="Fetching pre-made EBuilder subset from local machine." />
<unzip
src="${EBuilderlocal}/${scmProject}-${EBUILDER_HASH}.zip"
dest="${WORKSPACE}"
overwrite="true" />

<move
todir="${WORKSPACE}/${folderName}"
failonerror="true">
<fileset dir="${WORKSPACE}/${scmProject}-${EBUILDER_HASH}">
<include name="**/*" />
</fileset>
</move>

<echo message="Copied EBuilder to ${WORKSPACE}/${folderName}" />
<property
name="EBuilderfetched"
value="true" />
</target>
<target
name="fetchEBuilderDirect"
depends="init"
if="downloadURL"
unless="EBuilderfetched">
<echo message="Fetching pre-made EBuilder subset from download site." />
<get
src="${downloadURL}/${scmProject}-${EBUILDER_HASH}.zip"
dest="${WORKSPACE}"
httpusecaches="false"
verbose="true" />

<available
file="${WORKSPACE}/${scmProject}-${EBUILDER_HASH}.zip"
property="EzipExists" />

<antcall target="unzipDownloadedZip" />

<condition property="EBuilderfetched">
<available file="${WORKSPACE}/${folderName}/production" />
</condition>

</target>

<target
name="unzipDownloadedZip"
if="EzipExists">

<echo message="Fetched EBuilder from ${downloadURL}" />
<unzip
src="${WORKSPACE}/${scmProject}-${EBUILDER_HASH}.zip"
dest="${WORKSPACE}"
overwrite="true" />
depends="init">

<!-- Not sure this is needed, since 'folderName' and 'scmProject' are
the same. (At least, the defaults set in this script).
-->
<move todir="${WORKSPACE}/${folderName}">
<fileset dir="${WORKSPACE}/${scmProject}">
<include name="**/*" />
</fileset>
</move>
<echo message="Moved EBuilder (subset) to ${WORKSPACE}/${folderName}" />
<echo message="Fetching EBuilder from Git" />
<!--Create a partial and shallow clone and sprase-checkout only the 'production' sub-folder -->
<exec executable="git"
dir="${WORKSPACE}" failonerror="true">
<arg line="clone --depth=1 --filter=tree:0 --no-checkout"/>
<arg value="--branch=${testConfigurationBranch}"/>
<arg value="${gitHost}/${projectName}.git"/>
</exec>
<exec executable="git"
dir="${WORKSPACE}/${projectName}" failonerror="true">
<arg line="sparse-checkout set --no-cone production"/>
</exec>
<exec executable="git"
dir="${WORKSPACE}/${projectName}" failonerror="true">
<arg line="checkout"/>
</exec>

</target>


<target
name="initWorkspace"
unless="WORKSPACE">
<property environment="env" />
<condition
property="WORKSPACE"
value="${env.WORKSPACE}"
else="${basedir}">
<isset property="env.WORKSPACE" />
</condition>
</target>
</project>

0 comments on commit a8bbcec

Please sign in to comment.