Merge pull request #220 from IBM/develop

Release zAppBuild 2.5.0

build-conf/Cobol.properties

@@ -83,13 +83,13 @@ dbb.LinkEditScanner.excludeFilter = ${SDFHLOAD}.*, ${SCEELKED}.*
 # additional libraries for linkEdit SYSLIB concatenation, comma-separated, see definitions in application-conf
 # cobol_linkEditSyslibConcatenation=
 
-# cobol_dependenciesAlternativeLibraryNameMapping - an *optional* map to define target dataset definition for alternate include libraries
+# cobol_dependenciesAlternativeLibraryNameMapping - an *optional* JSON String of a map defining the target dataset definition for alternate include libraries
 #  this property is used to 
 #   * copy files the to mapped dataset definition (PLEASE NOTE! This setting takes precendence over cobol_dependenciesDatasetMapping)
 #   * defining additional allocations in the compile step
 #
 #  note that the SYSLIB is defaulted to the dataset definition 'cobol_cpyPDS' and is not required to be set here
-#  sample: cobol_dependenciesAlternativeLibraryNameMapping = [MYFILE: 'cobol_myfilePDS', DCLGEN : 'cobol_dclgenPDS']
+#  sample: cobol_dependenciesAlternativeLibraryNameMapping = {"MYFILE": "cobol_myfilePDS", "DCLGEN" : "cobol_dclgenPDS"}
 cobol_dependenciesAlternativeLibraryNameMapping=
 
 # cobol_dependenciesDatasetMapping - an optional dbb property mapping to map dependencies to different target datasets

build-conf/PLI.properties

@@ -70,12 +70,12 @@ dbb.DependencyScanner.languageHint=PLI :: **/*.pli, **/*.inc, **/*.cpy
 # additional libraries for linkEdit SYSLIB concatenation, comma-separated, see definitions in application-conf
 # pli_linkEditSyslibConcatenation=
 
-# pli_dependenciesAlternativeLibraryNameMapping - an *optional* map to define target dataset definition for alternate include libraries
+# pli_dependenciesAlternativeLibraryNameMapping -  an *optional* JSON String of a map defining the target dataset definition for alternate include libraries
 #  this property is used to 
 #   * copy files the to mapped dataset definition (this setting takes precendence over pli_dependenciesDatasetMapping)
 #   * defining additional allocations in the compile step
 #  note that the SYSLIB is defaulted to the dataset definition 'pli_cpyPDS' and is not required to be set here
-#  sample: pli_dependenciesAlternativeLibraryNameMapping = [MYFILE: 'pli_myfilePDS', DCLGEN : 'pli_dclgenPDS']
+#  sample: pli_dependenciesAlternativeLibraryNameMapping = {"MYFILE": "pli_myfilePDS", "DCLGEN" : "pli_dclgenPDS"}
 pli_dependenciesAlternativeLibraryNameMapping=
 
 # pli_dependenciesDatasetMapping - an optional dbb property mapping to map dependencies to different target datasets

build-conf/README.md

@@ -57,6 +57,7 @@ dbb.RepositoryClient.passwordFile | DBB configuration property for web applicati
 ### dependencyReport.properties
 Properties used by the impact utilities to generate a report of external impacted files
 
+Property | Description
 --- | ---
 reportExternalImpacts | Flag to indicate if an *impactBuild* should analyze and report external impacted files in other collections ***Can be overridden by build.groovy option -re, --reportExternalImpacts***
 reportExternalImpactsAnalysisDepths | Configuration of the analysis depths when performing impact analysis for external impacts (simple|deep) *** Can be overridden by application-conf ***
@@ -284,4 +285,4 @@ transfer_srcPDS | Dataset of any type of source
 transfer_jclPDS | Sample dataset for JCL members
 transfer_xmlPDS | Sample dataset for xml members
 transfer_srcOptions | BPXWDYN creation options for creating 'source' type data sets
-transfer_outputDatasets | List of output datasets to document deletions ** Can be overridden by a file property. ** If used for multiple, use a file property to set transfer_outputDatasets 
+transfer_outputDatasets | List of output datasets to document deletions ** Can be overridden by a file property. ** If used for multiple, use a file property to set transfer_outputDatasets 

build-conf/build.properties

@@ -45,7 +45,7 @@ applicationConfRootDir=
 #
 # Minimum required DBB ToolkitVersion to run this version of zAppBuild
 #  Build initialization process validates the DBB Toolkit Version in use and matches that against this setting 
-requiredDBBToolkitVersion=1.0.0
+requiredDBBToolkitVersion=1.0.3
 
 #
 # Comma separated list of required build properties for zAppBuild/build.groovy

build.groovy

@@ -184,7 +184,8 @@ options:
 	cli.h(longOpt:'hlq', args:1, required:true, 'High level qualifier for partition data sets')
 
 	// build options
-	cli.p(longOpt:'propFiles', args:1, 'Commas spearated list of additional property files to load. Absolute paths or relative to workspace.')
+	cli.p(longOpt:'propFiles', args:1, 'Comma separated list of additional property files to load. Absolute paths or relative to workspace.')
+	cli.po(longOpt:'propOverwrites', args:1, 'Comma separated list of key=value pairs for set and overwrite build properties.')
 	cli.l(longOpt:'logEncoding', args:1, 'Encoding of output logs. Default is EBCDIC')
 	cli.f(longOpt:'fullBuild', 'Flag indicating to build all programs for application')
 	cli.i(longOpt:'impactBuild', 'Flag indicating to build only programs impacted by changed files since last successful build.')
@@ -326,6 +327,23 @@ def populateBuildProperties(String[] args) {
 			props.load(new File(propFile))
 		}
 	}
+
+	// populate property overwrites from argument list
+	if (opts.po) props.propOverwrites = opts.po
+	if (props.propOverwrites) {
+		String[] propOverwrites = props.propOverwrites.split(',')
+		propOverwrites.each { buildPropertyOverwrite ->
+			(key, value) = buildPropertyOverwrite.tokenize('=')
+			if (key && value) {
+				if (opts.v) println "** Overwriting build property ${key} from cli argument --propOverwrite with value ${value}"
+				props.put(key, value)
+			}
+			else {
+				println "*! Overwriting build property from cli argument --propOverwrite failed due a null value ( key: $key , value :$value )"
+			}
+		}
+	}
+
 
 	// set flag indicating to run unit tests
 	if (opts.zTest) props.runzTests = 'true'
@@ -534,6 +552,12 @@ def createBuildList() {
 		println "** Scanning source code."
 		impactUtils.updateCollection(buildList, null, null, repositoryClient)
 	}
+
+	// Loading file/member level properties from member specific properties files
+	if (props.filePropertyValueKeySet().getAt("loadFileLevelProperties") || props.loadFileLevelProperties) {
+		println "** Populating file level properties from individual property files."
+		buildUtils.loadFileLevelPropertiesFromFile(buildList)
+	}
 
 	return [buildList, deleteList]
 }

languages/Assembler.groovy

@@ -13,6 +13,11 @@ import com.ibm.dbb.build.report.records.*
 @Field def impactUtils= loadScript(new File("${props.zAppBuildDir}/utilities/ImpactUtilities.groovy"))
 @Field RepositoryClient repositoryClient
 
+@Field def resolverUtils
+// Conditionally load the ResolverUtilities.groovy which require at least DBB 1.1.2
+if (props.useSearchConfiguration && props.useSearchConfiguration.toBoolean() && buildUtils.assertDbbBuildToolkitVersion(props.dbbToolkitVersion, "1.1.2")) {
+	resolverUtils = loadScript(new File("${props.zAppBuildDir}/utilities/ResolverUtilities.groovy"))}
+
 println("** Building files mapped to ${this.class.getName()}.groovy script")
 
 // verify required build properties
@@ -28,15 +33,24 @@ List<String> sortedList = buildUtils.sortBuildList(argMap.buildList, 'assembler_
 sortedList.each { buildFile ->
 	println "*** Building file $buildFile"
 
-	// copy build file to input data set
+	// configure dependency resolution and create logical file 	
+	def dependencyResolver
+	LogicalFile logicalFile
+
+	if (props.useSearchConfiguration && props.useSearchConfiguration.toBoolean() && props.assembler_dependencySearch && buildUtils.assertDbbBuildToolkitVersion(props.dbbToolkitVersion, "1.1.2")) { // use new SearchPathDependencyResolver
+		String dependencySearch = props.getFileProperty('assembler_dependencySearch', buildFile)
+		dependencyResolver = resolverUtils.createSearchPathDependencyResolver(dependencySearch)
+		logicalFile = resolverUtils.createLogicalFile(dependencyResolver, buildFile)
+	} else { // use deprecated DependencyResolver
+		String rules = props.getFileProperty('assembler_resolutionRules', buildFile)
+		dependencyResolver = buildUtils.createDependencyResolver(buildFile, rules)
+		logicalFile = dependencyResolver.getLogicalFile()
+	}
+
 	// copy build file and dependency files to data sets
-	String rules = props.getFileProperty('assembler_resolutionRules', buildFile)
-	String assembler_srcPDS = props.getFileProperty('assembler_srcPDS', buildFile)
-	DependencyResolver dependencyResolver = buildUtils.createDependencyResolver(buildFile, rules)
-	buildUtils.copySourceFiles(buildFile, assembler_srcPDS, 'assembler_dependenciesDatasetMapping', null ,dependencyResolver)
+	buildUtils.copySourceFiles(buildFile, props.assembler_srcPDS, 'assembler_dependenciesDatasetMapping', null ,dependencyResolver)
 
 	// create mvs commands
-	LogicalFile logicalFile = dependencyResolver.getLogicalFile()
 	String member = CopyToPDS.createMemberName(buildFile)
 	File logFile = new File( props.userBuild ? "${props.buildOutDir}/${member}.log" : "${props.buildOutDir}/${member}.asm.log")
 	if (logFile.exists())

languages/Cobol.groovy

@@ -15,6 +15,11 @@ import com.ibm.dbb.build.report.records.*
 @Field def bindUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BindUtilities.groovy"))
 @Field RepositoryClient repositoryClient
 
+@Field def resolverUtils
+// Conditionally load the ResolverUtilities.groovy which require at least DBB 1.1.2
+if (props.useSearchConfiguration && props.useSearchConfiguration.toBoolean() && buildUtils.assertDbbBuildToolkitVersion(props.dbbToolkitVersion, "1.1.2")) {
+	resolverUtils = loadScript(new File("${props.zAppBuildDir}/utilities/ResolverUtilities.groovy"))}
+
 println("** Building files mapped to ${this.class.getName()}.groovy script")
 
 // verify required build properties
@@ -39,16 +44,28 @@ sortedList.each { buildFile ->
 	// Check if this a testcase
 	isZUnitTestCase = (props.getFileProperty('cobol_testcase', buildFile).equals('true')) ? true : false
 
+	// configure dependency resolution and create logical file
+	def dependencyResolver
+	LogicalFile logicalFile
+
+	if (props.useSearchConfiguration && props.useSearchConfiguration.toBoolean() && props.cobol_dependencySearch && buildUtils.assertDbbBuildToolkitVersion(props.dbbToolkitVersion, "1.1.2")) { // use new SearchPathDependencyResolver
+		String dependencySearch = props.getFileProperty('cobol_dependencySearch', buildFile)
+		dependencyResolver = resolverUtils.createSearchPathDependencyResolver(dependencySearch)
+		logicalFile = resolverUtils.createLogicalFile(dependencyResolver, buildFile)
+
+	} else { // use deprecated DependencyResolver
+		String rules = props.getFileProperty('cobol_resolutionRules', buildFile)
+		dependencyResolver = buildUtils.createDependencyResolver(buildFile, rules)
+		logicalFile = dependencyResolver.getLogicalFile()
+	}
+
 	// copy build file and dependency files to data sets
-	String rules = props.getFileProperty('cobol_resolutionRules', buildFile)
-	DependencyResolver dependencyResolver = buildUtils.createDependencyResolver(buildFile, rules)
 	if(isZUnitTestCase){
 		buildUtils.copySourceFiles(buildFile, props.cobol_testcase_srcPDS, null, null, null)
 	}else{
 		buildUtils.copySourceFiles(buildFile, props.cobol_srcPDS, 'cobol_dependenciesDatasetMapping', props.cobol_dependenciesAlternativeLibraryNameMapping, dependencyResolver)
 	}
 	// create mvs commands
-	LogicalFile logicalFile = dependencyResolver.getLogicalFile()
 	String member = CopyToPDS.createMemberName(buildFile)
 	File logFile = new File( props.userBuild ? "${props.buildOutDir}/${member}.log" : "${props.buildOutDir}/${member}.cobol.log")
 	if (logFile.exists())
@@ -238,10 +255,18 @@ def createCompileCommand(String buildFile, LogicalFile logicalFile, String membe
 
 	// adding alternate library definitions
 	if (props.cobol_dependenciesAlternativeLibraryNameMapping) {
-		alternateLibraryNameAllocations = evaluate(props.cobol_dependenciesAlternativeLibraryNameMapping)
-		alternateLibraryNameAllocations.each { libraryName, datasetDSN ->
-			datasetDSN = props.getProperty(datasetDSN)
-			if (datasetDSN) compile.dd(new DDStatement().name(libraryName).dsn(datasetDSN).options("shr"))
+		alternateLibraryNameAllocations = buildUtils.parseJSONStringToMap(props.cobol_dependenciesAlternativeLibraryNameMapping)
+		alternateLibraryNameAllocations.each { libraryName, datasetDefinition ->
+			datasetName = props.getProperty(datasetDefinition)
+			if (datasetName) {
+				compile.dd(new DDStatement().name(libraryName).dsn(datasetName).options("shr"))
+			}
+			else {
+				String errorMsg = "*! Cobol.groovy. The dataset definition $datasetDefinition could not be resolved from the DBB Build properties."
+				println(errorMsg)
+				props.error = "true"
+				buildUtils.updateBuildResult(errorMsg:errorMsg,client:getRepositoryClient())
+			}
 		}
 	}
 

languages/LinkEdit.groovy

@@ -30,10 +30,17 @@ sortedList.each { buildFile ->
 	buildUtils.copySourceFiles(buildFile, props.linkedit_srcPDS, null, null, null)
 
 	// create mvs commands
-	String rules = props.getFileProperty('linkedit_resolutionRules', buildFile)
-	DependencyResolver dependencyResolver = buildUtils.createDependencyResolver(buildFile, rules)
-	LogicalFile logicalFile = dependencyResolver.getLogicalFile()
+	LogicalFile logicalFile
+	if (props.useSearchConfiguration && props.useSearchConfiguration.toBoolean()) { // use new SearchPathDependencyResolver
+		logicalFile = SearchPathDependencyResolver.getLogicalFile(buildFile,props.workspace)
+	}
+	else { // use deprecated DependencyResolver API
+		DependencyResolver dependencyResolver = buildUtils.createDependencyResolver(buildFile, null)
+		logicalFile = dependencyResolver.getLogicalFile()
+	}
+
 	String member = CopyToPDS.createMemberName(buildFile)
+
 	File logFile = new File( props.userBuild ? "${props.buildOutDir}/${member}.log" : "${props.buildOutDir}/${member}.linkedit.log")
 	if (logFile.exists())
 		logFile.delete()

languages/PLI.groovy

@@ -14,6 +14,11 @@ import com.ibm.dbb.build.report.records.*
 @Field def impactUtils= loadScript(new File("${props.zAppBuildDir}/utilities/ImpactUtilities.groovy"))
 @Field RepositoryClient repositoryClient
 
+@Field def resolverUtils
+// Conditionally load the ResolverUtilities.groovy which require at least DBB 1.1.2
+if (props.useSearchConfiguration && props.useSearchConfiguration.toBoolean() && buildUtils.assertDbbBuildToolkitVersion(props.dbbToolkitVersion, "1.1.2")) {
+	resolverUtils = loadScript(new File("${props.zAppBuildDir}/utilities/ResolverUtilities.groovy"))}
+
 println("** Building files mapped to ${this.class.getName()}.groovy script")
 
 // verify required build properties
@@ -37,19 +42,28 @@ sortedList.each { buildFile ->
 	// Check if this a testcase
 	isZUnitTestCase = (props.getFileProperty('pli_testcase', buildFile).equals('true')) ? true : false
 
-	// copy build file to input data set
-	// copy build file and dependency files to data sets
-	String rules = props.getFileProperty('pli_resolutionRules', buildFile)
-	DependencyResolver dependencyResolver = buildUtils.createDependencyResolver(buildFile, rules)
+	// configure dependency resolution and create logical file 	
+	def dependencyResolver
+	LogicalFile logicalFile
 
+	if (props.useSearchConfiguration && props.useSearchConfiguration.toBoolean() && props.pli_dependencySearch && buildUtils.assertDbbBuildToolkitVersion(props.dbbToolkitVersion, "1.1.2")) { // use new SearchPathDependencyResolver
+		String dependencySearch = props.getFileProperty('pli_dependencySearch', buildFile)
+		dependencyResolver = resolverUtils.createSearchPathDependencyResolver(dependencySearch)
+		logicalFile = resolverUtils.createLogicalFile(dependencyResolver, buildFile)
+	} else { // use deprecated DependencyResolver
+		String rules = props.getFileProperty('pli_resolutionRules', buildFile)
+		dependencyResolver = buildUtils.createDependencyResolver(buildFile, rules)
+		logicalFile = dependencyResolver.getLogicalFile()
+	}
+
+	// copy build file and dependency files to data sets
 	if(isZUnitTestCase){
 		buildUtils.copySourceFiles(buildFile, props.pli_testcase_srcPDS, null, null, null)
 	}else{
 		buildUtils.copySourceFiles(buildFile, props.pli_srcPDS, 'pli_dependenciesDatasetMapping', props.pli_dependenciesAlternativeLibraryNameMapping, dependencyResolver)
 	}
 
 	// create mvs commands
-	LogicalFile logicalFile = dependencyResolver.getLogicalFile()
 	String member = CopyToPDS.createMemberName(buildFile)
 	File logFile = new File( props.userBuild ? "${props.buildOutDir}/${member}.log" : "${props.buildOutDir}/${member}.pli.log")
 	if (logFile.exists())
@@ -189,7 +203,7 @@ def createCompileCommand(String buildFile, LogicalFile logicalFile, String membe
 	// add additional datasets with dependencies based on the dependenciesDatasetMapping
 	PropertyMappings dsMapping = new PropertyMappings('pli_dependenciesDatasetMapping')
 	dsMapping.getValues().each { targetDataset ->
-		// exclude the defaults cobol_cpyPDS and any overwrite in the alternativeLibraryNameMap
+		// exclude the defaults pli_cpyPDS and any overwrite in the alternativeLibraryNameMap
 		if (targetDataset != 'pli_incPDS')
 			compile.dd(new DDStatement().dsn(props.getProperty(targetDataset)).options("shr"))
 	}
@@ -224,19 +238,27 @@ def createCompileCommand(String buildFile, LogicalFile logicalFile, String membe
 		compile.dd(new DDStatement().name("DBRMLIB").dsn("$props.pli_dbrmPDS($member)").options('shr').output(true).deployType('DBRM'))
 
 	// adding alternate library definitions
-	if (props.cobol_dependenciesAlternativeLibraryNameMapping) {
-		alternateLibraryNameAllocations = evaluate(props.pli_dependenciesAlternativeLibraryNameMapping)
-		alternateLibraryNameAllocations.each { libraryName, datasetDSN ->
-			datasetDSN = props.getProperty(datasetDSN)
-			if (datasetDSN) compile.dd(new DDStatement().name(libraryName).dsn(datasetDSN).options("shr"))
+	if (props.pli_dependenciesAlternativeLibraryNameMapping) {
+		alternateLibraryNameAllocations = buildUtils.parseJSONStringToMap(props.pli_dependenciesAlternativeLibraryNameMapping)
+		alternateLibraryNameAllocations.each { libraryName, datasetDefinition ->
+			datasetName = props.getProperty(datasetDefinition)
+			if (datasetName) {
+				compile.dd(new DDStatement().name(libraryName).dsn(datasetName).options("shr"))
+			}
+			else {
+				String errorMsg = "*! PLI.groovy. The dataset definition $datasetDefinition could not be resolved from the DBB Build properties."
+				println(errorMsg)
+				props.error = "true"
+				buildUtils.updateBuildResult(errorMsg:errorMsg,client:getRepositoryClient())
+			}
 		}
 	}
 
 	// add IDz User Build Error Feedback DDs
 	if (props.errPrefix) {
 		compile.dd(new DDStatement().name("SYSADATA").options("DUMMY"))
 		// SYSXMLSD.XML suffix is mandatory for IDZ/ZOD to populate remote error list
-		compile.dd(new DDStatement().name("SYSXMLSD").dsn("${props.hlq}.${props.errPrefix}.SYSXMLSD.XML").options(props.cobol_compileErrorFeedbackXmlOptions))
+		compile.dd(new DDStatement().name("SYSXMLSD").dsn("${props.hlq}.${props.errPrefix}.SYSXMLSD.XML").options(props.pli_compileErrorFeedbackXmlOptions))
 	}
 
 	// add a copy command to the compile command to copy the SYSPRINT from the temporary dataset to an HFS log file

languages/PSBgen.groovy

@@ -221,7 +221,7 @@ def createACBgenCommand(String buildFile, String member, File logFile) {
 
 	// retrieve target pds and deploytype
 	String acbgen_loadPDS = props.getFileProperty('acbgen_loadPDS', buildFile)
-	String deployType = buildUtils.getDeployType("acbgen", buildFile, logicalFile)
+	String deployType = buildUtils.getDeployType("acbgen", buildFile, null)
 	acbgen.dd(new DDStatement().name("IMSACB").dsn("${acbgen_loadPDS}").options('shr').output(true).deployType(deployType))
 
 	// addional allocations