Merge pull request #354 from IBM/develop

Release zAppBuild 3.2.0

README.md

@@ -28,6 +28,7 @@ The zAppBuild sample provides the following *language* build scripts by default:
 * PSBgen.groovy
 * MFS.groovy
 * ZunitConfig.groovy
+* Transfer.groovy (for transport non-buildable files like JCL or PROC into build libraries and register them as build output)
 
 All language scripts both compile and optionally link-edit programs. The language build scripts are intended to be useful out of the box but depending on the complexity of your applications' build requirements, may require modifications to meet your development team's needs.  By following the examples used in the existing language build scripts of keeping all application specific references out of the build scripts and instead using configuration properties with strong default values, the zAppBuild sample can continue to be a generic build solution for all of your specific applications.
 
@@ -46,11 +47,16 @@ zAppBuild supports a number of build scenarios:
 * **Merge Build** - Build only changed programs which will be merged back into the mainBuildBranch by using a triple-dot git diff. 
 * **Scan Source** - Skip the actual building and only scan source files to store dependency data in collection (migration scenario).
 * **Scan Source + Outputs** - Skip the actual building and only scan source files and existing load modules to dependency data in source and output collection (migration scenario with static linkage scenarios).
+* **Build Preview** - Supplemental build option. Process all phases of the supplied build option, but will not execute the commands. A build report and a build result are generated with a specific status that excludes them in subsequent impact build calculations. 
 
 
-Links to additional documentation is provided in the table below.  Instructions on invoking a zAppBuild is included in [BUILD.md](BUILD.md) as well as invocation samples for the above mentioned build scenarios including sample console log.
+Instructions on invoking a zAppBuild is included in [docs/BUILD.md](docs/BUILD.md) as well as invocation samples for the above mentioned build scenarios including sample console logs.
 
-zAppBuild comes with a set of reporting features. It helps development teams to understand the impact of changed files across multiple applications. Another feature helps to identify conflicts due to concurrent development activities within their application. An overview of these features are documented in [REPORT.md](REPORT.md).
+Application-level build properties such as compile and link options can be defined in various ways to set global defaults, application-level overrides or even file level (member-level) overrides. The various supported strategies are documented in [docs/FilePropertyManagement.md](docs/FilePropertyManagement.md).
+
+zAppBuild comes with a set of reporting features. It helps development teams to understand the impact of changed files across multiple applications. Another feature helps to identify conflicts due to concurrent development activities within their application. An overview of these features are documented in [docs/REPORTS.md](docs/REPORTS.md).
+
+Links to additional documentation is provided in the table below.  
 
 ## Repository Legend
 Folder/File | Description | Documentation Link

build-conf/Assembler.properties

@@ -18,6 +18,14 @@ assembler_dbrmPDS=${hlq}.DBRM
 # assembler load data sets
 assembler_loadPDS=${hlq}.LOAD
 
+#
+# assembler sysadata data set
+assembler_sysadataPDS=${hlq}.SYSADATA
+
+#
+# output data set for eqalangx side file
+assembler_debugPDS=${hlq}.EQALANGX
+
 #
 # List the data sets that need to be created and their creation options
 assembler_srcDatasets=${assembler_srcPDS},${assembler_macroPDS},${assembler_objPDS},${assembler_dbrmPDS}
@@ -26,8 +34,17 @@ assembler_srcOptions=cyl space(1,1) lrecl(80) dsorg(PO) recfm(F,B) dsntype(libra
 assembler_loadDatasets=${assembler_loadPDS}
 assembler_loadOptions=cyl space(1,1) dsorg(PO) recfm(U) blksize(32760) dsntype(library)
 
+#
+# temporary data set allocations
 assembler_tempOptions=cyl space(5,5) unit(vio) blksize(80) lrecl(80) recfm(f,b) new
 
+#
+# sysadata data set creation options
+assembler_sysadataOptions=cyl space(5,5) lrecl(32756) dsorg(PO) recfm(v,b) blksize(32760) dsntype(library)
+
+# eqalangx sidefile data set creation options
+assembler_sidefileOptions=cyl dsorg(PO) dsntype(library) space(5,5) unit(vio) blksize(27998) lrecl(1562) recfm(v,b)
+
 # Allocation of SYSMLSD Dataset used for extracting Compile Messages to Remote Error List
 assembler_compileErrorFeedbackXmlOptions=tracks space(200,40) dsorg(PS) blksize(27998) lrecl(16383) recfm(v,b) new keep
 
@@ -38,6 +55,7 @@ assembler_outputDatasets=${assembler_loadPDS},${assembler_dbrmPDS}
 # default assembler execs
 assembler_db2precompiler=DSNHPC
 assember_cicsprecompiler=DFHEAP1$
+assembler_eqalangx=EQALANGX
 assembler_pgm=ASMA90
 assembler_linkEditor=IEWBLINK
 

build-conf/README.md

@@ -76,12 +76,16 @@ assembler_srcPDS | Dataset to move assembler source files to from USS
 assembler_macroPDS | Dataset to move macro files to from USS
 assembler_objPDS | Dataset to create object decks in from Assembler step
 assembler_dbrmPDS | Dataset to create DB2 DBRM modules in from Assembler step
+assembler_sysadataPDS | Dataset to create  sysadata file that contains source and symbolic data about the program
+assembler_debugPDS | Dataset to create the eqalangx side file for the IBM Debug Tool
 assembler_loadPDS | Dataset to create load modules in from link edit step
 assembler_srcDataSets | Comma separated list of 'source' type data sets
 assembler_srcOptions | BPXWDYN creation options for creating 'source' type data sets
 assembler_loadDatasets | Comma separated list of 'load module' type data sets
 assembler_loadOptions | BPXWDYN creation options for 'load module' type data sets
 assembler_tempOptions | BPXWDYN creation options for temporary data sets
+assembler_sysadataOptions | BPXWDYN creation options for assembler sysadata data sets
+assembler_sidefileOptions | BPXWDYN creation options for eqalangx sidefile
 assembler_compileErrorFeedbackXmlOptions | BPXWDYN creation options for SYSXMLSD data set
 assembler_outputDatasets | List of output datasets to document deletions ** Can be overridden by a file property.
 assembler_pgm | MVS program name of the high level assembler

build-conf/Transfer.properties

@@ -1,23 +1,25 @@
 # Releng properties used by language/Transfer.groovy
 
 # Comma separated list of required build properties for Cobol.groovy
-transfer_requiredBuildProperties=transfer_srcPDS,transfer_srcOptions,\
+transfer_requiredBuildProperties=transfer_srcPDS,transfer_dsOptions,\
   transfer_deployType
 
 #
-# transfer source data sets
+# Transfer source data sets
 # Add additional dataset definitions, depending on your requirements
 #
 # Please note, that files in the repository require to be mapped by a PropertyMapping in file.properties
-# to one of the dataset definitions
+# to one of the dataset definitions. See samples/application-conf/file.properties
 #
 transfer_srcPDS=${hlq}.SOURCE
 transfer_jclPDS=${hlq}.JCL
 transfer_xmlPDS=${hlq}.XML
 
 #
 # dataset creation options
-transfer_srcOptions=cyl space(1,1) lrecl(80) dsorg(PO) recfm(F,B) dsntype(library)
-
-# List of output datasets to document deletions
-transfer_outputDatasets=${transfer_srcPDS}
+#
+# This is using the DBB PropertyMapping syntax allowing multiple mappings of
+#  target source dataset definitions to the required options
+# 
+transfer_dsOptions=cyl space(1,1) lrecl(80) dsorg(PO) recfm(F,B) dsntype(library) :: transfer_srcPDS, transfer_jclPDS
+transfer_dsOptions=cyl space(1,1) lrecl(144) dsorg(PO) recfm(F,B) dsntype(library) :: transfer_xmlPDS

build-conf/datasets.properties

@@ -71,3 +71,6 @@ SBZUSAMP=
 
 # REXX Compiler Data Sets. Example: REXX.V1R4.SFANLMD
 SFANLMD=
+
+# PD Tools Common Component load library. Example : PDTCC.V1R8.SIPVMODA 
+PDTCCMOD=

build.groovy

@@ -17,6 +17,7 @@ import groovy.cli.commons.*
 @Field def gitUtils= loadScript(new File("utilities/GitUtilities.groovy"))
 @Field def buildUtils= loadScript(new File("utilities/BuildUtilities.groovy"))
 @Field def impactUtils= loadScript(new File("utilities/ImpactUtilities.groovy"))
+@Field def reportingUtils= loadScript(new File("utilities/ReportingUtilities.groovy"))
 @Field def filePropUtils= loadScript(new File("utilities/FilePropUtilities.groovy"))
 @Field String hashPrefix = ':githash:'
 @Field String giturlPrefix = ':giturl:'
@@ -104,6 +105,12 @@ def initializeBuildProcess(String[] args) {
 	// verify required build properties
 	buildUtils.assertBuildProperties(props.requiredBuildProperties)
 
+	// evaluate preview flag to set the reportOnly
+	if (props.preview) {
+		println "** Running in reportOnly mode. Will process build options but not execute any steps."
+		props.put("dbb.command.reportOnly","true")
+	}
+
 	// create metadata store for this script
 	if (!props.userBuild) {
 		if (props.metadataStoreType == 'file')
@@ -190,13 +197,18 @@ def initializeBuildProcess(String[] args) {
 	// initialize build result (requires MetadataStore)
 	if (metadataStore) {
 		def buildResult = metadataStore.createBuildResult(props.applicationBuildGroup, props.applicationBuildLabel)
+		// set build state and status
 		buildResult.setState(buildResult.PROCESSING)
+		if (props.preview) buildResult.setStatus(4)
+
 		if (props.scanOnly) buildResult.setProperty('scanOnly', 'true')
 		if (props.fullBuild) buildResult.setProperty('fullBuild', 'true')
 		if (props.impactBuild) buildResult.setProperty('impactBuild', 'true')
 		if (props.topicBranchBuild) buildResult.setProperty('topicBranchBuild', 'true')
+		if (props.preview) buildResult.setProperty('preview', 'true')
+
 		if (props.buildFile) buildResult.setProperty('buildFile', XmlUtil.escapeXml(props.buildFile))
-
+				
 		println("** Build result created for BuildGroup:${props.applicationBuildGroup} BuildLabel:${props.applicationBuildLabel}")
 	}
 
@@ -231,7 +243,8 @@ options:
 	cli.m(longOpt:'mergeBuild', 'Flag indicating to build only changes which will be merged back to the mainBuildBranch.')	
 	cli.r(longOpt:'reset', 'Deletes the dependency collections and build result group from the MetadataStore')
 	cli.v(longOpt:'verbose', 'Flag to turn on script trace')
-
+	cli.pv(longOpt:'preview', 'Supplemental flag indicating to run build in preview mode without processing the execute commands')
+
 	// scan options
 	cli.s(longOpt:'scanOnly', 'Flag indicating to only scan source files for application without building anything (deprecated use --scanSource)')
 	cli.ss(longOpt:'scanSource', 'Flag indicating to only scan source files for application without building anything')
@@ -392,7 +405,8 @@ def populateBuildProperties(def opts) {
 	if (opts.v) props.verbose = 'true'
 	if (opts.b) props.baselineRef = opts.b
 	if (opts.m) props.mergeBuild = 'true'
-
+	if (opts.pv) props.preview = 'true'
+
 	// scan options
 	if (opts.s) props.scanOnly = 'true'
 	if (opts.ss) props.scanOnly = 'true'
@@ -492,9 +506,13 @@ def createBuildList() {
 	Set<String> changedBuildProperties = new HashSet<String>() // not yet used for any post-processing
 	String action = (props.scanOnly) || (props.scanLoadmodules) ? 'Scanning' : 'Building'
 
+	// check if preview sub-option
+	if (props.preview) { println "** --preview cli option provided. Processing all phases of the supplied build option, but will not execute the commands." }
+
 	// check if full build
 	if (props.fullBuild) {
 		println "** --fullBuild option selected. $action all programs for application ${props.application}"
+
 		buildSet = buildUtils.createFullBuildList()
 	}
 	// check if impact build
@@ -515,7 +533,7 @@ def createBuildList() {
 			println "*! Merge build requires a Filesystem or Db2 MetadataStore"
 		}
 	}
-
+		
 	// if build file present add additional files to build list (mandatory build list)
 	if (props.buildFile) {
 
@@ -596,18 +614,18 @@ def createBuildList() {
 	if (props.reportExternalImpacts && props.reportExternalImpacts.toBoolean()){
 		if (buildSet && changedFiles) {
 			println "** Perform analysis and reporting of external impacted files for the build list including changed files."
-			impactUtils.reportExternalImpacts(buildSet.plus(changedFiles))
+			reportingUtils.reportExternalImpacts(buildSet.plus(changedFiles))
 		}
 		else if(buildSet) {
 			println "** Perform analysis and reporting of external impacted files for the build list."
-			impactUtils.reportExternalImpacts(buildSet)
+			reportingUtils.reportExternalImpacts(buildSet)
 		}
 	}
 
 	// Document and validate concurrent changes
 	if (props.reportConcurrentChanges && props.reportConcurrentChanges.toBoolean()){
 		println "** Calculate and document concurrent changes."
-		impactUtils.calculateConcurrentChanges(buildSet)
+		reportingUtils.calculateConcurrentChanges(buildSet)
 	}
 
 	// document deletions in build report
@@ -675,7 +693,6 @@ def finalizeBuildProcess(Map args) {
 		buildResult.setState(buildResult.COMPLETE)
 
 
-
 		// store build result properties in BuildReport.json
 		PropertiesRecord buildReportRecord = new PropertiesRecord("DBB.BuildResultProperties")
 		def buildResultProps = buildResult.getPropertyNames()
@@ -688,7 +705,7 @@ def finalizeBuildProcess(Map args) {
 		// }
 		buildReport.addRecord(buildReportRecord)
 	}
-
+		
 	// create build report data file
 	def jsonOutputFile = new File("${props.buildOutDir}/BuildReport.json")
 	def buildReportEncoding = "UTF-8"
@@ -719,6 +736,7 @@ def finalizeBuildProcess(Map args) {
 	def state = (props.error) ? "ERROR" : "CLEAN"
 	println("** Build ended at $endTime")
 	println("** Build State : $state")
+	if (props.preview) println("** Build ran in preview mode.")
 	println("** Total files processed : ${args.count}")
 	println("** Total build time  : $duration\n")
 }