Update zAppBuild to support IDE passed dependency information (#141)

* added -df/--dependencyFile parameter and add to buildProperties * added functionality to check properties for userBuildDependencyFile, if present, skip dep resolution and copy files over to dependencyPDS line by line as listed in file * added conversion to absolute paths if paths are provided as relative to workspace * updated print description for user build dependency file * added some verbose statements to print the json file * added conversion to absolute paths if paths are provided as relative to workspace * removed scanner creation is userbuild && userbuild dep file is provided Co-authored-by: Luke Burgess <[email protected]>
IBM · Oct 31, 2021 · 5dd529a · 5dd529a
1 parent 8971bed
commit 5dd529a
Show file tree

Hide file tree

Showing 6 changed files with 192 additions and 8 deletions.
diff --git a/BUILD.md b/BUILD.md
@@ -43,6 +43,10 @@ $DBB_HOME/bin/groovyz build.groovy --workspace /u/build/repos --application app1
 **Use Code Coverage Headless Collector in zUnit Tests and specify parameters through command-line options (which override properties defined in ZunitConfig.properties)**
 ```
 $DBB_HOME/bin/groovyz build.groovy --workspace /u/build/repos --application app1 --outDir /u/build/out --hlq BUILD.APP1 --fullBuild --cc --cch localhost --ccp 8009 --cco "e=CCPDF"
+```
+ **Build one program using a [user build dependency file](samples/userBuildDependencyFile) predefining dependency information to skip DBB scans and dependency resolution.**
+```
+$DBB_HOME/bin/groovyz build.groovy --workspace /u/build/repos --application app1 --outDir /u/build/out --hlq BUILD.APP1 --userBuild --dependencyFile userBuildDependencyFile.json app1/cobol/epsmpmt.cbl
 ```
 
 ## Command Line Options Summary
@@ -103,6 +107,7 @@ web application credentials
 IDz/ZOD User Build options
  -u,--userBuild               Flag indicating running a user build
  -e,--errPrefix <arg>         Unique id used for IDz error message datasets
+ -df,--dependencyFile <arg>   Absolute or relative path (from workspace) to user build JSON file containing dependency information.
 
 utility options
  -help,--help                 Prints this message

diff --git a/build.groovy b/build.groovy
@@ -202,6 +202,9 @@ options:
 	// build framework options
 	cli.re(longOpt:'reportExternalImpacts', 'Flag to activate analysis and report of external impacted files within DBB collections')
 
+	// IDE user build dependency file options
+	cli.df(longOpt:'dependencyFile', args:1, 'Absolute or relative (from workspace) path to user build JSON file containing dependency information.')
+
 	// utility options
 	cli.help(longOpt:'help', 'Prints this message')
 
@@ -353,6 +356,8 @@ def populateBuildProperties(String[] args) {
 	if (opts.e) props.errPrefix = opts.e
 	if (opts.u) props.userBuild = 'true'
 	if (opts.t) props.team = opts.t
+	// support IDE passing dependency file parameter
+	if (opts.df) props.userBuildDependencyFile = opts.df
 
 	// set build file from first non-option argument
 	if (opts.arguments()) props.buildFile = opts.arguments()[0].trim()
@@ -378,6 +383,9 @@ def populateBuildProperties(String[] args) {
 		props.buildOutDir = ((props.createBuildOutputSubfolder && props.createBuildOutputSubfolder.toBoolean()) ? "${props.outDir}/${props.applicationBuildLabel}" : "${props.outDir}") as String
 	}
 
+	// Validate User Build Dependency file is used only with user build
+	if (props.userBuildDependencyFile) assert (props.userBuild) : "*! User Build Dependency File requires User Build option."
+
 	// Validate Build Properties  
 	if(props.reportExternalImpactsAnalysisDepths) assert (props.reportExternalImpactsAnalysisDepths == 'simple' || props.reportExternalImpactsAnalysisDepths == 'deep' ) : "*! Build Property props.reportExternalImpactsAnalysisDepths has an invalid value"
 	if(props.baselineRef) assert (props.impactBuild) : "*! Build Property props.baselineRef is exclusive to an impactBuild scenario"

diff --git a/samples/userBuildDependencyFile/README.md b/samples/userBuildDependencyFile/README.md
@@ -0,0 +1,20 @@
+## User Build Dependency File
+
+In order to increase performance of User Build running on ZD&T, the IDEs can **optionally** pass dependency information about the program being built to zAppBuild allowing it to skip running dependency resolution which depending on the size and number of build dependencies the program references can be time consuming on ZD&T platforms.
+
+### Dependency File Option
+Providing the following option when calling *build.groovy* will skip scanning and dependency resolution within zAppBuild.
+
+    --dependencyFile <pathToFile>
+    -df <pathToFile>
+If not provided, zAppBuild will run the traditional scan and dependency resolution on the build file.
+If it is provided, zAppBuild will skip scanning and resolution and refer to the dependencies and information from the file. 
+
+### Dependency File Location
+
+The location of the user build dependency file on USS is unimportant, as long as that path is correctly specified when passing the **-\-dependencyFile \<path>** option to zAppBuild. 
+
+### Additional Resources
+View the user build dependency file schema and a sample file using the links below. 
+##### [Dependency File Schema](schema.json)
+##### [Sample Dependency File](sample.json)
diff --git a/samples/userBuildDependencyFile/sample.json b/samples/userBuildDependencyFile/sample.json
@@ -0,0 +1,14 @@
+{
+    "fileName": "/u/burgess/dbb/dbb-zappbuild/samples/MortgageApplication/cobol/epscmort.cbl",
+    "isCICS": true,
+    "isSQL": true,
+    "isDLI": false,
+    "isMQ": false,
+    "dependencies": [
+        "/u/burgess/dbb/dbb-zappbuild/samples/MortgageApplication/copybook/epsmtcom.cpy",
+        "/u/burgess/dbb/dbb-zappbuild/samples/MortgageApplication/copybook/epsnbrpm.cpy",
+        "MortgageApplication/copybook/epsmtinp.cpy",
+        "MortgageApplication/copybook/epsmtout.cpy"
+    ],
+    "schemaVersion": "1.0"
+}
diff --git a/samples/userBuildDependencyFile/schema.json b/samples/userBuildDependencyFile/schema.json
@@ -0,0 +1,61 @@
+{
+    "$schema": "http://json-schema.org/draft-06/schema#",
+    "definitions": {
+        "UserBuildDependencyFile": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+                "fileName": { 
+                    "description": "Represents the absolute or relative path (from the sandbox) of the source file to build. This field should include the file extension (if applicable) and is case sensitive. ",
+                    "type": "string"
+                },
+                "isCICS": {
+                    "type": "boolean",
+                    "description": "Represents the existence of EXEC CICS statements in the program or its listed dependencies.",
+                    "default": "false"
+                },
+                "isSQL": {
+                    "type": "boolean",
+                    "description": "Represents the existence of EXEC CICS statements in the program or its listed dependencies.",
+                    "default": "false"
+                },
+                "isDLI": {
+                    "type": "boolean",
+                    "description": "Represents the existence of EXEC CICS statements in the program or its listed dependencies.",
+                    "default": "false"
+                },
+                "isMQ": {
+                    "type": "boolean",
+                    "description": "Represents the existence of EXEC CICS statements in the program or its listed dependencies.",
+                    "default": "false"
+                },
+                "dependencies": {
+                    "description": "An array of strings representing the list of program dependencies as paths. ",
+                    "type": "array",
+                    "minItems": 0,
+                    "uniqueItems": true,
+                    "items": {
+                        "description": "The absolute or relative path (from the sandbox) to a program dependency. Each entry should include the file extension (if applicable) and is case sensitive. ",
+                        "type": "string"
+                    }
+                },
+                "schemaVersion": {
+                    "description": "Represents the schema version of this JSON structure. ",
+                    "type": "string",
+                    "default": "1.0"
+                }
+            },
+            "required": [
+                "fileName",
+                "isCICS",
+                "isSQL",
+                "isDLI",
+                "isMQ",
+                "dependencies",
+                "schemaVersion"
+
+            ],
+            "title": ".userbuilddependencyfile"
+        }
+    }
+}
diff --git a/utilities/BuildUtilities.groovy b/utilities/BuildUtilities.groovy
@@ -76,8 +76,58 @@ def copySourceFiles(String buildFile, String srcPDS, String dependencyPDS, Depen
 				.execute()
 	}
 
-	// resolve the logical dependencies to physical files to copy to data sets
-	if (dependencyPDS && dependencyResolver) {
+	if (dependencyPDS && props.userBuildDependencyFile && props.userBuild) {
+		if (props.verbose) println "*** User Build Dependency File Detected. Skipping DBB Dependency Resolution."
+		// userBuildDependencyFile present (passed from the IDE)
+		// skip dependency resolution, extract dependencies from userBuildDependencyFile, and copy directly to dependencyPDS
+
+		// parse JSON and validate fields of userBuildDependencyFile
+		def depFileData = validateDependencyFile(buildFile, props.userBuildDependencyFile)
+
+		// Manually create logical file for the user build program
+		String lname = CopyToPDS.createMemberName(buildFile)
+		String language = props.getFileProperty('dbb.DependencyScanner.languageHint', buildFile) ?: 'UNKN'
+		LogicalFile lfile = new LogicalFile(lname, buildFile, language, depFileData.isCICS, depFileData.isSQL, depFileData.isDLI)
+		// save logical file to dependency resolver
+		if (dependencyResolver)
+			dependencyResolver.setLogicalFile(lfile)
+
+ 		// get list of dependencies from userBuildDependencyFile
+		List<String> dependencyPaths = depFileData.dependencies
+
+		// copy each dependency from USS to member of depedencyPDS
+		dependencyPaths.each { dependencyPath ->
+			// if dependency is relative, convert to absolute path
+			String dependencyLoc = getAbsolutePath(dependencyPath)
+
+			// only copy the dependency file once per script invocation
+			if (!copiedFileCache.contains(dependencyLoc)) {
+				copiedFileCache.add(dependencyLoc)
+				// create member name
+				String memberName = CopyToPDS.createMemberName(dependencyPath)
+				// retrieve zUnit playback file extension
+				zunitFileExtension = (props.zunit_playbackFileExtension) ? props.zunit_playbackFileExtension : null
+				// get index of last '.' in file path to extract the file extension
+				def extIndex = dependencyLoc.lastIndexOf('.')
+				if( zunitFileExtension && !zunitFileExtension.isEmpty() && (dependencyLoc.substring(extIndex).contains(zunitFileExtension))){
+					new CopyToPDS().file(new File(dependencyLoc))
+							.copyMode(CopyMode.BINARY)
+							.dataset(dependencyPDS)
+							.member(memberName)
+							.execute()
+				} 
+				else
+				{
+					new CopyToPDS().file(new File(dependencyLoc))
+							.dataset(dependencyPDS)
+							.member(memberName)
+							.execute()
+				}			
+			}
+		} 
+	}
+	else if (dependencyPDS && dependencyResolver) {
+		// resolve the logical dependencies to physical files to copy to data sets
 		List<PhysicalDependency> physicalDependencies = dependencyResolver.resolve()
 		if (props.verbose) {
 			println "*** Resolution rules for $buildFile:"
@@ -93,21 +143,22 @@ def copySourceFiles(String buildFile, String srcPDS, String dependencyPDS, Depen
 				// only copy the dependency file once per script invocation
 				if (!copiedFileCache.contains(physicalDependencyLoc)) {
 					copiedFileCache.add(physicalDependencyLoc)
-
+					// create member name
+					String memberName = CopyToPDS.createMemberName(physicalDependency.getFile())
 					//retrieve zUnitFileExtension plbck
 					zunitFileExtension = (props.zunit_playbackFileExtension) ? props.zunit_playbackFileExtension : null
 
 					if( zunitFileExtension && !zunitFileExtension.isEmpty() && ((physicalDependency.getFile().substring(physicalDependency.getFile().indexOf("."))).contains(zunitFileExtension))){
 						new CopyToPDS().file(new File(physicalDependencyLoc))
 								.copyMode(CopyMode.BINARY)
 								.dataset(dependencyPDS)
-								.member(CopyToPDS.createMemberName(physicalDependency.getFile()))
+								.member(memberName)
 								.execute()
 					} else
 					{
 						new CopyToPDS().file(new File(physicalDependencyLoc))
 								.dataset(dependencyPDS)
-								.member(CopyToPDS.createMemberName(physicalDependency.getFile()))
+								.member(memberName)
 								.execute()
 					}
 				}
@@ -202,12 +253,14 @@ def updateBuildResult(Map args) {
 def createDependencyResolver(String buildFile, String rules) {
 	if (props.verbose) println "*** Creating dependency resolver for $buildFile with $rules rules"
 
-	def scanner = getScanner(buildFile)
-
 	// create a dependency resolver for the build file
 	DependencyResolver resolver = new DependencyResolver().file(buildFile)
 			.sourceDir(props.workspace)
-			.scanner(scanner)
+
+	// add scanner if userBuild Dep File not provided, or not a user build
+	if (!props.userBuildDependencyFile || !props.userBuild)
+		resolver.setScanner(getScanner(buildFile))
+
 	// add resolution rules
 	if (rules)
 		resolver.setResolutionRules(parseResolutionRules(rules))
@@ -457,3 +510,26 @@ def getDeployType(String langQualifier, String buildFile, LogicalFile logicalFil
 	}
 	return deployType
 }
+/*
+ * parse and validates the user build dependency file 
+ * returns a parsed json object 
+ */
+def validateDependencyFile(String buildFile, String depFilePath) {
+	// if depFilePath is relatvie, convert to absolute path
+	depFilePath = getAbsolutePath(depFilePath)
+	File depFile = new File(depFilePath)
+	assert depFile.exists() : "*! Dependency file not found: ${depFilePath}"
+	JsonSlurper slurper = new groovy.json.JsonSlurper()
+	if (props.verbose) println "Dependency File (${depFilePath}): \n" + groovy.json.JsonOutput.prettyPrint(depFile.getText())
+	// parse dependency File
+	def depFileData = slurper.parse(depFile)
+
+	/* Begin Validation */ 
+	String[] reqDepFileProps = ["fileName", "isCICS", "isSQL", "isDLI", "isMQ", "dependencies", "schemaVersion"]
+	reqDepFileProps.each { depFileProp ->
+		assert depFileData."${depFileProp}" != null : "*! Missing required dependency file field '$depFileProp'"
+	}
+	// validate that depFileData.fileName == buildFile
+	assert getAbsolutePath(depFileData.fileName) == getAbsolutePath(buildFile) : "*! Dependency file mismatch: fileName does not match build file"
+	return depFileData // return the parsed JSON object
+}